diff --git a/conv/j2t/conv_test.go b/conv/j2t/conv_test.go index a63d6e7a..1c3f03f1 100644 --- a/conv/j2t/conv_test.go +++ b/conv/j2t/conv_test.go @@ -37,7 +37,7 @@ import ( "github.com/cloudwego/dynamicgo/conv" "github.com/cloudwego/dynamicgo/http" - sjson "github.com/cloudwego/dynamicgo/internal/json" + sjson "github.com/bytedance/sonic/ast" "github.com/cloudwego/dynamicgo/internal/native" "github.com/cloudwego/dynamicgo/internal/native/types" "github.com/cloudwego/dynamicgo/internal/rt" diff --git a/conv/t2j/conv_test.go b/conv/t2j/conv_test.go index 1afb729f..b56af8c9 100644 --- a/conv/t2j/conv_test.go +++ b/conv/t2j/conv_test.go @@ -31,15 +31,16 @@ import ( "time" "unsafe" + sjson "github.com/bytedance/sonic/ast" "github.com/cloudwego/dynamicgo/conv" "github.com/cloudwego/dynamicgo/http" - sjson "github.com/cloudwego/dynamicgo/internal/json" "github.com/cloudwego/dynamicgo/internal/util_test" "github.com/cloudwego/dynamicgo/meta" kbase "github.com/cloudwego/dynamicgo/testdata/kitex_gen/base" "github.com/cloudwego/dynamicgo/testdata/kitex_gen/example3" "github.com/cloudwego/dynamicgo/thrift" "github.com/cloudwego/dynamicgo/thrift/base" + "github.com/davecgh/go-spew/spew" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -642,6 +643,7 @@ func TestOptionalDefaultValue(t *testing.T) { ctx := context.WithValue(context.Background(), conv.CtxKeyHTTPResponse, resp) out, err := cv.Do(ctx, desc, in) require.NoError(t, err) + spew.Dump(string(out)) act := &example3.ExampleOptionalDefaultValue{} err = json.Unmarshal(out, act) require.NoError(t, err) diff --git a/internal/json/api_amd64.go b/internal/json/api_amd64.go index eff2af27..87101854 100644 --- a/internal/json/api_amd64.go +++ b/internal/json/api_amd64.go @@ -1,21 +1,19 @@ //go:build amd64 && go1.16 // +build amd64,go1.16 -/** - * Copyright 2023 CloudWeGo Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +// Copyright 2023 CloudWeGo Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package json @@ -23,25 +21,14 @@ import ( "runtime" "unsafe" - "github.com/bytedance/sonic/encoder" "github.com/chenzhuoyu/base64x" "github.com/cloudwego/dynamicgo/internal/native" "github.com/cloudwego/dynamicgo/internal/native/types" "github.com/cloudwego/dynamicgo/internal/rt" - uq "github.com/cloudwego/dynamicgo/internal/unquote" ) -var typeByte = rt.UnpackEface(byte(0)).Type -func quote(buf *[]byte, val string) { - *buf = append(*buf, '"') - if len(val) == 0 { - *buf = append(*buf, '"') - } - NoQuote(buf, val) - - *buf = append(*buf, '"') -} +var typeByte = rt.UnpackEface(byte(0)).Type //go:nocheckptr // NoQuote only escape inner '\' and '"' of one string, but it does add quotes besides string. @@ -78,44 +65,10 @@ func NoQuote(buf *[]byte, val string) { runtime.KeepAlive(sp) } -func unquote(src string) (string, types.ParsingError) { - return uq.String(src) -} - -func decodeBase64(src string) ([]byte, error) { - return base64x.StdEncoding.DecodeString(src) -} - -func encodeBase64(src []byte) string { - return base64x.StdEncoding.EncodeToString(src) -} - -func (self *Parser) decodeValue() (val types.JsonState) { - sv := (*rt.GoString)(unsafe.Pointer(&self.s)) - self.p = native.Value(sv.Ptr, sv.Len, self.p, &val, 0) - return -} - -func (self *Parser) skip() (int, types.ParsingError) { - fsm := types.NewStateMachine() - start := native.SkipOne(&self.s, &self.p, fsm) - types.FreeStateMachine(fsm) - - if start < 0 { - return self.p, types.ParsingError(-start) - } - return start, 0 -} - -func (self *Node) encodeInterface(buf *[]byte) error { - //WARN: NOT compatible with json.Encoder - return encoder.EncodeInto(buf, self.packAny(), 0) -} - func i64toa(buf *[]byte, val int64) int { rt.GuardSlice(buf, types.MaxInt64StringLen) s := len(*buf) - ret := native.I64toa((*byte)(rt.IndexPtr((*rt.GoSlice)(unsafe.Pointer(buf)).Ptr, byteType.Size, s)), val) + ret := native.I64toa((*byte)(rt.IndexPtr((*rt.GoSlice)(unsafe.Pointer(buf)).Ptr, typeByte.Size, s)), val) if ret < 0 { *buf = append((*buf)[s:], '0') return 1 @@ -127,11 +80,15 @@ func i64toa(buf *[]byte, val int64) int { func f64toa(buf *[]byte, val float64) int { rt.GuardSlice(buf, types.MaxFloat64StringLen) s := len(*buf) - ret := native.F64toa((*byte)(rt.IndexPtr((*rt.GoSlice)(unsafe.Pointer(buf)).Ptr, byteType.Size, s)), val) + ret := native.F64toa((*byte)(rt.IndexPtr((*rt.GoSlice)(unsafe.Pointer(buf)).Ptr, typeByte.Size, s)), val) if ret < 0 { *buf = append((*buf)[s:], '0') return 1 } *buf = (*buf)[:s+ret] return ret +} + +func encodeBase64(src []byte) string { + return base64x.StdEncoding.EncodeToString(src) } \ No newline at end of file diff --git a/internal/json/api_amd64_test.go b/internal/json/api_amd64_test.go deleted file mode 100644 index 88e6d272..00000000 --- a/internal/json/api_amd64_test.go +++ /dev/null @@ -1,71 +0,0 @@ -//go:build amd64 && go1.16 -// +build amd64,go1.16 - -/** - * Copyright 2023 CloudWeGo Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package json - -import ( - "testing" - - "github.com/bytedance/sonic/encoder" - "github.com/cloudwego/dynamicgo/internal/native" - "github.com/cloudwego/dynamicgo/internal/native/types" - "github.com/stretchr/testify/assert" -) - -func TestSortNodeTwitter(t *testing.T) { - root, err := NewSearcher(_TwitterJson).GetByPath() - if err != nil { - t.Fatal(err) - } - obj, err := root.MapUseNumber() - if err != nil { - t.Fatal(err) - } - exp, err := encoder.Encode(obj, encoder.SortMapKeys) - if err != nil { - t.Fatal(err) - } - if err := root.SortKeys(true); err != nil { - t.Fatal(err) - } - act, err := root.MarshalJSON() - if err != nil { - t.Fatal(err) - } - assert.Equal(t, len(exp), len(act)) - assert.Equal(t, string(exp), string(act)) -} - -func BenchmarkSkipValue(b *testing.B) { - var src = ` [ 1, "[", 3, "\" \\[", [ [ "", { } ], { "a": [ ] } ] ] ` - b.Run("dyanmicgo", func(b *testing.B) { - for i := 0; i < b.N; i++ { - _, _ = SkipValue(src, 0) - } - }) - b.Run("native", func(b *testing.B) { - var fsm = types.NewStateMachine() - var p int - for i := 0; i < b.N; i++ { - fsm.Sp = 0 - p = 0 - _ = native.SkipOne(&src, &p, fsm) - } - }) -} \ No newline at end of file diff --git a/internal/json/api_compat.go b/internal/json/api_compat.go index b35aa329..e2e1e374 100644 --- a/internal/json/api_compat.go +++ b/internal/json/api_compat.go @@ -1,85 +1,105 @@ //go:build !amd64 || !go1.16 // +build !amd64 !go1.16 -/** - * Copyright 2023 CloudWeGo Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +// Copyright 2023 CloudWeGo Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package json import ( - "strconv" "encoding/base64" - "encoding/json" - - "github.com/cloudwego/dynamicgo/internal/native/types" - "github.com/cloudwego/dynamicgo/internal/rt" + "strconv" + "unicode/utf8" + _ "unsafe" ) -func quote(buf *[]byte, val string) { - *buf = append(*buf, '"') - quoteString(buf, val) - *buf = append(*buf, '"') -} - func NoQuote(buf *[]byte, val string) { quoteString(buf, val) } -func unquote(src string) (string, types.ParsingError) { - sp := rt.IndexChar(src, -1) - out, ok := unquoteBytes(rt.BytesFrom(sp, len(src)+2, len(src)+2)) - if !ok { - return "", types.ERR_INVALID_ESCAPE - } - return rt.Mem2Str(out), 0 -} - -func decodeBase64(src string) ([]byte, error) { - return base64.StdEncoding.DecodeString(src) -} +var ( + //go:linkname safeSet encoding/json.safeSet + safeSet [utf8.RuneSelf]bool -func encodeBase64(src []byte) string { - return base64.StdEncoding.EncodeToString(src) -} + //go:linkname hex encoding/json.hex + hex string +) -func (self *Parser) decodeValue() (val types.JsonState) { - e, v := DecodeValue(self.s, self.p) - if e < 0 { - return v +func quoteString(e *[]byte, s string) { + start := 0 + for i := 0; i < len(s); { + if b := s[i]; b < utf8.RuneSelf { + if safeSet[b] { + i++ + continue + } + if start < i { + *e = append(*e, s[start:i]...) + } + *e = append(*e, '\\') + switch b { + case '\\', '"': + *e = append(*e, b) + case '\n': + *e = append(*e, 'n') + case '\r': + *e = append(*e, 'r') + case '\t': + *e = append(*e, 't') + default: + // This encodes bytes < 0x20 except for \t, \n and \r. + // If escapeHTML is set, it also escapes <, >, and & + // because they can lead to security holes when + // user-controlled strings are rendered into JSON + // and served to some browsers. + *e = append(*e, `u00`...) + *e = append(*e, hex[b>>4]) + *e = append(*e, hex[b&0xF]) + } + i++ + start = i + continue + } + c, size := utf8.DecodeRuneInString(s[i:]) + // if c == utf8.RuneError && size == 1 { + // if start < i { + // e.Write(s[start:i]) + // } + // e.WriteString(`\ufffd`) + // i += size + // start = i + // continue + // } + if c == '\u2028' || c == '\u2029' { + if start < i { + *e = append(*e, s[start:i]...) + } + *e = append(*e, `\u202`...) + *e = append(*e, hex[c&0xF]) + i += size + start = i + continue + } + i += size } - self.p = e - return v -} - -func (self *Parser) skip() (int, types.ParsingError) { - e, s := SkipValue(self.s, self.p) - if e < 0 { - return self.p, types.ParsingError(-e) + if start < len(s) { + *e = append(*e, s[start:]...) } - self.p = e - return s, 0 } -func (self *Node) encodeInterface(buf *[]byte) error { - out, err := json.Marshal(self.packAny()) - if err != nil { - return err - } - *buf = append(*buf, out...) - return nil +func encodeBase64(src []byte) string { + return base64.StdEncoding.EncodeToString(src) } func i64toa(buf *[]byte, v int64) { diff --git a/internal/json/asm.s b/internal/json/asm.s deleted file mode 100644 index e69de29b..00000000 diff --git a/internal/json/decode.go b/internal/json/decode.go deleted file mode 100644 index 0730ca30..00000000 --- a/internal/json/decode.go +++ /dev/null @@ -1,452 +0,0 @@ -/** - * Copyright 2023 CloudWeGo Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package json - -import ( - "encoding/base64" - "runtime" - "strconv" - "unsafe" - - "github.com/cloudwego/dynamicgo/internal/native/types" - "github.com/cloudwego/dynamicgo/internal/rt" -) - -const _blankCharsMask = (1 << ' ') | (1 << '\t') | (1 << '\r') | (1 << '\n') - -const ( - bytesNull = "null" - bytesTrue = "true" - bytesFalse = "false" - bytesObject = "{}" - bytesArray = "[]" -) - -func IsSpace(c byte) bool { - return (int(1<= se { - return -int(types.ERR_EOF) - } - runtime.KeepAlive(src) - return int(sp - uintptr(rt.IndexChar(src, 0))) -} - -func decodeNull(src string, pos int) (ret int) { - ret = pos + 4 - if ret > len(src) { - return -int(types.ERR_EOF) - } - if src[pos:ret] == bytesNull { - return ret - } else { - return -int(types.ERR_INVALID_CHAR) - } -} - -func decodeTrue(src string, pos int) (ret int) { - ret = pos + 4 - if ret > len(src) { - return -int(types.ERR_EOF) - } - if src[pos:ret] == bytesTrue { - return ret - } else { - return -int(types.ERR_INVALID_CHAR) - } - -} - -func decodeFalse(src string, pos int) (ret int) { - ret = pos + 5 - if ret > len(src) { - return -int(types.ERR_EOF) - } - if src[pos:ret] == bytesFalse { - return ret - } - return -int(types.ERR_INVALID_CHAR) -} - -func decodeString(src string, pos int) (ret int, v string) { - ret, ep := skipString(src, pos) - if ep == -1 { - (*rt.GoString)(unsafe.Pointer(&v)).Ptr = rt.IndexChar(src, pos+1) - (*rt.GoString)(unsafe.Pointer(&v)).Len = ret - pos - 2 - return ret, v - } - - vv, ok := unquoteBytes(rt.Str2Mem(src[pos:ret])) - if !ok { - return -int(types.ERR_INVALID_CHAR), "" - } - - runtime.KeepAlive(src) - return ret, rt.Mem2Str(vv) -} - -func decodeBinary(src string, pos int) (ret int, v []byte) { - var vv string - ret, vv = decodeString(src, pos) - if ret < 0 { - return ret, nil - } - var err error - v, err = base64.StdEncoding.DecodeString(vv) - if err != nil { - return -int(types.ERR_INVALID_CHAR), nil - } - return ret, v -} - -func isDigit(c byte) bool { - return c >= '0' && c <= '9' -} - -//go:nocheckptr -func decodeInt64(src string, pos int) (ret int, v int64, err error) { - sp := uintptr(rt.IndexChar(src, pos)) - ss := uintptr(sp) - se := uintptr(rt.IndexChar(src, len(src))) - if uintptr(sp) >= se { - return -int(types.ERR_EOF), 0, nil - } - - if c := *(*byte)(unsafe.Pointer(sp)); c == '-' { - sp += 1 - } - if sp == se { - return -int(types.ERR_EOF), 0, nil - } - - for ; sp < se; sp += uintptr(1) { - if !isDigit(*(*byte)(unsafe.Pointer(sp))) { - break - } - } - - if sp < se { - if c := *(*byte)(unsafe.Pointer(sp)); c == '.' || c == 'e' || c == 'E' { - return -int(types.ERR_INVALID_NUMBER_FMT), 0, nil - } - } - - var vv string - ret = int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr)) - (*rt.GoString)(unsafe.Pointer(&vv)).Ptr = unsafe.Pointer(ss) - (*rt.GoString)(unsafe.Pointer(&vv)).Len = ret - pos - - v, err = strconv.ParseInt(vv, 10, 64) - if err != nil { - //NOTICE: allow overflow here - if err.(*strconv.NumError).Err == strconv.ErrRange { - return ret, 0, err - } - return -int(types.ERR_INVALID_CHAR), 0, err - } - - runtime.KeepAlive(src) - return ret, v, nil -} - -func isNumberChars(c byte) bool { - return (c >= '0' && c <= '9') || c == '+' || c == '-' || c == 'e' || c == 'E' || c == '.' -} - -//go:nocheckptr -func decodeFloat64(src string, pos int) (ret int, v float64, err error) { - sp := uintptr(rt.IndexChar(src, pos)) - ss := uintptr(sp) - se := uintptr(rt.IndexChar(src, len(src))) - if uintptr(sp) >= se { - return -int(types.ERR_EOF), 0, nil - } - - if c := *(*byte)(unsafe.Pointer(sp)); c == '-' { - sp += 1 - } - if sp == se { - return -int(types.ERR_EOF), 0, nil - } - - for ; sp < se; sp += uintptr(1) { - if !isNumberChars(*(*byte)(unsafe.Pointer(sp))) { - break - } - } - - var vv string - ret = int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr)) - (*rt.GoString)(unsafe.Pointer(&vv)).Ptr = unsafe.Pointer(ss) - (*rt.GoString)(unsafe.Pointer(&vv)).Len = ret - pos - - v, err = strconv.ParseFloat(vv, 64) - if err != nil { - //NOTICE: allow overflow here - if err.(*strconv.NumError).Err == strconv.ErrRange { - return ret, 0, err - } - return -int(types.ERR_INVALID_CHAR), 0, err - } - - runtime.KeepAlive(src) - return ret, v, nil -} - -func DecodeValue(src string, pos int) (ret int, v types.JsonState) { - pos = SkipBlank(src, pos) - if pos < 0 { - return pos, types.JsonState{Vt: types.ValueType(pos)} - } - switch c := src[pos]; c { - case 'n': - ret = decodeNull(src, pos) - if ret < 0 { - return ret, types.JsonState{Vt: types.ValueType(ret)} - } - return ret, types.JsonState{Vt: types.V_NULL} - case '"': - var ep int - ret, ep = skipString(src, pos) - if ret < 0 { - return ret, types.JsonState{Vt: types.ValueType(ret)} - } - return ret, types.JsonState{Vt: types.V_STRING, Iv: int64(pos + 1), Ep: int64(ep)} - case '{': - return pos + 1, types.JsonState{Vt: types.V_OBJECT} - case '[': - return pos + 1, types.JsonState{Vt: types.V_ARRAY} - case 't': - ret = decodeTrue(src, pos) - if ret < 0 { - return ret, types.JsonState{Vt: types.ValueType(ret)} - } - return ret, types.JsonState{Vt: types.V_TRUE} - case 'f': - ret = decodeFalse(src, pos) - if ret < 0 { - return ret, types.JsonState{Vt: types.ValueType(ret)} - } - return ret, types.JsonState{Vt: types.V_FALSE} - case '-', '+', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': - var iv int64 - ret, iv, _ = decodeInt64(src, pos) - if ret >= 0 { - return ret, types.JsonState{Vt: types.V_INTEGER, Iv: iv, Ep: int64(pos)} - } else if ret != -int(types.ERR_INVALID_NUMBER_FMT) { - return ret, types.JsonState{Vt: types.ValueType(ret)} - } - var fv float64 - ret, fv, _ = decodeFloat64(src, pos) - if ret >= 0 { - return ret, types.JsonState{Vt: types.V_DOUBLE, Dv: fv, Ep: int64(pos)} - } else { - return ret, types.JsonState{Vt: types.ValueType(ret)} - } - default: - return -int(types.ERR_INVALID_CHAR), types.JsonState{Vt: -types.ValueType(types.ERR_INVALID_CHAR)} - } -} - -//go:nocheckptr -func skipNumber(src string, pos int) (ret int) { - sp := uintptr(rt.IndexChar(src, pos)) - se := uintptr(rt.IndexChar(src, len(src))) - if uintptr(sp) >= se { - return -int(types.ERR_EOF) - } - - if c := *(*byte)(unsafe.Pointer(sp)); c == '-' { - sp += 1 - } - ss := sp - - var pointer bool - var exponent bool - var lastIsDigit bool - var nextNeedDigit = true - - for ; sp < se; sp += uintptr(1) { - c := *(*byte)(unsafe.Pointer(sp)) - if isDigit(c) { - lastIsDigit = true - nextNeedDigit = false - continue - } else if nextNeedDigit { - return -int(types.ERR_INVALID_CHAR) - } else if c == '.' { - if !lastIsDigit || pointer || sp == ss { - return -int(types.ERR_INVALID_CHAR) - } - pointer = true - lastIsDigit = false - nextNeedDigit = true - continue - } else if c == 'e' || c == 'E' { - if !lastIsDigit || exponent { - return -int(types.ERR_INVALID_CHAR) - } - if sp == se-1 { - return -int(types.ERR_EOF) - } - exponent = true - lastIsDigit = false - nextNeedDigit = false - continue - } else if c == '-' || c == '+' { - if prev := *(*byte)(unsafe.Pointer(sp - 1)); prev != 'e' && prev != 'E' { - return -int(types.ERR_INVALID_CHAR) - } - lastIsDigit = false - nextNeedDigit = true - continue - } else { - break - } - } - - if nextNeedDigit { - return -int(types.ERR_EOF) - } - - runtime.KeepAlive(src) - return int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr)) -} - -//go:nocheckptr -func skipString(src string, pos int) (ret int, ep int) { - if pos+1 >= len(src) { - return -int(types.ERR_EOF), -1 - } - - sp := uintptr(rt.IndexChar(src, pos)) - se := uintptr(rt.IndexChar(src, len(src))) - - if *(*byte)(unsafe.Pointer(sp)) != '"' { - return -int(types.ERR_INVALID_CHAR), -1 - } - sp += 1 - - ep = -1 - for sp < se { - c := *(*byte)(unsafe.Pointer(sp)) - if c == '\\' { - if ep == -1 { - ep = int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr)) - } - sp += 2 - continue - } - sp += 1 - if c == '"' { - break - } - } - - if sp > se { - return -int(types.ERR_EOF), -1 - } - - runtime.KeepAlive(src) - return int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr)), ep -} - -//go:nocheckptr -func skipPair(src string, pos int, lchar byte, rchar byte) (ret int) { - if pos+1 >= len(src) { - return -int(types.ERR_EOF) - } - - sp := uintptr(rt.IndexChar(src, pos)) - se := uintptr(rt.IndexChar(src, len(src))) - - if *(*byte)(unsafe.Pointer(sp)) != lchar { - return -int(types.ERR_INVALID_CHAR) - } - - sp += 1 - nbrace := 1 - inquote := false - - for sp < se { - c := *(*byte)(unsafe.Pointer(sp)) - if c == '\\' { - sp += 2 - continue - } else if c == '"' { - inquote = !inquote - } else if c == lchar { - if !inquote { - nbrace += 1 - } - } else if c == rchar { - if !inquote { - nbrace -= 1 - if nbrace == 0 { - sp += 1 - break - } - } - } - sp += 1 - } - - if nbrace != 0 { - return -int(types.ERR_INVALID_CHAR) - } - - runtime.KeepAlive(src) - return int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr)) -} - -func SkipValue(src string, pos int) (ret int, start int) { - pos = SkipBlank(src, pos) - if pos < 0 { - return pos, -1 - } - switch c := src[pos]; c { - case 'n': - ret = decodeNull(src, pos) - case '"': - ret, _ = skipString(src, pos) - case '{': - ret = skipPair(src, pos, '{', '}') - case '[': - ret = skipPair(src, pos, '[', ']') - case 't': - ret = decodeTrue(src, pos) - case 'f': - ret = decodeFalse(src, pos) - case '-', '+', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': - ret = skipNumber(src, pos) - default: - ret = -int(types.ERR_INVALID_CHAR) - } - return ret, pos -} diff --git a/internal/json/encode.go b/internal/json/encode.go deleted file mode 100644 index c27ee88c..00000000 --- a/internal/json/encode.go +++ /dev/null @@ -1,266 +0,0 @@ -/** - * Copyright 2023 CloudWeGo Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package json - -import ( - "sync" - "unicode/utf8" -) - -const ( - _MaxBuffer = 1024 // 1KB buffer size -) - -func quoteString(e *[]byte, s string) { - start := 0 - for i := 0; i < len(s); { - if b := s[i]; b < utf8.RuneSelf { - if safeSet[b] { - i++ - continue - } - if start < i { - *e = append(*e, s[start:i]...) - } - *e = append(*e, '\\') - switch b { - case '\\', '"': - *e = append(*e, b) - case '\n': - *e = append(*e, 'n') - case '\r': - *e = append(*e, 'r') - case '\t': - *e = append(*e, 't') - default: - // This encodes bytes < 0x20 except for \t, \n and \r. - // If escapeHTML is set, it also escapes <, >, and & - // because they can lead to security holes when - // user-controlled strings are rendered into JSON - // and served to some browsers. - *e = append(*e, `u00`...) - *e = append(*e, hex[b>>4]) - *e = append(*e, hex[b&0xF]) - } - i++ - start = i - continue - } - c, size := utf8.DecodeRuneInString(s[i:]) - // if c == utf8.RuneError && size == 1 { - // if start < i { - // e.Write(s[start:i]) - // } - // e.WriteString(`\ufffd`) - // i += size - // start = i - // continue - // } - if c == '\u2028' || c == '\u2029' { - if start < i { - *e = append(*e, s[start:i]...) - } - *e = append(*e, `\u202`...) - *e = append(*e, hex[c&0xF]) - i += size - start = i - continue - } - i += size - } - if start < len(s) { - *e = append(*e, s[start:]...) - } -} - -var bytesPool = sync.Pool{} - -func (self *Node) MarshalJSON() ([]byte, error) { - buf := newBuffer() - err := self.encode(buf) - if err != nil { - freeBuffer(buf) - return nil, err - } - - ret := make([]byte, len(*buf)) - copy(ret, *buf) - freeBuffer(buf) - return ret, err -} - -func newBuffer() *[]byte { - if ret := bytesPool.Get(); ret != nil { - return ret.(*[]byte) - } else { - buf := make([]byte, 0, _MaxBuffer) - return &buf - } -} - -func freeBuffer(buf *[]byte) { - *buf = (*buf)[:0] - bytesPool.Put(buf) -} - -func (self *Node) encode(buf *[]byte) error { - if self.IsRaw() { - return self.encodeRaw(buf) - } - switch self.Type() { - case V_NONE: - return ErrNotExist - case V_ERROR: - return self.Check() - case V_NULL: - return self.encodeNull(buf) - case V_TRUE: - return self.encodeTrue(buf) - case V_FALSE: - return self.encodeFalse(buf) - case V_ARRAY: - return self.encodeArray(buf) - case V_OBJECT: - return self.encodeObject(buf) - case V_STRING: - return self.encodeString(buf) - case V_NUMBER: - return self.encodeNumber(buf) - case V_ANY: - return self.encodeInterface(buf) - default: - return ErrUnsupportType - } -} - -func (self *Node) encodeRaw(buf *[]byte) error { - raw, err := self.Raw() - if err != nil { - return err - } - *buf = append(*buf, raw...) - return nil -} - -func (self *Node) encodeNull(buf *[]byte) error { - *buf = append(*buf, bytesNull...) - return nil -} - -func (self *Node) encodeTrue(buf *[]byte) error { - *buf = append(*buf, bytesTrue...) - return nil -} - -func (self *Node) encodeFalse(buf *[]byte) error { - *buf = append(*buf, bytesFalse...) - return nil -} - -func (self *Node) encodeNumber(buf *[]byte) error { - str := addr2str(self.p, self.v) - *buf = append(*buf, str...) - return nil -} - -func (self *Node) encodeString(buf *[]byte) error { - if self.v == 0 { - *buf = append(*buf, '"', '"') - return nil - } - - quote(buf, addr2str(self.p, self.v)) - return nil -} - -func (self *Node) encodeArray(buf *[]byte) error { - if self.isLazy() { - if err := self.skipAllIndex(); err != nil { - return err - } - } - - nb := self.len() - if nb == 0 { - *buf = append(*buf, bytesArray...) - return nil - } - - *buf = append(*buf, '[') - - var p = (*Node)(self.p) - err := p.encode(buf) - if err != nil { - return err - } - for i := 1; i < nb; i++ { - *buf = append(*buf, ',') - p = p.unsafe_next() - err := p.encode(buf) - if err != nil { - return err - } - } - - *buf = append(*buf, ']') - return nil -} - -func (self *Pair) encode(buf *[]byte) error { - if len(*buf) == 0 { - *buf = append(*buf, '"', '"', ':') - return self.Value.encode(buf) - } - - quote(buf, self.Key) - *buf = append(*buf, ':') - - return self.Value.encode(buf) -} - -func (self *Node) encodeObject(buf *[]byte) error { - if self.isLazy() { - if err := self.skipAllKey(); err != nil { - return err - } - } - - nb := self.len() - if nb == 0 { - *buf = append(*buf, bytesObject...) - return nil - } - - *buf = append(*buf, '{') - - var p = (*Pair)(self.p) - err := p.encode(buf) - if err != nil { - return err - } - for i := 1; i < nb; i++ { - *buf = append(*buf, ',') - p = p.unsafe_next() - err := p.encode(buf) - if err != nil { - return err - } - } - - *buf = append(*buf, '}') - return nil -} diff --git a/internal/json/encode_test.go b/internal/json/encode_test.go deleted file mode 100644 index 1f26bdfd..00000000 --- a/internal/json/encode_test.go +++ /dev/null @@ -1,208 +0,0 @@ -/** - * Copyright 2023 CloudWeGo Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package json - -import ( - "encoding/json" - "runtime" - "sync" - "testing" - - "github.com/cloudwego/dynamicgo/internal/native/types" - "github.com/stretchr/testify/assert" -) - -func TestGC_Encode(t *testing.T) { - if debugSyncGC { - return - } - root, err := NewSearcher(_TwitterJson).GetByPath() - if err != nil { - t.Fatal(err) - } - root.LoadAll() - _, err = root.MarshalJSON() - if err != nil { - t.Fatal(err) - } - wg := &sync.WaitGroup{} - N := 10000 - for i := 0; i < N; i++ { - wg.Add(1) - go func(wg *sync.WaitGroup) { - defer wg.Done() - root, err := NewSearcher(_TwitterJson).GetByPath() - if err != nil { - t.Fatal(err) - } - root.Load() - _, err = root.MarshalJSON() - if err != nil { - t.Fatal(err) - } - runtime.GC() - }(wg) - } - wg.Wait() -} - -func TestEncodeValue(t *testing.T) { - obj := new(_TwitterStruct) - if err := json.Unmarshal([]byte(_TwitterJson), obj); err != nil { - t.Fatal(err) - } - // buf, err := encoder.Encode(obj, encoder.EscapeHTML|encoder.SortMapKeys) - buf, err := json.Marshal(obj) - if err != nil { - t.Fatal(err) - } - // quote, err := json.Marshal(_TwitterJson) - if err != nil { - t.Fatal(err) - } - type Case struct { - node Node - exp string - err bool - } - input := []Case{ - // {NewNull(), "null", false}, - // {NewBool(true), "true", false}, - // {NewBool(false), "false", false}, - // {NewNumber("0.0"), "0.0", false}, - // {NewString(""), `""`, false}, - // {NewString(`\"\"`), `"\\\"\\\""`, false}, - // {NewString(_TwitterJson), string(quote), false}, - // {NewArray([]Node{}), "[]", false}, - // {NewArray([]Node{NewBool(true), NewString("true"), NewString("\t")}), `[true,"true","\t"]`, false}, - // {NewObject([]Pair{Pair{"a", NewNull()}, Pair{"b", NewNumber("0")}}), `{"a":null,"b":0}`, false}, - {NewObject([]Pair{Pair{"\ta", NewString("\t")}, Pair{"\bb", NewString("\b")}, Pair{"\nb", NewString("\n")}, Pair{"\ra", NewString("\r")}}), `{"\ta":"\t","\u0008b":"\u0008","\nb":"\n","\ra":"\r"}`, false}, - {NewObject([]Pair{}), `{}`, false}, - {NewBytes([]byte("hello, world")), `"aGVsbG8sIHdvcmxk"`, false}, - {NewAny(obj), string(buf), false}, - {NewRaw(`[{ }]`), "[{}]", false}, - {Node{}, "", true}, - {Node{t: types.ValueType(1)}, "", true}, - } - for i, c := range input { - t.Log(i) - buf, err := json.Marshal(&c.node) - if c.err { - if err == nil { - t.Fatal(i) - } - continue - } - if err != nil { - t.Fatal(i, err) - } - assert.Equal(t, c.exp, string(buf)) - } -} - -func TestEncodeNode(t *testing.T) { - data := `{"a":[{},[],-0.1,true,false,null,""],"b":0,"c":true,"d":false,"e":null,"g":""}` - root, e := NewSearcher(data).GetByPath() - if e != nil { - t.Fatal(root) - } - ret, err := root.MarshalJSON() - if err != nil { - t.Fatal(err) - } - if string(ret) != data { - t.Fatal(string(ret)) - } - root.skipAllKey() - ret, err = root.MarshalJSON() - if err != nil { - t.Fatal(err) - } - if string(ret) != data { - t.Fatal(string(ret)) - } - root.loadAllKey() - ret, err = root.MarshalJSON() - if err != nil { - t.Fatal(err) - } - if string(ret) != data { - t.Fatal(string(ret)) - } -} - -func BenchmarkEncodeRaw_Sonic(b *testing.B) { - data := _TwitterJson - root, e := NewSearcher(data).GetByPath() - if e != nil { - b.Fatal(root) - } - _, err := root.MarshalJSON() - if err != nil { - b.Fatal(err) - } - b.SetBytes(int64(len(data))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, err := root.MarshalJSON() - if err != nil { - b.Fatal(err) - } - } -} - -func BenchmarkEncodeSkip_Sonic(b *testing.B) { - data := _TwitterJson - root, e := NewParser(data).Parse() - if e != 0 { - b.Fatal(root) - } - root.skipAllKey() - _, err := root.MarshalJSON() - if err != nil { - b.Fatal(err) - } - b.SetBytes(int64(len(data))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, err := root.MarshalJSON() - if err != nil { - b.Fatal(err) - } - } -} - -func BenchmarkEncodeLoad_Sonic(b *testing.B) { - data := _TwitterJson - root, e := NewParser(data).Parse() - if e != 0 { - b.Fatal(root) - } - root.loadAllKey() - _, err := root.MarshalJSON() - if err != nil { - b.Fatal(err) - } - b.SetBytes(int64(len(data))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, err := root.MarshalJSON() - if err != nil { - b.Fatal(err) - } - } -} diff --git a/internal/json/encoding.go b/internal/json/encoding.go index 9c50b089..e1649f99 100644 --- a/internal/json/encoding.go +++ b/internal/json/encoding.go @@ -17,7 +17,19 @@ package json import ( + "runtime" + "unsafe" + "github.com/cloudwego/dynamicgo/internal/native/types" + "github.com/cloudwego/dynamicgo/internal/rt" +) + +const ( + bytesNull = "null" + bytesTrue = "true" + bytesFalse = "false" + bytesObject = "{}" + bytesArray = "[]" ) func EncodeNull(buf []byte) []byte { @@ -48,16 +60,17 @@ func EncodeBase64(buf []byte, val []byte) []byte { } func EncodeString(buf []byte, val string) []byte { + buf = append(buf, '"') if len(val) == 0 { - buf = append(buf, '"', '"') + buf = append(buf, '"') return buf } - - quote(&buf, val) + NoQuote(&buf, val) + buf = append(buf, '"') + println(val, string(buf)) return buf } -// func EncodeNumber(buf []byte, val json.Number) ([]byte, error) func EncodeInt64(buf []byte, val int64) []byte { i64toa(&buf, val) @@ -105,57 +118,26 @@ func EncodeEmptyObject(buf []byte) []byte { return append(buf, '{', '}') } -func DecodeNull(src string, pos int) (ret int) { - return decodeNull(src, SkipBlank(src, pos)) +func IsSpace(c byte) bool { + return (int(1< len(src) { - return -int(types.ERR_EOF), false - } - if src[pos:ret] == bytesTrue { - return ret, true - } - ret += 1 - if ret > len(src) { - return -int(types.ERR_EOF), false - } - if src[pos:ret] == bytesFalse { - return ret, false - } - return -int(types.ERR_INVALID_CHAR), false -} - -func DecodeString(src string, pos int) (ret int, v string) { - return decodeString(src, SkipBlank(src, pos)) -} +const _blankCharsMask = (1 << ' ') | (1 << '\t') | (1 << '\r') | (1 << '\n') -func DecodeBinary(src string, pos int) (ret int, v []byte) { - return decodeBinary(src, SkipBlank(src, pos)) -} +//go:nocheckptr +func SkipBlank(src string, pos int) int { + se := uintptr(rt.IndexChar(src, len(src))) + sp := uintptr(rt.IndexChar(src, pos)) -func DecodeInt64(src string, pos int) (ret int, v int64, err error) { - return decodeInt64(src, SkipBlank(src, pos)) -} - -func DecodeFloat64(src string, pos int) (ret int, v float64, err error) { - return decodeFloat64(src, SkipBlank(src, pos)) -} - -func SkipNumber(src string, pos int) int { - return skipNumber(src, SkipBlank(src, pos)) -} - -func SkipString(src string, pos int) (int, int) { - return skipString(src, SkipBlank(src, pos)) -} - -func SkipArray(src string, pos int) int { - return skipPair(src, SkipBlank(src, pos), '[', ']') -} - -func SkipObject(src string, pos int) int { - return skipPair(src, SkipBlank(src, pos), '{', '}') -} + for sp < se { + if !IsSpace(*(*byte)(unsafe.Pointer(sp))) { + break + } + sp += 1 + } + if sp >= se { + return -int(types.ERR_EOF) + } + runtime.KeepAlive(src) + return int(sp - uintptr(rt.IndexChar(src, 0))) +} \ No newline at end of file diff --git a/internal/json/encoding_test.go b/internal/json/encoding_test.go deleted file mode 100644 index ca471acf..00000000 --- a/internal/json/encoding_test.go +++ /dev/null @@ -1,336 +0,0 @@ -/** - * Copyright 2023 CloudWeGo Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package json - -import ( - "encoding/base64" - "encoding/json" - "strconv" - "testing" - - "github.com/cloudwego/dynamicgo/internal/native/types" - "github.com/stretchr/testify/require" -) - -func TestUnquote(t *testing.T) { - var src = `"\ud83d\ude00"` - var out string - err := json.Unmarshal([]byte(src), &out) - require.Nil(t, err) - require.Equal(t, "πŸ˜€", out) -} - -func TestDecodeBool(t *testing.T) { - var src = "true," - ret, v := DecodeBool(src, 0) - require.Equal(t, 4, ret) - require.Equal(t, true, v) - - src = "false," - ret, v = DecodeBool(src, 0) - require.Equal(t, 5, ret) - require.Equal(t, false, v) - - src = "tru" - ret, v = DecodeBool(src, 0) - require.Equal(t, -1, ret) - require.Equal(t, false, v) - - src = "trua" - ret, v = DecodeBool(src, 0) - require.Equal(t, -1, ret) - require.Equal(t, false, v) - - src = "fals" - ret, v = DecodeBool(src, 0) - require.Equal(t, -1, ret) - require.Equal(t, false, v) - - src = "falss" - ret, v = DecodeBool(src, 0) - require.Equal(t, -2, ret) - require.Equal(t, false, v) -} - -func TestSkipSpace(t *testing.T) { - var src = "\ra\nb c\td " - ret := SkipBlank(src, 0) - require.Equal(t, 1, ret) - ret = SkipBlank(src, 1) - require.Equal(t, 1, ret) - ret = SkipBlank(src, 2) - require.Equal(t, 3, ret) - ret = SkipBlank(src, 3) - require.Equal(t, 3, ret) - ret = SkipBlank(src, 4) - require.Equal(t, 5, ret) - ret = SkipBlank(src, 5) - require.Equal(t, 5, ret) - ret = SkipBlank(src, 6) - require.Equal(t, 7, ret) - ret = SkipBlank(src, 7) - require.Equal(t, 7, ret) - ret = SkipBlank(src, 8) - require.Equal(t, -1, ret) - ret = SkipBlank(src, 9) - require.Equal(t, -1, ret) - // ret = skipSpace(src, -1) - // require.Equal(t, -1, ret) -} - -func TestSkipNumber(t *testing.T) { - type test struct { - src string - pos int - ret int - } - var tests = []test{ - {"123,", 0, 3}, - {"-123,", 0, 4}, - {"123.456,", 0, 7}, - {"123.456E789,", 0, 11}, - {"123.456E+789,", 0, 12}, - {"123.456E-789,", 0, 12}, - {"-", 0, -1}, - {"+", 0, -2}, - {".1", 0, -2}, - {"1.", 0, -1}, - {"1.e", 0, -2}, - {"1.1.", 0, -2}, - {"e", 0, -2}, - {"1e1e", 0, -2}, - {"1e", 0, -1}, - {"1+", 0, -2}, - {"1e+", 0, -1}, - } - for _, test := range tests { - t.Log(test.src) - ret := skipNumber(test.src, test.pos) - require.Equal(t, test.ret, ret) - } -} - -func TestDecodeString(t *testing.T) { - var src = `"\u666f\\1\\\"\bc\"d"` - exp, err := strconv.Unquote(src) - require.NoError(t, err) - ret, v := decodeString((src), 0) - require.Equal(t, len(src), ret) - require.Equal(t, exp, (v)) - - src = `"abcd"` - exp, err = strconv.Unquote(src) - require.NoError(t, err) - ret, v = decodeString((src), 0) - require.Equal(t, len(src), ret) - require.Equal(t, exp, (v)) - - src = `"abcd",` - ret, v = decodeString((src), 0) - require.Equal(t, len(src)-1, ret) - require.Equal(t, exp, (v)) -} - -func TestDecodeBinary(t *testing.T) { - var src = `"\naGVsbG8s\nIHdvcmxk\r"` - var exp []byte - require.NoError(t, json.Unmarshal([]byte(src), &exp)) - ret, v := DecodeBinary((src), 0) - require.Equal(t, len(src), ret) - require.Equal(t, exp, (v)) - - src = `"\naGVsbG8s\nIHdvcmxk\r",` - ret, v = DecodeBinary((src), 0) - require.Equal(t, len(src)-1, ret) - require.Equal(t, exp, (v)) -} - -func TestDecodeInt64(t *testing.T) { - var src = "-12345678000009999999999" - ret, v, e := decodeInt64(src, 0) - require.Equal(t, len(src), ret) - require.Equal(t, int64(0), v) - require.NotNil(t, e) - - src = "-123456789" - ret, v, _ = decodeInt64(src, 0) - exp, err := strconv.ParseInt(src, 10, 64) - require.NoError(t, err) - require.Equal(t, len(src), ret) - require.Equal(t, exp, v) -} - -func TestDecodeFloat64(t *testing.T) { - var src = "-123.45678E+999999" - ret, v, e := decodeFloat64(src, 0) - require.Equal(t, len(src), ret) - require.Equal(t, float64(0), v) - require.NotNil(t, e) - - src = "-123.45678E+9" - exp, err := strconv.ParseFloat(src, 64) - require.NoError(t, err) - ret, v, _ = decodeFloat64(src, 0) - require.Equal(t, len(src), ret) - require.Equal(t, exp, v) -} - -func TestDecodeValue(t *testing.T) { - type testCase struct { - src string - ret int - exp types.JsonState - err error - } - var cases = []testCase{ - {src: ` null, `, ret: 5, exp: types.JsonState{Vt: types.V_NULL}}, - {src: ` true, `, ret: 5, exp: types.JsonState{Vt: types.V_TRUE}}, - {src: ` false, `, ret: 6, exp: types.JsonState{Vt: types.V_FALSE}}, - {src: ` 123, `, ret: 4, exp: types.JsonState{Vt: types.V_INTEGER, Iv: 123, Ep: 1}}, - {src: ` 123.456, `, ret: 8, exp: types.JsonState{Vt: types.V_DOUBLE, Dv: 123.456, Ep: 1}}, - {src: ` 123.456E+9, `, ret: 11, exp: types.JsonState{Vt: types.V_DOUBLE, Dv: 123.456e+9, Ep: 1}}, - {src: ` "abcd", `, ret: 7, exp: types.JsonState{Vt: types.V_STRING, Iv: 2, Ep: -1}}, - {src: ` "ab\"cd", `, ret: 9, exp: types.JsonState{Vt: types.V_STRING, Iv: 2, Ep: 4}}, - {src: ` "aGVsbG8sIHdvcmxk", `, ret: 19, exp: types.JsonState{Vt: types.V_STRING, Iv: 2, Ep: -1}}, - {src: ` "a\nGVsbG8sIHdvcmxk", `, ret: 21, exp: types.JsonState{Vt: types.V_STRING, Iv: 2, Ep: 3}}, - {src: ` [1,2,3], `, ret: 2, exp: types.JsonState{Vt: types.V_ARRAY}}, - {src: ` {"a":1,"b":2}, `, ret: 2, exp: types.JsonState{Vt: types.V_OBJECT}}, - } - - for _, c := range cases { - t.Run(c.src, func(t *testing.T) { - ret, v := DecodeValue(c.src, 0) - if c.err != nil { - require.Equal(t, c.err.Error(), types.ParsingError(-ret).Error()) - } - require.Equal(t, c.ret, ret) - require.Equal(t, c.exp, v) - }) - } -} - -func TestSkipValue(t *testing.T) { - type testCase struct { - src string - ret int - start int - } - var cases = []testCase{ - {src: ` null, `, ret: 5, start: 1}, - {src: ` true, `, ret: 5, start: 1}, - {src: ` false, `, ret: 6, start: 1}, - {src: ` 123, `, ret: 4, start: 1}, - {src: ` 123.456, `, ret: 8, start: 1}, - {src: ` 123.456E+9, `, ret: 11, start: 1}, - {src: ` "abcd", `, ret: 7, start: 1}, - {src: ` "ab\"cd", `, ret: 9, start: 1}, - {src: ` "aGVsbG8sIHdvcmxk", `, ret: 19, start: 1}, - {src: ` "a\nGVsbG8sIHdvcmxk", `, ret: 21, start: 1}, - {src: ` [1,2,3], `, ret: 8, start: 1}, - {src: ` {"a":1,"b":2}, `, ret: 14, start: 1}, - } - - for _, c := range cases { - t.Run(c.src, func(t *testing.T) { - ret, start := SkipValue(c.src, 0) - require.Equal(t, c.ret, ret) - require.Equal(t, c.start, start) - }) - } -} - -func TestSkipPair(t *testing.T) { - var src = ` [1,"[", 3,"\" \\[", [[ ],{} ] ] ` - ret := skipPair(src, 1, '[', ']') - require.Equal(t, len(src)-1, ret) - - src = ` {1,"{", 3,"\" \\}", [[ ],{} ] } ` - ret = skipPair(src, 1, '{', '}') - require.Equal(t, len(src)-1, ret) -} - -func BenchmarkSkipNumber(b *testing.B) { - var src = "123.456E789," - b.Run("skipNumber", func(b *testing.B) { - for i := 0; i < b.N; i++ { - skipNumber(src, 0) - } - }) - b.Run("strconv", func(b *testing.B) { - for i := 0; i < b.N; i++ { - strconv.ParseFloat(src, 64) - } - }) -} - -func BenchmarkDecodeBool(b *testing.B) { - b.Run("dyanmicgo", func(b *testing.B) { - for i := 0; i < b.N; i++ { - _, _ = DecodeBool("false", 0) - } - }) - b.Run("std", func(b *testing.B) { - for i := 0; i < b.N; i++ { - _, _ = strconv.ParseBool("false") - } - }) -} - -func BenchmarkSkipSpace(b *testing.B) { - b.Run("4 blanks", func(b *testing.B) { - for i := 0; i < b.N; i++ { - _ = SkipBlank(" b", 0) - } - }) - b.Run("1 blanks", func(b *testing.B) { - for i := 0; i < b.N; i++ { - _ = SkipBlank(" b", 0) - } - }) - b.Run("0 blanks", func(b *testing.B) { - for i := 0; i < b.N; i++ { - _ = SkipBlank("b", 0) - } - }) -} - -func BenchmarkDecodeString(b *testing.B) { - b.Run("dyanmicgo", func(b *testing.B) { - for i := 0; i < b.N; i++ { - _, _ = decodeString(`"abcd"`, 0) - } - }) - b.Run("std", func(b *testing.B) { - for i := 0; i < b.N; i++ { - _, _ = strconv.Unquote(`"abcd"`) - } - }) -} - -func BenchmarkDecodeBinary(b *testing.B) { - b.Run("dyanmicgo", func(b *testing.B) { - for i := 0; i < b.N; i++ { - _, _ = DecodeBinary(`"aGVsbG8sIHdvcmxk"`, 0) - } - }) - b.Run("std", func(b *testing.B) { - for i := 0; i < b.N; i++ { - vv, _ := strconv.Unquote(`"aGVsbG8sIHdvcmxk"`) - _, _ = base64.StdEncoding.DecodeString(vv) - } - }) -} diff --git a/internal/json/errors.go b/internal/json/error.go similarity index 60% rename from internal/json/errors.go rename to internal/json/error.go index 56e52a47..de585089 100644 --- a/internal/json/errors.go +++ b/internal/json/error.go @@ -1,46 +1,26 @@ -/** - * Copyright 2023 CloudWeGo Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +// Copyright 2023 CloudWeGo Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package json import ( "fmt" "strings" - "unsafe" "github.com/cloudwego/dynamicgo/internal/native/types" ) -func (self *Parser) syntaxError(err types.ParsingError) SyntaxError { - return SyntaxError{ - Pos: self.p, - Src: self.s, - Code: err, - } -} - -func newSyntaxError(err SyntaxError) *Node { - msg := err.Description() - return &Node{ - t: V_ERROR, - v: int64(err.Code), - p: unsafe.Pointer(&msg), - } -} - type SyntaxError struct { Pos int Src string @@ -111,4 +91,4 @@ func clamp_zero(v int) int { } else { return v } -} +} \ No newline at end of file diff --git a/internal/json/iterator.go b/internal/json/iterator.go deleted file mode 100644 index a914ec98..00000000 --- a/internal/json/iterator.go +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Copyright 2023 CloudWeGo Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package json - -import ( - "fmt" - - "github.com/cloudwego/dynamicgo/internal/native/types" -) - -type Pair struct { - Key string - Value Node -} - -// Values returns iterator for array's children traversal -func (self *Node) Values() (ListIterator, error) { - if err := self.should(types.V_ARRAY, "an array"); err != nil { - return ListIterator{}, err - } - return ListIterator{Iterator{p: self}}, nil -} - -// Properties returns iterator for object's children traversal -func (self *Node) Properties() (ObjectIterator, error) { - if err := self.should(types.V_OBJECT, "an object"); err != nil { - return ObjectIterator{}, err - } - return ObjectIterator{Iterator{p: self}}, nil -} - -type Iterator struct { - i int - p *Node -} - -func (self *Iterator) Pos() int { - return self.i -} - -func (self *Iterator) Len() int { - return self.p.len() -} - -// HasNext reports if it is the end of iteration or has error. -func (self *Iterator) HasNext() bool { - if !self.p.isLazy() { - return self.p.Valid() && self.i < self.p.len() - } else if self.p.t == _V_ARRAY_LAZY { - return self.p.skipNextNode().Valid() - } else if self.p.t == _V_OBJECT_LAZY { - pair := self.p.skipNextPair() - if pair == nil { - return false - } - return pair.Value.Valid() - } - return false -} - -// ListIterator is specialized iterator for V_ARRAY -type ListIterator struct { - Iterator -} - -// ObjectIterator is specialized iterator for V_ARRAY -type ObjectIterator struct { - Iterator -} - -// Next scans through children of underlying V_ARRAY, -// copies each child to v, and returns .HasNext(). -func (self *ListIterator) Next(v *Node) bool { - if !self.HasNext() { - return false - } else { - *v, self.i = *self.p.nodeAt(self.i), self.i+1 - return true - } -} - -// Next scans through children of underlying V_OBJECT, -// copies each child to v, and returns .HasNext(). -func (self *ObjectIterator) Next(p *Pair) bool { - if !self.HasNext() { - return false - } else { - *p, self.i = *self.p.pairAt(self.i), self.i+1 - return true - } -} - -// Sequence represents scanning path of single-layer nodes. -// Index indicates the value's order in both V_ARRAY and V_OBJECT json. -// Key is the value's key (for V_OBJECT json only, otherwise it will be nil). -type Sequence struct { - Index int - Key *string - // Level int -} - -// String is string representation of one Sequence -func (s Sequence) String() string { - k := "" - if s.Key != nil { - k = *s.Key - } - return fmt.Sprintf("Sequence(%d, %q)", s.Index, k) -} - -type Scanner func(path Sequence, node *Node) bool - -// ForEach scans one V_OBJECT node's children from JSON head to tail, -// and pass the Sequence and Node of corresponding JSON value. -// -// Especailly, if the node is not V_ARRAY or V_OBJECT, -// the node itself will be returned and Sequence.Index == -1. -func (self *Node) ForEach(sc Scanner) error { - switch self.itype() { - case types.V_ARRAY: - ns, err := self.UnsafeArray() - if err != nil { - return err - } - for i := range ns { - if !sc(Sequence{i, nil}, &ns[i]) { - return err - } - } - case types.V_OBJECT: - ns, err := self.UnsafeMap() - if err != nil { - return err - } - for i := range ns { - if !sc(Sequence{i, &ns[i].Key}, &ns[i].Value) { - return err - } - } - default: - sc(Sequence{-1, nil}, self) - } - return self.Check() -} - -type PairSlice []Pair - -func (self PairSlice) Sort() { - radixQsort(self, 0, maxDepth(len(self))) -} diff --git a/internal/json/iterator_test.go b/internal/json/iterator_test.go deleted file mode 100644 index f7eae35f..00000000 --- a/internal/json/iterator_test.go +++ /dev/null @@ -1,267 +0,0 @@ -/** - * Copyright 2023 CloudWeGo Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package json - -import ( - "fmt" - "strconv" - "testing" - - "github.com/stretchr/testify/assert" -) - -func getTestIteratorSample(loop int) (string, int) { - var data []int - var v1 = "" - var v2 = "" - for i := 0; i < loop; i++ { - data = append(data, i*i) - v1 += strconv.Itoa(i) - v2 += `"k` + strconv.Itoa(i) + `":` + strconv.Itoa(i) - if i != loop-1 { - v1 += `,` - v2 += `,` - } - } - return `{"array":[` + v1 + `], "object":{` + v2 + `}}`, loop -} - -func TestForEach(t *testing.T) { - pathes := []Sequence{} - values := []*Node{} - sc := func(path Sequence, node *Node) bool { - pathes = append(pathes, path) - values = append(values, node) - if path.Key != nil && *path.Key == "array" { - node.ForEach(func(path Sequence, node *Node) bool { - pathes = append(pathes, path) - values = append(values, node) - return true - }) - } - return true - } - - str, _ := getTestIteratorSample(3) - fmt.Println(str) - root, err := NewSearcher(str).GetByPath() - if err != nil { - t.Fatal(err) - } - err = root.ForEach(sc) - if err != nil { - t.Fatal(err) - } - eObjKey := "object" - eArrKey := "array" - expPath := []Sequence{ - {0, &eArrKey}, - {0, nil}, - {1, nil}, - {2, nil}, - {1, &eObjKey}, - } - expValue := []*Node{ - root.Get("array"), - root.GetByPath("array", 0), - root.GetByPath("array", 1), - root.GetByPath("array", 2), - root.Get("object"), - } - // fmt.Printf("pathes:%+v\n", pathes) - // fmt.Printf("values:%+v\n", values) - assert.Equal(t, expPath, pathes) - assert.Equal(t, expValue, values) - -} - -func TestRawIterator(t *testing.T) { - str, loop := getTestIteratorSample(_DEFAULT_NODE_CAP) - fmt.Println(str) - - root, err := NewSearcher(str).GetByPath("array") - if err != nil { - t.Fatal(err) - } - ai, _ := root.Values() - i := int64(0) - for ai.HasNext() { - v := &Node{} - if !ai.Next(v) { - t.Fatalf("no next") - } - x, _ := v.Int64() - if i < int64(loop) && x != i { - t.Fatalf("exp:%v, got:%v", i, v) - } - if i != int64(ai.Pos())-1 || i >= int64(ai.Len()) { - t.Fatal(i) - } - i++ - } - if i != int64(loop) { - t.Fatal(i) - } - - root, err = NewSearcher(str).GetByPath("object") - if err != nil { - t.Fatal(err) - } - mi, _ := root.Properties() - i = int64(0) - for mi.HasNext() { - v := &Pair{} - if !mi.Next(v) { - t.Fatalf("no next") - } - x, _ := v.Value.Int64() - if i < int64(loop) && (x != i || v.Key != fmt.Sprintf("k%d", i)) { - vv, _ := v.Value.Interface() - t.Fatalf("exp:%v, got:%v", i, vv) - } - if i != int64(mi.Pos())-1 || i >= int64(mi.Len()) { - t.Fatal(i) - } - i++ - } - if i != int64(loop) { - t.Fatal(i) - } -} - -func TestIterator(t *testing.T) { - str, loop := getTestIteratorSample(_DEFAULT_NODE_CAP) - fmt.Println(str) - - root, err := NewParser(str).Parse() - if err != 0 { - t.Fatal(err) - } - ai, _ := root.Get("array").Values() - i := int64(0) - for ai.HasNext() { - v := &Node{} - if !ai.Next(v) { - t.Fatalf("no next") - } - x, _ := v.Int64() - if i < int64(loop) && x != i { - t.Fatalf("exp:%v, got:%v", i, v) - } - if i != int64(ai.Pos())-1 || i >= int64(ai.Len()) { - t.Fatal(i) - } - i++ - } - if i != int64(loop) { - t.Fatal(i) - } - - root, err = NewParser(str).Parse() - if err != 0 { - t.Fatal(err) - } - mi, _ := root.Get("object").Properties() - i = int64(0) - for mi.HasNext() { - v := &Pair{} - if !mi.Next(v) { - t.Fatalf("no next") - } - x, _ := v.Value.Int64() - if i < int64(loop) && (x != i || v.Key != fmt.Sprintf("k%d", i)) { - vv, _ := v.Value.Interface() - t.Fatalf("exp:%v, got:%v", i, vv) - } - if i != int64(mi.Pos())-1 || i >= int64(mi.Len()) { - t.Fatal(i) - } - i++ - } - if i != int64(loop) { - t.Fatal(i) - } - - str, _ = getTestIteratorSample(0) - root, err = NewParser(str).Parse() - if err != 0 { - t.Fatal(err) - } - mi, _ = root.Get("object").Properties() - if mi.HasNext() { - t.Fatalf("should not have next") - } -} - -func BenchmarkArrays(b *testing.B) { - for i := 0; i < b.N; i++ { - root, err := NewSearcher(_TwitterJson).GetByPath("statuses", 1, "entities", "hashtags") - if err != nil { - b.Fatal(err) - } - a, _ := root.Array() - for _, v := range a { - _ = v - } - } -} - -func BenchmarkListIterator(b *testing.B) { - for i := 0; i < b.N; i++ { - root, err := NewSearcher(_TwitterJson).GetByPath("statuses", 1, "entities", "hashtags") - if err != nil { - b.Fatal(err) - } - it, _ := root.Values() - for it.HasNext() { - v := &Node{} - if !it.Next(v) { - b.Fatalf("no value") - } - } - } -} - -func BenchmarkMap(b *testing.B) { - for i := 0; i < b.N; i++ { - root, err := NewSearcher(_TwitterJson).GetByPath("statuses", 1, "user") - if err != nil { - b.Fatal(err) - } - m, _ := root.Map() - for k, v := range m { - _ = v - _ = k - } - } -} - -func BenchmarkObjectIterator(b *testing.B) { - for i := 0; i < b.N; i++ { - root, err := NewSearcher(_TwitterJson).GetByPath("statuses", 1, "user") - if err != nil { - b.Fatal(err) - } - it, _ := root.Properties() - for it.HasNext() { - v := &Pair{} - if !it.Next(v) { - b.Fatalf("no value") - } - } - } -} diff --git a/internal/json/node.go b/internal/json/node.go deleted file mode 100644 index ccbeadfd..00000000 --- a/internal/json/node.go +++ /dev/null @@ -1,1942 +0,0 @@ -/** - * Copyright 2023 CloudWeGo Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package json - -import ( - "encoding/json" - "fmt" - "strconv" - "unsafe" - - "github.com/cloudwego/dynamicgo/internal/native/types" - "github.com/cloudwego/dynamicgo/internal/rt" -) - -const ( - _CAP_BITS = 32 - _LEN_MASK = 1<<_CAP_BITS - 1 - - _NODE_SIZE = unsafe.Sizeof(Node{}) - _PAIR_SIZE = unsafe.Sizeof(Pair{}) -) - -const ( - _V_NONE types.ValueType = 0 - _V_NODE_BASE types.ValueType = 1 << 5 - _V_LAZY types.ValueType = 1 << 7 - _V_RAW types.ValueType = 1 << 8 - _V_NUMBER = _V_NODE_BASE + 1 - _V_ANY = _V_NODE_BASE + 2 - _V_ARRAY_LAZY = _V_LAZY | types.V_ARRAY - _V_OBJECT_LAZY = _V_LAZY | types.V_OBJECT - _MASK_LAZY = _V_LAZY - 1 - _MASK_RAW = _V_RAW - 1 -) - -const ( - V_NONE = 0 - V_ERROR = 1 - V_NULL = 2 - V_TRUE = 3 - V_FALSE = 4 - V_ARRAY = 5 - V_OBJECT = 6 - V_STRING = 7 - V_NUMBER = int(_V_NUMBER) - V_ANY = int(_V_ANY) -) - -type Node struct { - v int64 - t types.ValueType - p unsafe.Pointer -} - -// UnmarshalJSON is just an adapter to json.Unmarshaler. -// If you want better performance, use Searcher.GetByPath() directly -func (self *Node) UnmarshalJSON(data []byte) (err error) { - *self, err = NewSearcher(string(data)).GetByPath() - return -} - -/** Node Type Accessor **/ - -// Type returns json type represented by the node -// It will be one of belows: -// -// V_NONE = 0 (empty node) -// V_ERROR = 1 (error node) -// V_NULL = 2 (json value `null`) -// V_TRUE = 3 (json value `true`) -// V_FALSE = 4 (json value `false`) -// V_ARRAY = 5 (json value array) -// V_OBJECT = 6 (json value object) -// V_STRING = 7 (json value string) -// V_NUMBER = 33 (json value number ) -// V_ANY = 34 (golang interface{}) -func (self Node) Type() int { - return int(self.t & _MASK_LAZY & _MASK_RAW) -} - -func (self Node) itype() types.ValueType { - return self.t & _MASK_LAZY & _MASK_RAW -} - -// Exists returns false only if the self is nil or empty node V_NONE -func (self *Node) Exists() bool { - return self != nil && self.t != _V_NONE -} - -// Valid reports if self is NOT V_ERROR or nil -func (self *Node) Valid() bool { - if self == nil { - return false - } - return self.t != V_ERROR -} - -// Check checks if the node itself is valid, and return: -// - ErrNotFound If the node is nil -// - Its underlying error If the node is V_ERROR -func (self *Node) Check() error { - if self == nil { - return ErrNotExist - } else if self.t != V_ERROR { - return nil - } else { - return self - } -} - -// Error returns error message if the node is invalid -func (self Node) Error() string { - if self.t != V_ERROR { - return "" - } else { - return *(*string)(self.p) - } -} - -// IsRaw returns true if node's underlying value is raw json -func (self Node) IsRaw() bool { - return self.t&_V_RAW != 0 -} - -func (self *Node) isLazy() bool { - return self != nil && self.t&_V_LAZY != 0 -} - -func (self *Node) isAny() bool { - return self != nil && self.t == _V_ANY -} - -/** Simple Value Methods **/ - -// Raw returns json representation of the node, -func (self *Node) Raw() (string, error) { - if !self.IsRaw() { - buf, err := self.MarshalJSON() - return rt.Mem2Str(buf), err - } - return addr2str(self.p, self.v), nil -} - -func (self *Node) checkRaw() error { - if err := self.Check(); err != nil { - return err - } - if self.IsRaw() { - self.parseRaw(false) - } - return nil -} - -// Bool returns bool value represented by this node, -// including types.V_TRUE|V_FALSE|V_NUMBER|V_STRING|V_ANY|V_NULL, -// V_NONE will return error -func (self *Node) Bool() (bool, error) { - if err := self.checkRaw(); err != nil { - return false, err - } - switch self.t { - case types.V_TRUE: - return true, nil - case types.V_FALSE: - return false, nil - case types.V_NULL: - return false, nil - case _V_NUMBER: - if i, err := numberToInt64(self); err == nil { - return i != 0, nil - } else if f, err := numberToFloat64(self); err == nil { - return f != 0, nil - } else { - return false, err - } - case types.V_STRING: - return strconv.ParseBool(addr2str(self.p, self.v)) - case _V_ANY: - any := self.packAny() - switch v := any.(type) { - case bool: - return v, nil - case int: - return v != 0, nil - case int8: - return v != 0, nil - case int16: - return v != 0, nil - case int32: - return v != 0, nil - case int64: - return v != 0, nil - case uint: - return v != 0, nil - case uint8: - return v != 0, nil - case uint16: - return v != 0, nil - case uint32: - return v != 0, nil - case uint64: - return v != 0, nil - case float32: - return v != 0, nil - case float64: - return v != 0, nil - case string: - return strconv.ParseBool(v) - case json.Number: - if i, err := v.Int64(); err == nil { - return i != 0, nil - } else if f, err := v.Float64(); err == nil { - return f != 0, nil - } else { - return false, err - } - default: - return false, ErrUnsupportType - } - default: - return false, ErrUnsupportType - } -} - -// Int64 casts the node to int64 value, -// including V_NUMBER|V_TRUE|V_FALSE|V_ANY|V_STRING -// V_NONE it will return error -func (self *Node) Int64() (int64, error) { - if err := self.checkRaw(); err != nil { - return 0, err - } - switch self.t { - case _V_NUMBER, types.V_STRING: - if i, err := numberToInt64(self); err == nil { - return i, nil - } else if f, err := numberToFloat64(self); err == nil { - return int64(f), nil - } else { - return 0, err - } - case types.V_TRUE: - return 1, nil - case types.V_FALSE: - return 0, nil - case types.V_NULL: - return 0, nil - case _V_ANY: - any := self.packAny() - switch v := any.(type) { - case bool: - if v { - return 1, nil - } else { - return 0, nil - } - case int: - return int64(v), nil - case int8: - return int64(v), nil - case int16: - return int64(v), nil - case int32: - return int64(v), nil - case int64: - return int64(v), nil - case uint: - return int64(v), nil - case uint8: - return int64(v), nil - case uint16: - return int64(v), nil - case uint32: - return int64(v), nil - case uint64: - return int64(v), nil - case float32: - return int64(v), nil - case float64: - return int64(v), nil - case string: - if i, err := strconv.ParseInt(v, 10, 64); err == nil { - return i, nil - } else if f, err := strconv.ParseFloat(v, 64); err == nil { - return int64(f), nil - } else { - return 0, err - } - case json.Number: - if i, err := v.Int64(); err == nil { - return i, nil - } else if f, err := v.Float64(); err == nil { - return int64(f), nil - } else { - return 0, err - } - default: - return 0, ErrUnsupportType - } - default: - return 0, ErrUnsupportType - } -} - -// StrictInt64 exports underlying int64 value, including V_NUMBER, V_ANY -func (self *Node) StrictInt64() (int64, error) { - if err := self.checkRaw(); err != nil { - return 0, err - } - switch self.t { - case _V_NUMBER: - return numberToInt64(self) - case _V_ANY: - any := self.packAny() - switch v := any.(type) { - case int: - return int64(v), nil - case int8: - return int64(v), nil - case int16: - return int64(v), nil - case int32: - return int64(v), nil - case int64: - return int64(v), nil - case uint: - return int64(v), nil - case uint8: - return int64(v), nil - case uint16: - return int64(v), nil - case uint32: - return int64(v), nil - case uint64: - return int64(v), nil - case json.Number: - if i, err := v.Int64(); err == nil { - return i, nil - } else { - return 0, err - } - default: - return 0, ErrUnsupportType - } - default: - return 0, ErrUnsupportType - } -} - -func castNumber(v bool) json.Number { - if v { - return json.Number("1") - } else { - return json.Number("0") - } -} - -// Number casts node to float64, -// including V_NUMBER|V_TRUE|V_FALSE|V_ANY|V_STRING|V_NULL, -// V_NONE it will return error -func (self *Node) Number() (json.Number, error) { - if err := self.checkRaw(); err != nil { - return json.Number(""), err - } - switch self.t { - case _V_NUMBER: - return toNumber(self), nil - case types.V_STRING: - if _, err := numberToInt64(self); err == nil { - return toNumber(self), nil - } else if _, err := numberToFloat64(self); err == nil { - return toNumber(self), nil - } else { - return json.Number(""), err - } - case types.V_TRUE: - return json.Number("1"), nil - case types.V_FALSE: - return json.Number("0"), nil - case types.V_NULL: - return json.Number("0"), nil - case _V_ANY: - any := self.packAny() - switch v := any.(type) { - case bool: - return castNumber(v), nil - case int: - return castNumber(v != 0), nil - case int8: - return castNumber(v != 0), nil - case int16: - return castNumber(v != 0), nil - case int32: - return castNumber(v != 0), nil - case int64: - return castNumber(v != 0), nil - case uint: - return castNumber(v != 0), nil - case uint8: - return castNumber(v != 0), nil - case uint16: - return castNumber(v != 0), nil - case uint32: - return castNumber(v != 0), nil - case uint64: - return castNumber(v != 0), nil - case float32: - return castNumber(v != 0), nil - case float64: - return castNumber(v != 0), nil - case string: - if _, err := strconv.ParseFloat(v, 64); err == nil { - return json.Number(v), nil - } else { - return json.Number(""), err - } - case json.Number: - return v, nil - default: - return json.Number(""), ErrUnsupportType - } - default: - return json.Number(""), ErrUnsupportType - } -} - -// Number exports underlying float64 value, including V_NUMBER, V_ANY of json.Number -func (self *Node) StrictNumber() (json.Number, error) { - if err := self.checkRaw(); err != nil { - return json.Number(""), err - } - switch self.t { - case _V_NUMBER: - return toNumber(self), nil - case _V_ANY: - if v, ok := self.packAny().(json.Number); ok { - return v, nil - } else { - return json.Number(""), ErrUnsupportType - } - default: - return json.Number(""), ErrUnsupportType - } -} - -// String cast node to string, -// including V_NUMBER|V_TRUE|V_FALSE|V_ANY|V_STRING|V_NULL, -// V_NONE it will return error -func (self *Node) String() (string, error) { - if err := self.checkRaw(); err != nil { - return "", err - } - switch self.t { - case types.V_NULL: - return "", nil - case types.V_TRUE: - return "true", nil - case types.V_FALSE: - return "false", nil - case types.V_STRING, _V_NUMBER: - return addr2str(self.p, self.v), nil - case _V_ANY: - any := self.packAny() - switch v := any.(type) { - case bool: - return strconv.FormatBool(v), nil - case int: - return strconv.Itoa(v), nil - case int8: - return strconv.Itoa(int(v)), nil - case int16: - return strconv.Itoa(int(v)), nil - case int32: - return strconv.Itoa(int(v)), nil - case int64: - return strconv.Itoa(int(v)), nil - case uint: - return strconv.Itoa(int(v)), nil - case uint8: - return strconv.Itoa(int(v)), nil - case uint16: - return strconv.Itoa(int(v)), nil - case uint32: - return strconv.Itoa(int(v)), nil - case uint64: - return strconv.Itoa(int(v)), nil - case float32: - return strconv.FormatFloat(float64(v), 'g', -1, 64), nil - case float64: - return strconv.FormatFloat(float64(v), 'g', -1, 64), nil - case string: - return v, nil - case json.Number: - return v.String(), nil - default: - return "", ErrUnsupportType - } - default: - return "", ErrUnsupportType - } -} - -// StrictString returns string value (unescaped), includeing V_STRING, V_ANY of string. -// In other cases, it will return empty string. -func (self *Node) StrictString() (string, error) { - if err := self.checkRaw(); err != nil { - return "", err - } - switch self.t { - case types.V_STRING: - return addr2str(self.p, self.v), nil - case _V_ANY: - if v, ok := self.packAny().(string); ok { - return v, nil - } else { - return "", ErrUnsupportType - } - default: - return "", ErrUnsupportType - } -} - -// Float64 cast node to float64, -// including V_NUMBER|V_TRUE|V_FALSE|V_ANY|V_STRING|V_NULL, -// V_NONE it will return error -func (self *Node) Float64() (float64, error) { - if err := self.checkRaw(); err != nil { - return 0.0, err - } - switch self.t { - case _V_NUMBER, types.V_STRING: - return numberToFloat64(self) - case types.V_TRUE: - return 1.0, nil - case types.V_FALSE: - return 0.0, nil - case types.V_NULL: - return 0.0, nil - case _V_ANY: - any := self.packAny() - switch v := any.(type) { - case bool: - if v { - return 1.0, nil - } else { - return 0.0, nil - } - case int: - return float64(v), nil - case int8: - return float64(v), nil - case int16: - return float64(v), nil - case int32: - return float64(v), nil - case int64: - return float64(v), nil - case uint: - return float64(v), nil - case uint8: - return float64(v), nil - case uint16: - return float64(v), nil - case uint32: - return float64(v), nil - case uint64: - return float64(v), nil - case float32: - return float64(v), nil - case float64: - return float64(v), nil - case string: - if f, err := strconv.ParseFloat(v, 64); err == nil { - return float64(f), nil - } else { - return 0, err - } - case json.Number: - if f, err := v.Float64(); err == nil { - return float64(f), nil - } else { - return 0, err - } - default: - return 0, ErrUnsupportType - } - default: - return 0.0, ErrUnsupportType - } -} - -// Float64 exports underlying float64 value, includeing V_NUMBER, V_ANY -func (self *Node) StrictFloat64() (float64, error) { - if err := self.checkRaw(); err != nil { - return 0.0, err - } - switch self.t { - case _V_NUMBER: - return numberToFloat64(self) - case _V_ANY: - any := self.packAny() - switch v := any.(type) { - case float32: - return float64(v), nil - case float64: - return float64(v), nil - default: - return 0, ErrUnsupportType - } - default: - return 0.0, ErrUnsupportType - } -} - -/** Sequencial Value Methods **/ - -// Len returns children count of a array|object|string node -// For partially loaded node, it also works but only counts the parsed children -func (self *Node) Len() (int, error) { - if err := self.checkRaw(); err != nil { - return 0, err - } - if self.t == types.V_ARRAY || self.t == types.V_OBJECT || self.t == _V_ARRAY_LAZY || self.t == _V_OBJECT_LAZY { - return int(self.v & _LEN_MASK), nil - } else if self.t == types.V_STRING { - return int(self.v), nil - } else if self.t == _V_NONE || self.t == types.V_NULL { - return 0, nil - } else { - return 0, ErrUnsupportType - } -} - -func (self Node) len() int { - return int(self.v & _LEN_MASK) -} - -// Cap returns malloc capacity of a array|object node for children -func (self *Node) Cap() (int, error) { - if err := self.checkRaw(); err != nil { - return 0, err - } - if self.t == types.V_ARRAY || self.t == types.V_OBJECT || self.t == _V_ARRAY_LAZY || self.t == _V_OBJECT_LAZY { - return int(self.v >> _CAP_BITS), nil - } else if self.t == _V_NONE || self.t == types.V_NULL { - return 0, nil - } else { - return 0, ErrUnsupportType - } -} - -func (self Node) cap() int { - return int(self.v >> _CAP_BITS) -} - -// Set sets the node of given key under self, and reports if the key has existed. -// -// If self is V_NONE or V_NULL, it becomes V_OBJECT and sets the node at the key. -func (self *Node) Set(key string, node Node) (bool, error) { - if self != nil && (self.t == _V_NONE || self.t == types.V_NULL) { - *self = NewObject([]Pair{{key, node}}) - return false, nil - } - - if err := node.Check(); err != nil { - return false, err - } - - p := self.Get(key) - if !p.Exists() { - l := self.len() - c := self.cap() - if l == c { - // TODO: maybe change append size in future - c += _DEFAULT_NODE_CAP - mem := unsafe_NewArray(_PAIR_TYPE, c) - memmove(mem, self.p, _PAIR_SIZE*uintptr(l)) - self.p = mem - } - v := self.pairAt(l) - v.Key = key - v.Value = node - self.setCapAndLen(c, l+1) - return false, nil - - } else if err := p.Check(); err != nil { - return false, err - } - - *p = node - return true, nil -} - -// SetAny wraps val with V_ANY node, and Set() the node. -func (self *Node) SetAny(key string, val interface{}) (bool, error) { - return self.Set(key, NewAny(val)) -} - -// Unset remove the node of given key under object parent, and reports if the key has existed. -func (self *Node) Unset(key string) (bool, error) { - self.must(types.V_OBJECT, "an object") - p, i := self.skipKey(key) - if !p.Exists() { - return false, nil - } else if err := p.Check(); err != nil { - return false, err - } - - self.removePair(i) - return true, nil -} - -// SetByIndex sets the node of given index, and reports if the key has existed. -// -// The index must be within self's children. -func (self *Node) SetByIndex(index int, node Node) (bool, error) { - if err := node.Check(); err != nil { - return false, err - } - - p := self.Index(index) - if !p.Exists() { - return false, ErrNotExist - } else if err := p.Check(); err != nil { - return false, err - } - - *p = node - return true, nil -} - -// SetAny wraps val with V_ANY node, and SetByIndex() the node. -func (self *Node) SetAnyByIndex(index int, val interface{}) (bool, error) { - return self.SetByIndex(index, NewAny(val)) -} - -// UnsetByIndex remove the node of given index -func (self *Node) UnsetByIndex(index int) (bool, error) { - var p *Node - it := self.itype() - if it == types.V_ARRAY { - p = self.Index(index) - } else if it == types.V_OBJECT { - pr := self.skipIndexPair(index) - if pr == nil { - return false, ErrNotExist - } - p = &pr.Value - } else { - return false, ErrUnsupportType - } - - if !p.Exists() { - return false, ErrNotExist - } - - if it == types.V_ARRAY { - self.removeNode(index) - } else if it == types.V_OBJECT { - self.removePair(index) - } - return true, nil -} - -// Add appends the given node under self. -// -// If self is V_NONE or V_NULL, it becomes V_ARRAY and sets the node at index 0. -func (self *Node) Add(node Node) error { - if self != nil && (self.t == _V_NONE || self.t == types.V_NULL) { - *self = NewArray([]Node{node}) - return nil - } - - if err := self.should(types.V_ARRAY, "an array"); err != nil { - return err - } - if err := self.skipAllIndex(); err != nil { - return err - } - - var p rt.GoSlice - p.Cap = self.cap() - p.Len = self.len() - p.Ptr = self.p - - s := *(*[]Node)(unsafe.Pointer(&p)) - s = append(s, node) - - self.p = unsafe.Pointer(&s[0]) - self.setCapAndLen(cap(s), len(s)) - return nil -} - -// SetAny wraps val with V_ANY node, and Add() the node. -func (self *Node) AddAny(val interface{}) error { - return self.Add(NewAny(val)) -} - -// GetByPath load given path on demands, -// which only ensure nodes before this path got parsed -func (self *Node) GetByPath(path ...interface{}) *Node { - if !self.Valid() { - return self - } - var s = self - for _, p := range path { - switch p := p.(type) { - case int: - s = s.Index(p) - if !s.Valid() { - return s - } - case string: - s = s.Get(p) - if !s.Valid() { - return s - } - default: - panic("path must be either int or string") - } - } - return s -} - -// Get loads given key of an object node on demands -func (self *Node) Get(key string) *Node { - if err := self.should(types.V_OBJECT, "an object"); err != nil { - return unwrapError(err) - } - n, _ := self.skipKey(key) - return n -} - -// Index indexies node at given idx, -// node type CAN be either V_OBJECT or V_ARRAY -func (self *Node) Index(idx int) *Node { - if err := self.checkRaw(); err != nil { - return unwrapError(err) - } - - it := self.itype() - if it == types.V_ARRAY { - return self.skipIndex(idx) - - } else if it == types.V_OBJECT { - pr := self.skipIndexPair(idx) - if pr == nil { - return newError(_ERR_NOT_FOUND, "value not exists") - } - return &pr.Value - - } else { - return newError(_ERR_UNSUPPORT_TYPE, fmt.Sprintf("unsupported type: %v", self.itype())) - } -} - -// IndexPair indexies pair at given idx, -// node type MUST be either V_OBJECT -func (self *Node) IndexPair(idx int) *Pair { - if err := self.should(types.V_OBJECT, "an object"); err != nil { - return nil - } - return self.skipIndexPair(idx) -} - -// IndexOrGet firstly use idx to index a value and check if its key matches -// If not, then use the key to search value -func (self *Node) IndexOrGet(idx int, key string) *Node { - if err := self.should(types.V_OBJECT, "an object"); err != nil { - return unwrapError(err) - } - - pr := self.skipIndexPair(idx) - if pr != nil && pr.Key == key { - return &pr.Value - } - n, _ := self.skipKey(key) - return n -} - -/** Generic Value Converters **/ - -// Map loads all keys of an object node -func (self *Node) Map() (map[string]interface{}, error) { - if self.isAny() { - any := self.packAny() - if v, ok := any.(map[string]interface{}); ok { - return v, nil - } else { - return nil, ErrUnsupportType - } - } - if err := self.should(types.V_OBJECT, "an object"); err != nil { - return nil, err - } - if err := self.loadAllKey(); err != nil { - return nil, err - } - return self.toGenericObject() -} - -// MapUseNumber loads all keys of an object node, with numeric nodes casted to json.Number -func (self *Node) MapUseNumber() (map[string]interface{}, error) { - if self.isAny() { - any := self.packAny() - if v, ok := any.(map[string]interface{}); ok { - return v, nil - } else { - return nil, ErrUnsupportType - } - } - if err := self.should(types.V_OBJECT, "an object"); err != nil { - return nil, err - } - if err := self.loadAllKey(); err != nil { - return nil, err - } - return self.toGenericObjectUseNumber() -} - -// MapUseNode scans both parsed and non-parsed chidren nodes, -// and map them by their keys -func (self *Node) MapUseNode() (map[string]Node, error) { - if self.isAny() { - any := self.packAny() - if v, ok := any.(map[string]Node); ok { - return v, nil - } else { - return nil, ErrUnsupportType - } - } - if err := self.should(types.V_OBJECT, "an object"); err != nil { - return nil, err - } - if err := self.skipAllKey(); err != nil { - return nil, err - } - return self.toGenericObjectUseNode() -} - -// MapUnsafe exports the underlying pointer to its children map -// WARN: don't use it unless you know what you are doing -func (self *Node) UnsafeMap() ([]Pair, error) { - if err := self.should(types.V_OBJECT, "an object"); err != nil { - return nil, err - } - if err := self.skipAllKey(); err != nil { - return nil, err - } - s := ptr2slice(self.p, int(self.len()), self.cap()) - return *(*[]Pair)(s), nil -} - -// SortKeys sorts children of a V_OBJECT node in ascending key-order. -// If recurse is true, it recursively sorts children's children as long as a V_OBJECT node is found. -func (self *Node) SortKeys(recurse bool) (err error) { - ps, err := self.UnsafeMap() - if err != nil { - return err - } - PairSlice(ps).Sort() - if recurse { - var sc Scanner - sc = func(path Sequence, node *Node) bool { - if node.itype() == types.V_OBJECT { - if err := node.SortKeys(recurse); err != nil { - return false - } - } - if node.itype() == types.V_ARRAY { - if err := node.ForEach(sc); err != nil { - return false - } - } - return true - } - self.ForEach(sc) - } - return nil -} - -// Array loads all indexes of an array node -func (self *Node) Array() ([]interface{}, error) { - if self.isAny() { - any := self.packAny() - if v, ok := any.([]interface{}); ok { - return v, nil - } else { - return nil, ErrUnsupportType - } - } - if err := self.should(types.V_ARRAY, "an array"); err != nil { - return nil, err - } - if err := self.loadAllIndex(); err != nil { - return nil, err - } - return self.toGenericArray() -} - -// ArrayUseNumber loads all indexes of an array node, with numeric nodes casted to json.Number -func (self *Node) ArrayUseNumber() ([]interface{}, error) { - if self.isAny() { - any := self.packAny() - if v, ok := any.([]interface{}); ok { - return v, nil - } else { - return nil, ErrUnsupportType - } - } - if err := self.should(types.V_ARRAY, "an array"); err != nil { - return nil, err - } - if err := self.loadAllIndex(); err != nil { - return nil, err - } - return self.toGenericArrayUseNumber() -} - -// ArrayUseNode copys both parsed and non-parsed chidren nodes, -// and indexes them by original order -func (self *Node) ArrayUseNode() ([]Node, error) { - if self.isAny() { - any := self.packAny() - if v, ok := any.([]Node); ok { - return v, nil - } else { - return nil, ErrUnsupportType - } - } - if err := self.should(types.V_ARRAY, "an array"); err != nil { - return nil, err - } - if err := self.skipAllIndex(); err != nil { - return nil, err - } - return self.toGenericArrayUseNode() -} - -// ArrayUnsafe exports the underlying pointer to its children array -// WARN: don't use it unless you know what you are doing -func (self *Node) UnsafeArray() ([]Node, error) { - if err := self.should(types.V_ARRAY, "an array"); err != nil { - return nil, err - } - if err := self.skipAllIndex(); err != nil { - return nil, err - } - s := ptr2slice(self.p, self.len(), self.cap()) - return *(*[]Node)(s), nil -} - -// Interface loads all children under all pathes from this node, -// and converts itself as generic type. -// WARN: all numberic nodes are casted to float64 -func (self *Node) Interface() (interface{}, error) { - if err := self.checkRaw(); err != nil { - return nil, err - } - switch self.t { - case V_ERROR: - return nil, self.Check() - case types.V_NULL: - return nil, nil - case types.V_TRUE: - return true, nil - case types.V_FALSE: - return false, nil - case types.V_ARRAY: - return self.toGenericArray() - case types.V_OBJECT: - return self.toGenericObject() - case types.V_STRING: - return addr2str(self.p, self.v), nil - case _V_NUMBER: - v, err := numberToFloat64(self) - if err != nil { - return nil, err - } - return v, nil - case _V_ARRAY_LAZY: - if err := self.loadAllIndex(); err != nil { - return nil, err - } - return self.toGenericArray() - case _V_OBJECT_LAZY: - if err := self.loadAllKey(); err != nil { - return nil, err - } - return self.toGenericObject() - case _V_ANY: - switch v := self.packAny().(type) { - case Node: - return v.Interface() - case *Node: - return v.Interface() - default: - return v, nil - } - default: - return nil, ErrUnsupportType - } -} - -func (self *Node) packAny() interface{} { - return *(*interface{})(self.p) -} - -// InterfaceUseNumber works same with Interface() -// except numberic nodes are casted to json.Number -func (self *Node) InterfaceUseNumber() (interface{}, error) { - if err := self.checkRaw(); err != nil { - return nil, err - } - switch self.t { - case V_ERROR: - return nil, self.Check() - case types.V_NULL: - return nil, nil - case types.V_TRUE: - return true, nil - case types.V_FALSE: - return false, nil - case types.V_ARRAY: - return self.toGenericArrayUseNumber() - case types.V_OBJECT: - return self.toGenericObjectUseNumber() - case types.V_STRING: - return addr2str(self.p, self.v), nil - case _V_NUMBER: - return toNumber(self), nil - case _V_ARRAY_LAZY: - if err := self.loadAllIndex(); err != nil { - return nil, err - } - return self.toGenericArrayUseNumber() - case _V_OBJECT_LAZY: - if err := self.loadAllKey(); err != nil { - return nil, err - } - return self.toGenericObjectUseNumber() - case _V_ANY: - return self.packAny(), nil - default: - return nil, ErrUnsupportType - } -} - -// InterfaceUseNode clone itself as a new node, -// or its children as map[string]Node (or []Node) -func (self *Node) InterfaceUseNode() (interface{}, error) { - if err := self.checkRaw(); err != nil { - return nil, err - } - switch self.t { - case types.V_ARRAY: - return self.toGenericArrayUseNode() - case types.V_OBJECT: - return self.toGenericObjectUseNode() - case _V_ARRAY_LAZY: - if err := self.skipAllIndex(); err != nil { - return nil, err - } - return self.toGenericArrayUseNode() - case _V_OBJECT_LAZY: - if err := self.skipAllKey(); err != nil { - return nil, err - } - return self.toGenericObjectUseNode() - default: - return *self, self.Check() - } -} - -// LoadAll loads all the node's children and children's children as parsed. -// After calling it, the node can be safely used on concurrency -func (self *Node) LoadAll() error { - if self.IsRaw() { - self.parseRaw(true) - return self.Check() - } - - switch self.itype() { - case types.V_ARRAY: - e := self.len() - if err := self.loadAllIndex(); err != nil { - return err - } - for i := 0; i < e; i++ { - n := self.nodeAt(i) - if n.IsRaw() { - n.parseRaw(true) - } - if err := n.Check(); err != nil { - return err - } - } - return nil - case types.V_OBJECT: - e := self.len() - if err := self.loadAllKey(); err != nil { - return err - } - for i := 0; i < e; i++ { - n := self.pairAt(i) - if n.Value.IsRaw() { - n.Value.parseRaw(true) - } - if err := n.Value.Check(); err != nil { - return err - } - } - return nil - default: - return self.Check() - } -} - -// Load loads the node's children as parsed. -// After calling it, only the node itself can be used on concurrency (not include its children) -func (self *Node) Load() error { - if self.IsRaw() { - self.parseRaw(false) - return self.Load() - } - - switch self.t { - case _V_ARRAY_LAZY: - return self.skipAllIndex() - case _V_OBJECT_LAZY: - return self.skipAllKey() - default: - return self.Check() - } -} - -/**---------------------------------- Internal Helper Methods ----------------------------------**/ - -var ( - _NODE_TYPE = rt.UnpackEface(Node{}).Type - _PAIR_TYPE = rt.UnpackEface(Pair{}).Type -) - -func (self *Node) setCapAndLen(cap int, len int) { - if self.t == types.V_ARRAY || self.t == types.V_OBJECT || self.t == _V_ARRAY_LAZY || self.t == _V_OBJECT_LAZY { - self.v = int64(len&_LEN_MASK | cap<<_CAP_BITS) - } else { - panic("value does not have a length") - } -} - -func (self *Node) unsafe_next() *Node { - return (*Node)(unsafe.Pointer(uintptr(unsafe.Pointer(self)) + _NODE_SIZE)) -} - -func (self *Pair) unsafe_next() *Pair { - return (*Pair)(unsafe.Pointer(uintptr(unsafe.Pointer(self)) + _PAIR_SIZE)) -} - -func (self *Node) must(t types.ValueType, s string) { - if err := self.checkRaw(); err != nil { - panic(err) - } - if err := self.Check(); err != nil { - panic(err) - } - if self.itype() != t { - panic("value cannot be represented as " + s) - } -} - -func (self *Node) should(t types.ValueType, s string) error { - if err := self.checkRaw(); err != nil { - return err - } - if self.itype() != t { - return ErrUnsupportType - } - return nil -} - -func (self *Node) nodeAt(i int) *Node { - var p = self.p - if self.isLazy() { - _, stack := self.getParserAndArrayStack() - p = *(*unsafe.Pointer)(unsafe.Pointer(&stack.v)) - } - return (*Node)(unsafe.Pointer(uintptr(p) + uintptr(i)*_NODE_SIZE)) -} - -func (self *Node) pairAt(i int) *Pair { - var p = self.p - if self.isLazy() { - _, stack := self.getParserAndObjectStack() - p = *(*unsafe.Pointer)(unsafe.Pointer(&stack.v)) - } - return (*Pair)(unsafe.Pointer(uintptr(p) + uintptr(i)*_PAIR_SIZE)) -} - -func (self *Node) getParserAndArrayStack() (*Parser, *parseArrayStack) { - stack := (*parseArrayStack)(self.p) - ret := (*rt.GoSlice)(unsafe.Pointer(&stack.v)) - ret.Len = self.len() - ret.Cap = self.cap() - return &stack.parser, stack -} - -func (self *Node) getParserAndObjectStack() (*Parser, *parseObjectStack) { - stack := (*parseObjectStack)(self.p) - ret := (*rt.GoSlice)(unsafe.Pointer(&stack.v)) - ret.Len = self.len() - ret.Cap = self.cap() - return &stack.parser, stack -} - -func (self *Node) skipAllIndex() error { - if !self.isLazy() { - return nil - } - var err types.ParsingError - parser, stack := self.getParserAndArrayStack() - parser.skipValue = true - parser.noLazy = true - *self, err = parser.decodeArray(stack.v) - if err != 0 { - return parser.ExportError(err) - } - return nil -} - -func (self *Node) skipAllKey() error { - if !self.isLazy() { - return nil - } - var err types.ParsingError - parser, stack := self.getParserAndObjectStack() - parser.skipValue = true - parser.noLazy = true - *self, err = parser.decodeObject(stack.v) - if err != 0 { - return parser.ExportError(err) - } - return nil -} - -func (self *Node) skipKey(key string) (*Node, int) { - nb := self.len() - lazy := self.isLazy() - - if nb > 0 { - /* linear search */ - var p *Pair - if lazy { - s := (*parseObjectStack)(self.p) - p = &s.v[0] - } else { - p = (*Pair)(self.p) - } - - if p.Key == key { - return &p.Value, 0 - } - for i := 1; i < nb; i++ { - p = p.unsafe_next() - if p.Key == key { - return &p.Value, i - } - } - } - - /* not found */ - if !lazy { - return nil, -1 - } - - // lazy load - for last, i := self.skipNextPair(), nb; last != nil; last, i = self.skipNextPair(), i+1 { - if last.Value.Check() != nil { - return &last.Value, -1 - } - if last.Key == key { - return &last.Value, i - } - } - - return nil, -1 -} - -func (self *Node) skipIndex(index int) *Node { - nb := self.len() - if nb > index { - v := self.nodeAt(index) - return v - } - if !self.isLazy() { - return nil - } - - // lazy load - for last := self.skipNextNode(); last != nil; last = self.skipNextNode() { - if last.Check() != nil { - return last - } - if self.len() > index { - return last - } - } - - return nil -} - -func (self *Node) skipIndexPair(index int) *Pair { - nb := self.len() - if nb > index { - return self.pairAt(index) - } - if !self.isLazy() { - return nil - } - - // lazy load - for last := self.skipNextPair(); last != nil; last = self.skipNextPair() { - if last.Value.Check() != nil { - return last - } - if self.len() > index { - return last - } - } - - return nil -} - -func (self *Node) loadAllIndex() error { - if !self.isLazy() { - return nil - } - var err types.ParsingError - parser, stack := self.getParserAndArrayStack() - parser.noLazy = true - *self, err = parser.decodeArray(stack.v) - if err != 0 { - return parser.ExportError(err) - } - return nil -} - -func (self *Node) loadAllKey() error { - if !self.isLazy() { - return nil - } - var err types.ParsingError - parser, stack := self.getParserAndObjectStack() - parser.noLazy = true - *self, err = parser.decodeObject(stack.v) - if err != 0 { - return parser.ExportError(err) - } - return nil -} - -func (self *Node) removeNode(i int) { - nb := self.len() - 1 - node := self.nodeAt(i) - if i == nb { - self.setCapAndLen(self.cap(), nb) - *node = Node{} - return - } - - from := self.nodeAt(i + 1) - memmove(unsafe.Pointer(node), unsafe.Pointer(from), _NODE_SIZE*uintptr(nb-i)) - - last := self.nodeAt(nb) - *last = Node{} - - self.setCapAndLen(self.cap(), nb) -} - -func (self *Node) removePair(i int) { - nb := self.len() - 1 - node := self.pairAt(i) - if i == nb { - self.setCapAndLen(self.cap(), nb) - *node = Pair{} - return - } - - from := self.pairAt(i + 1) - memmove(unsafe.Pointer(node), unsafe.Pointer(from), _PAIR_SIZE*uintptr(nb-i)) - - last := self.pairAt(nb) - *last = Pair{} - - self.setCapAndLen(self.cap(), nb) -} - -func (self *Node) toGenericArray() ([]interface{}, error) { - nb := self.len() - ret := make([]interface{}, nb) - if nb == 0 { - return ret, nil - } - - /* convert each item */ - var p = (*Node)(self.p) - x, err := p.Interface() - if err != nil { - return nil, err - } - ret[0] = x - - for i := 1; i < nb; i++ { - p = p.unsafe_next() - x, err := p.Interface() - if err != nil { - return nil, err - } - ret[i] = x - } - - /* all done */ - return ret, nil -} - -func (self *Node) toGenericArrayUseNumber() ([]interface{}, error) { - nb := self.len() - ret := make([]interface{}, nb) - if nb == 0 { - return ret, nil - } - - /* convert each item */ - var p = (*Node)(self.p) - x, err := p.InterfaceUseNumber() - if err != nil { - return nil, err - } - ret[0] = x - - for i := 1; i < nb; i++ { - p = p.unsafe_next() - x, err := p.InterfaceUseNumber() - if err != nil { - return nil, err - } - ret[i] = x - } - - /* all done */ - return ret, nil -} - -func (self *Node) toGenericArrayUseNode() ([]Node, error) { - var nb = self.len() - var out = make([]Node, nb) - if nb == 0 { - return out, nil - } - - var p = (*Node)(self.p) - out[0] = *p - if err := p.Check(); err != nil { - return nil, err - } - - for i := 1; i < nb; i++ { - p = p.unsafe_next() - if err := p.Check(); err != nil { - return nil, err - } - out[i] = *p - } - - return out, nil -} - -func (self *Node) toGenericObject() (map[string]interface{}, error) { - nb := self.len() - ret := make(map[string]interface{}, nb) - if nb == 0 { - return ret, nil - } - - /* convert each item */ - var p = (*Pair)(self.p) - x, err := p.Value.Interface() - if err != nil { - return nil, err - } - ret[p.Key] = x - - for i := 1; i < nb; i++ { - p = p.unsafe_next() - x, err := p.Value.Interface() - if err != nil { - return nil, err - } - ret[p.Key] = x - } - - /* all done */ - return ret, nil -} - -func (self *Node) toGenericObjectUseNumber() (map[string]interface{}, error) { - nb := self.len() - ret := make(map[string]interface{}, nb) - if nb == 0 { - return ret, nil - } - - /* convert each item */ - var p = (*Pair)(self.p) - x, err := p.Value.InterfaceUseNumber() - if err != nil { - return nil, err - } - ret[p.Key] = x - - for i := 1; i < nb; i++ { - p = p.unsafe_next() - x, err := p.Value.InterfaceUseNumber() - if err != nil { - return nil, err - } - ret[p.Key] = x - } - - /* all done */ - return ret, nil -} - -func (self *Node) toGenericObjectUseNode() (map[string]Node, error) { - var nb = self.len() - var out = make(map[string]Node, nb) - if nb == 0 { - return out, nil - } - - var p = (*Pair)(self.p) - out[p.Key] = p.Value - if err := p.Value.Check(); err != nil { - return nil, err - } - - for i := 1; i < nb; i++ { - p = p.unsafe_next() - if err := p.Value.Check(); err != nil { - return nil, err - } - out[p.Key] = p.Value - } - - /* all done */ - return out, nil -} - -/**------------------------------------ Factory Methods ------------------------------------**/ - -var ( - nullNode = Node{t: types.V_NULL} - trueNode = Node{t: types.V_TRUE} - falseNode = Node{t: types.V_FALSE} - - emptyArrayNode = Node{t: types.V_ARRAY} - emptyObjectNode = Node{t: types.V_OBJECT} -) - -// NewRaw creates a node of raw json, and decides its type by first char. -func NewRaw(json string) Node { - if json == "" { - panic("empty json string") - } - it := switchRawType(json[0]) - return newRawNode(json, it) -} - -// NewAny creates a node of type V_ANY if any's type isn't Node or *Node, -// which stores interface{} and can be only used for `.Interface()`\`.MarshalJSON()`. -func NewAny(any interface{}) Node { - switch n := any.(type) { - case Node: - return n - case *Node: - return *n - default: - return Node{ - t: _V_ANY, - v: 0, - p: unsafe.Pointer(&any), - } - } -} - -// NewBytes encodes given src with Base64 (RFC 4648), and creates a node of type V_STRING. -func NewBytes(src []byte) Node { - if len(src) == 0 { - panic("empty src bytes") - } - out := encodeBase64(src) - return NewString(out) -} - -// NewNull creates a node of type V_NULL -func NewNull() Node { - return Node{ - v: 0, - p: nil, - t: types.V_NULL, - } -} - -// NewBool creates a node of type bool: -// -// If v is true, returns V_TRUE node -// If v is false, returns V_FALSE node -func NewBool(v bool) Node { - var t = types.V_FALSE - if v { - t = types.V_TRUE - } - return Node{ - v: 0, - p: nil, - t: t, - } -} - -// NewNumber creates a json.Number node -// v must be a decimal string complying with RFC8259 -func NewNumber(v string) Node { - return Node{ - v: int64(len(v) & _LEN_MASK), - p: str2ptr(v), - t: _V_NUMBER, - } -} - -func toNumber(node *Node) json.Number { - return json.Number(addr2str(node.p, node.v)) -} - -func numberToFloat64(node *Node) (float64, error) { - ret, err := toNumber(node).Float64() - if err != nil { - return 0, err - } - return ret, nil -} - -func numberToInt64(node *Node) (int64, error) { - ret, err := toNumber(node).Int64() - if err != nil { - return 0, err - } - return ret, nil -} - -func newBytes(v []byte) Node { - return Node{ - t: types.V_STRING, - p: mem2ptr(v), - v: int64(len(v) & _LEN_MASK), - } -} - -// NewString creates a node of type V_STRING. -// v is considered to be a valid UTF-8 string, -// which means it won't be validated and unescaped. -// when the node is encoded to json, v will be escaped. -func NewString(v string) Node { - return Node{ - t: types.V_STRING, - p: str2ptr(v), - v: int64(len(v) & _LEN_MASK), - } -} - -// NewArray creates a node of type V_ARRAY, -// using v as its underlying children -func NewArray(v []Node) Node { - return Node{ - t: types.V_ARRAY, - v: int64(len(v)&_LEN_MASK | cap(v)<<_CAP_BITS), - p: *(*unsafe.Pointer)(unsafe.Pointer(&v)), - } -} - -func (self *Node) setArray(v []Node) { - self.t = types.V_ARRAY - self.setCapAndLen(cap(v), len(v)) - self.p = *(*unsafe.Pointer)(unsafe.Pointer(&v)) -} - -// NewObject creates a node of type V_OBJECT, -// using v as its underlying children -func NewObject(v []Pair) Node { - return Node{ - t: types.V_OBJECT, - v: int64(len(v)&_LEN_MASK | cap(v)<<_CAP_BITS), - p: *(*unsafe.Pointer)(unsafe.Pointer(&v)), - } -} - -func (self *Node) setObject(v []Pair) { - self.t = types.V_OBJECT - self.setCapAndLen(cap(v), len(v)) - self.p = *(*unsafe.Pointer)(unsafe.Pointer(&v)) -} - -type parseObjectStack struct { - parser Parser - v []Pair -} - -type parseArrayStack struct { - parser Parser - v []Node -} - -func newLazyArray(p *Parser, v []Node) Node { - s := new(parseArrayStack) - s.parser = *p - s.v = v - return Node{ - t: _V_ARRAY_LAZY, - v: int64(len(v)&_LEN_MASK | cap(v)<<_CAP_BITS), - p: unsafe.Pointer(s), - } -} - -func (self *Node) setLazyArray(p *Parser, v []Node) { - s := new(parseArrayStack) - s.parser = *p - s.v = v - self.t = _V_ARRAY_LAZY - self.setCapAndLen(cap(v), len(v)) - self.p = (unsafe.Pointer)(s) -} - -func newLazyObject(p *Parser, v []Pair) Node { - s := new(parseObjectStack) - s.parser = *p - s.v = v - return Node{ - t: _V_OBJECT_LAZY, - v: int64(len(v)&_LEN_MASK | cap(v)<<_CAP_BITS), - p: unsafe.Pointer(s), - } -} - -func (self *Node) setLazyObject(p *Parser, v []Pair) { - s := new(parseObjectStack) - s.parser = *p - s.v = v - self.t = _V_OBJECT_LAZY - self.setCapAndLen(cap(v), len(v)) - self.p = (unsafe.Pointer)(s) -} - -func newRawNode(str string, typ types.ValueType) Node { - return Node{ - t: _V_RAW | typ, - p: str2ptr(str), - v: int64(len(str) & _LEN_MASK), - } -} - -func (self *Node) parseRaw(full bool) { - raw := addr2str(self.p, self.v) - parser := NewParser(raw) - if full { - parser.noLazy = true - parser.skipValue = false - } - var e types.ParsingError - *self, e = parser.Parse() - if e != 0 { - *self = *newSyntaxError(parser.syntaxError(e)) - } -} - -func newError(err types.ParsingError, msg string) *Node { - return &Node{ - t: V_ERROR, - v: int64(err), - p: unsafe.Pointer(&msg), - } -} - -var typeJumpTable = [256]types.ValueType{ - '"': types.V_STRING, - '-': _V_NUMBER, - '0': _V_NUMBER, - '1': _V_NUMBER, - '2': _V_NUMBER, - '3': _V_NUMBER, - '4': _V_NUMBER, - '5': _V_NUMBER, - '6': _V_NUMBER, - '7': _V_NUMBER, - '8': _V_NUMBER, - '9': _V_NUMBER, - '[': types.V_ARRAY, - 'f': types.V_FALSE, - 'n': types.V_NULL, - 't': types.V_TRUE, - '{': types.V_OBJECT, -} - -func switchRawType(c byte) types.ValueType { - return typeJumpTable[c] -} - -func unwrapError(err error) *Node { - if se, ok := err.(*Node); ok { - return se - } else if sse, ok := err.(Node); ok { - return &sse - } else { - msg := err.Error() - return &Node{ - t: V_ERROR, - v: 0, - p: unsafe.Pointer(&msg), - } - } -} diff --git a/internal/json/node_test.go b/internal/json/node_test.go deleted file mode 100644 index fcb58973..00000000 --- a/internal/json/node_test.go +++ /dev/null @@ -1,1523 +0,0 @@ -/** - * Copyright 2023 CloudWeGo Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package json - -import ( - "encoding/json" - "errors" - "fmt" - "reflect" - "runtime" - "runtime/debug" - "strconv" - "testing" - - "github.com/cloudwego/dynamicgo/internal/native/types" - "github.com/cloudwego/dynamicgo/internal/rt" - "github.com/stretchr/testify/assert" -) - -func TestNodeSortKeys(t *testing.T) { - var src = `{"b":1,"a":2,"c":3}` - root, err := NewSearcher(src).GetByPath() - if err != nil { - t.Fatal(err) - } - obj, err := root.MapUseNumber() - if err != nil { - t.Fatal(err) - } - exp, err := json.Marshal(obj) - if err != nil { - t.Fatal(err) - } - if err := root.SortKeys(true); err != nil { - t.Fatal(err) - } - act, err := root.MarshalJSON() - if err != nil { - t.Fatal(err) - } - assert.Equal(t, len(exp), len(act)) - assert.Equal(t, string(exp), string(act)) -} - -func BenchmarkNodeSortKeys(b *testing.B) { - root, err := NewSearcher(_TwitterJson).GetByPath() - if err != nil { - b.Fatal(err) - } - if err := root.LoadAll(); err != nil { - b.Fatal(err) - } - - b.Run("single", func(b *testing.B) { - r := root.Get("statuses") - if r.Check() != nil { - b.Fatal(r.Error()) - } - b.SetBytes(int64(len(_TwitterJson))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - _ = root.SortKeys(false) - } - }) - b.Run("recurse", func(b *testing.B) { - b.SetBytes(int64(len(_TwitterJson))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - _ = root.SortKeys(true) - } - }) -} - -//go:noinline -func stackObj() interface{} { - var a int = 1 - return rt.UnpackEface(a).Pack() -} - -func TestStackAny(t *testing.T) { - var obj = stackObj() - any := NewAny(obj) - fmt.Printf("any: %#v\n", any) - runtime.GC() - debug.FreeOSMemory() - println("finish GC") - buf, err := any.MarshalJSON() - println("finish marshal") - if err != nil { - t.Fatal(err) - } - if string(buf) != `1` { - t.Fatal(string(buf)) - } -} - -func TestLoadAll(t *testing.T) { - e := Node{} - err := e.Load() - if err != nil { - t.Fatal(err) - } - err = e.LoadAll() - if err != nil { - t.Fatal(err) - } - - root, err := NewSearcher(`{"a":{"1":[1],"2":2},"b":[{"1":1},2],"c":[1,2]}`).GetByPath() - if err != nil { - t.Fatal(err) - } - if err = root.Load(); err != nil { - t.Fatal(err) - } - if root.len() != 3 { - t.Fatal(root.len()) - } - - c := root.Get("c") - if !c.IsRaw() { - t.Fatal(err) - } - err = c.LoadAll() - if err != nil { - t.Fatal(err) - } - if c.len() != 2 { - t.Fatal(c.len()) - } - c1 := c.nodeAt(0) - if n, err := c1.Int64(); err != nil || n != 1 { - t.Fatal(n, err) - } - - a := root.pairAt(0) - if a.Key != "a" { - t.Fatal(a.Key) - } else if !a.Value.IsRaw() { - t.Fatal(a.Value.itype()) - } else if n, err := a.Value.Len(); n != 0 || err != nil { - t.Fatal(n, err) - } - if err := a.Value.Load(); err != nil { - t.Fatal(err) - } - if a.Value.len() != 2 { - t.Fatal(a.Value.len()) - } - a1 := a.Value.Get("1") - if !a1.IsRaw() { - t.Fatal(a1) - } - a.Value.LoadAll() - if a1.t != types.V_ARRAY || a1.len() != 1 { - t.Fatal(a1.t, a1.len()) - } - - b := root.pairAt(1) - if b.Key != "b" { - t.Fatal(b.Key) - } else if !b.Value.IsRaw() { - t.Fatal(b.Value.itype()) - } else if n, err := b.Value.Len(); n != 0 || err != nil { - t.Fatal(n, err) - } - if err := b.Value.Load(); err != nil { - t.Fatal(err) - } - if b.Value.len() != 2 { - t.Fatal(b.Value.len()) - } - b1 := b.Value.Index(0) - if !b1.IsRaw() { - t.Fatal(b1) - } - b.Value.LoadAll() - if b1.t != types.V_OBJECT || b1.len() != 1 { - t.Fatal(a1.t, a1.len()) - } -} - -func TestIndexPair(t *testing.T) { - root, _ := NewParser(`{"a":1,"b":2}`).Parse() - a := root.IndexPair(0) - if a == nil || a.Key != "a" { - t.Fatal(a) - } - b := root.IndexPair(1) - if b == nil || b.Key != "b" { - t.Fatal(b) - } - c := root.IndexPair(2) - if c != nil { - t.Fatal(c) - } -} - -func TestIndexOrGet(t *testing.T) { - root, _ := NewParser(`{"a":1,"b":2}`).Parse() - a := root.IndexOrGet(0, "a") - if v, err := a.Int64(); err != nil || v != int64(1) { - t.Fatal(a) - } - a = root.IndexOrGet(0, "b") - if v, err := a.Int64(); err != nil || v != int64(2) { - t.Fatal(a) - } - a = root.IndexOrGet(0, "c") - if a.Valid() { - t.Fatal(a) - } -} - -func TestTypeCast(t *testing.T) { - type tcase struct { - method string - node Node - exp interface{} - err error - } - var nonEmptyErr error = errors.New("") - a1 := NewAny(1) - lazyArray, _ := NewParser("[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]").Parse() - lazyObject, _ := NewParser(`{"0":0,"1":1,"2":2,"3":3,"4":4,"5":5,"6":6,"7":7,"8":8,"9":9,"10":10,"11":11,"12":12,"13":13,"14":14,"15":15,"16":16}`).Parse() - var cases = []tcase{ - {"Interface", Node{}, interface{}(nil), ErrUnsupportType}, - {"Interface", NewAny(NewNumber("1")), float64(1), nil}, - {"Interface", NewAny(int64(1)), int64(1), nil}, - {"Interface", NewNumber("1"), float64(1), nil}, - {"InterfaceUseNode", Node{}, Node{}, nil}, - {"InterfaceUseNode", a1, a1, nil}, - {"InterfaceUseNode", NewNumber("1"), NewNumber("1"), nil}, - {"InterfaceUseNumber", Node{}, interface{}(nil), ErrUnsupportType}, - {"InterfaceUseNumber", NewAny(1), 1, nil}, - {"InterfaceUseNumber", NewNumber("1"), json.Number("1"), nil}, - {"Map", Node{}, map[string]interface{}(nil), ErrUnsupportType}, - {"Map", NewAny(map[string]Node{"a": NewNumber("1")}), map[string]interface{}(nil), ErrUnsupportType}, - {"Map", NewAny(map[string]interface{}{"a": 1}), map[string]interface{}{"a": 1}, nil}, - {"Map", NewObject([]Pair{{"a", NewNumber("1")}}), map[string]interface{}{"a": float64(1.0)}, nil}, - {"MapUseNode", Node{}, map[string]Node(nil), ErrUnsupportType}, - {"MapUseNode", NewAny(map[string]interface{}{"a": 1}), map[string]Node(nil), ErrUnsupportType}, - {"MapUseNode", NewAny(map[string]Node{"a": NewNumber("1")}), map[string]Node{"a": NewNumber("1")}, nil}, - {"MapUseNode", NewObject([]Pair{{"a", NewNumber("1")}}), map[string]Node{"a": NewNumber("1")}, nil}, - {"MapUseNumber", Node{}, map[string]interface{}(nil), ErrUnsupportType}, - {"MapUseNumber", NewAny(map[string]interface{}{"a": 1}), map[string]interface{}{"a": 1}, nil}, - {"MapUseNumber", NewObject([]Pair{{"a", NewNumber("1")}}), map[string]interface{}{"a": json.Number("1")}, nil}, - {"Array", Node{}, []interface{}(nil), ErrUnsupportType}, - {"Array", NewAny([]interface{}{1}), []interface{}{1}, nil}, - {"Array", NewArray([]Node{NewNumber("1")}), []interface{}{float64(1.0)}, nil}, - {"ArrayUseNode", Node{}, []Node(nil), ErrUnsupportType}, - {"ArrayUseNode", NewAny([]interface{}{1}), []Node(nil), ErrUnsupportType}, - {"ArrayUseNode", NewAny([]Node{NewNumber("1")}), []Node{NewNumber("1")}, nil}, - {"ArrayUseNode", NewArray([]Node{NewNumber("1")}), []Node{NewNumber("1")}, nil}, - {"ArrayUseNumber", Node{}, []interface{}(nil), ErrUnsupportType}, - {"ArrayUseNumber", NewAny([]interface{}{1}), []interface{}{1}, nil}, - {"ArrayUseNumber", NewAny([]Node{NewNumber("1")}), []interface{}(nil), ErrUnsupportType}, - {"ArrayUseNumber", NewArray([]Node{NewNumber("1")}), []interface{}{json.Number("1")}, nil}, - {"Raw", Node{}, "", ErrNotExist}, - {"Raw", NewAny(""), `""`, nil}, - {"Raw", NewRaw("[ ]"), "[ ]", nil}, - {"Raw", NewBool(true), "true", nil}, - {"Raw", NewNumber("-0.0"), "-0.0", nil}, - {"Raw", NewString(""), `""`, nil}, - {"Raw", NewBytes([]byte("hello, world")), `"aGVsbG8sIHdvcmxk"`, nil}, - {"Bool", Node{}, false, ErrUnsupportType}, - {"Bool", NewAny(true), true, nil}, - {"Bool", NewAny(false), false, nil}, - {"Bool", NewAny(int(0)), false, nil}, - {"Bool", NewAny(int8(1)), true, nil}, - {"Bool", NewAny(int16(1)), true, nil}, - {"Bool", NewAny(int32(1)), true, nil}, - {"Bool", NewAny(int64(1)), true, nil}, - {"Bool", NewAny(uint(1)), true, nil}, - {"Bool", NewAny(uint16(1)), true, nil}, - {"Bool", NewAny(uint32(1)), true, nil}, - {"Bool", NewAny(uint64(1)), true, nil}, - {"Bool", NewAny(float64(0)), false, nil}, - {"Bool", NewAny(float32(1)), true, nil}, - {"Bool", NewAny(float64(1)), true, nil}, - {"Bool", NewAny(json.Number("0")), false, nil}, - {"Bool", NewAny(json.Number("1")), true, nil}, - {"Bool", NewAny(json.Number("1.1")), true, nil}, - {"Bool", NewAny(json.Number("+x1.1")), false, nonEmptyErr}, - {"Bool", NewAny(string("0")), false, nil}, - {"Bool", NewAny(string("t")), true, nil}, - {"Bool", NewAny([]byte{0}), false, nonEmptyErr}, - {"Bool", NewRaw("true"), true, nil}, - {"Bool", NewRaw("false"), false, nil}, - {"Bool", NewRaw("null"), false, nil}, - {"Bool", NewString(`true`), true, nil}, - {"Bool", NewString(`false`), false, nil}, - {"Bool", NewString(``), false, nonEmptyErr}, - {"Bool", NewNumber("2"), true, nil}, - {"Bool", NewNumber("-2.1"), true, nil}, - {"Bool", NewNumber("-x-2.1"), false, nonEmptyErr}, - {"Int64", NewRaw("true"), int64(1), nil}, - {"Int64", NewRaw("false"), int64(0), nil}, - {"Int64", NewRaw("\"1\""), int64(1), nil}, - {"Int64", NewRaw("\"1.1\""), int64(1), nil}, - {"Int64", NewRaw("\"1.0\""), int64(1), nil}, - {"Int64", NewNumber("+x.0"), int64(0), nonEmptyErr}, - {"Int64", NewAny(false), int64(0), nil}, - {"Int64", NewAny(true), int64(1), nil}, - {"Int64", NewAny(int(1)), int64(1), nil}, - {"Int64", NewAny(int8(1)), int64(1), nil}, - {"Int64", NewAny(int16(1)), int64(1), nil}, - {"Int64", NewAny(int32(1)), int64(1), nil}, - {"Int64", NewAny(int64(1)), int64(1), nil}, - {"Int64", NewAny(uint(1)), int64(1), nil}, - {"Int64", NewAny(uint8(1)), int64(1), nil}, - {"Int64", NewAny(uint32(1)), int64(1), nil}, - {"Int64", NewAny(uint64(1)), int64(1), nil}, - {"Int64", NewAny(float32(1)), int64(1), nil}, - {"Int64", NewAny(float64(1)), int64(1), nil}, - {"Int64", NewAny("1"), int64(1), nil}, - {"Int64", NewAny("1.1"), int64(1), nil}, - {"Int64", NewAny("+1x.1"), int64(0), nonEmptyErr}, - {"Int64", NewAny(json.Number("1")), int64(1), nil}, - {"Int64", NewAny(json.Number("1.1")), int64(1), nil}, - {"Int64", NewAny(json.Number("+1x.1")), int64(0), nonEmptyErr}, - {"Int64", NewAny([]byte{0}), int64(0), ErrUnsupportType}, - {"Int64", Node{}, int64(0), ErrUnsupportType}, - {"Int64", NewRaw("0"), int64(0), nil}, - {"Int64", NewRaw("null"), int64(0), nil}, - {"StrictInt64", NewRaw("true"), int64(0), ErrUnsupportType}, - {"StrictInt64", NewRaw("false"), int64(0), ErrUnsupportType}, - {"StrictInt64", NewAny(int(0)), int64(0), nil}, - {"StrictInt64", NewAny(int8(0)), int64(0), nil}, - {"StrictInt64", NewAny(int16(0)), int64(0), nil}, - {"StrictInt64", NewAny(int32(0)), int64(0), nil}, - {"StrictInt64", NewAny(int64(0)), int64(0), nil}, - {"StrictInt64", NewAny(uint(0)), int64(0), nil}, - {"StrictInt64", NewAny(uint8(0)), int64(0), nil}, - {"StrictInt64", NewAny(uint32(0)), int64(0), nil}, - {"StrictInt64", NewAny(uint64(0)), int64(0), nil}, - {"StrictInt64", Node{}, int64(0), ErrUnsupportType}, - {"StrictInt64", NewRaw("0"), int64(0), nil}, - {"StrictInt64", NewRaw("null"), int64(0), ErrUnsupportType}, - {"Float64", NewRaw("true"), float64(1), nil}, - {"Float64", NewRaw("false"), float64(0), nil}, - {"Float64", NewRaw("\"1.0\""), float64(1.0), nil}, - {"Float64", NewRaw("\"xx\""), float64(0), nonEmptyErr}, - {"Float64", Node{}, float64(0), ErrUnsupportType}, - {"Float64", NewAny(false), float64(0), nil}, - {"Float64", NewAny(true), float64(1), nil}, - {"Float64", NewAny(int(1)), float64(1), nil}, - {"Float64", NewAny(int8(1)), float64(1), nil}, - {"Float64", NewAny(int16(1)), float64(1), nil}, - {"Float64", NewAny(int32(1)), float64(1), nil}, - {"Float64", NewAny(int64(1)), float64(1), nil}, - {"Float64", NewAny(uint(1)), float64(1), nil}, - {"Float64", NewAny(uint8(1)), float64(1), nil}, - {"Float64", NewAny(uint32(1)), float64(1), nil}, - {"Float64", NewAny(uint64(1)), float64(1), nil}, - {"Float64", NewAny(float32(1)), float64(1), nil}, - {"Float64", NewAny(float64(1)), float64(1), nil}, - {"Float64", NewAny("1.1"), float64(1.1), nil}, - {"Float64", NewAny("+1x.1"), float64(0), nonEmptyErr}, - {"Float64", NewAny(json.Number("0")), float64(0), nil}, - {"Float64", NewAny(json.Number("x")), float64(0), nonEmptyErr}, - {"Float64", NewAny([]byte{0}), float64(0), ErrUnsupportType}, - {"Float64", NewRaw("0.0"), float64(0.0), nil}, - {"Float64", NewRaw("1"), float64(1.0), nil}, - {"Float64", NewRaw("null"), float64(0.0), nil}, - {"StrictFloat64", NewRaw("true"), float64(0), ErrUnsupportType}, - {"StrictFloat64", NewRaw("false"), float64(0), ErrUnsupportType}, - {"StrictFloat64", Node{}, float64(0), ErrUnsupportType}, - {"StrictFloat64", NewAny(float32(0)), float64(0), nil}, - {"StrictFloat64", NewAny(float64(0)), float64(0), nil}, - {"StrictFloat64", NewRaw("0.0"), float64(0.0), nil}, - {"StrictFloat64", NewRaw("null"), float64(0.0), ErrUnsupportType}, - {"Number", Node{}, json.Number(""), ErrUnsupportType}, - {"Number", NewAny(false), json.Number("0"), nil}, - {"Number", NewAny(true), json.Number("1"), nil}, - {"Number", NewAny(int(1)), json.Number("1"), nil}, - {"Number", NewAny(int8(1)), json.Number("1"), nil}, - {"Number", NewAny(int16(1)), json.Number("1"), nil}, - {"Number", NewAny(int32(1)), json.Number("1"), nil}, - {"Number", NewAny(int64(1)), json.Number("1"), nil}, - {"Number", NewAny(uint(1)), json.Number("1"), nil}, - {"Number", NewAny(uint8(1)), json.Number("1"), nil}, - {"Number", NewAny(uint32(1)), json.Number("1"), nil}, - {"Number", NewAny(uint64(1)), json.Number("1"), nil}, - {"Number", NewAny(float32(1)), json.Number("1"), nil}, - {"Number", NewAny(float64(1)), json.Number("1"), nil}, - {"Number", NewAny("1.1"), json.Number("1.1"), nil}, - {"Number", NewAny("+1x.1"), json.Number(""), nonEmptyErr}, - {"Number", NewAny(json.Number("0")), json.Number("0"), nil}, - {"Number", NewAny(json.Number("x")), json.Number("x"), nil}, - {"Number", NewAny(json.Number("+1x.1")), json.Number("+1x.1"), nil}, - {"Number", NewAny([]byte{0}), json.Number(""), ErrUnsupportType}, - {"Number", NewRaw("x"), json.Number(""), nonEmptyErr}, - {"Number", NewRaw("0.0"), json.Number("0.0"), nil}, - {"Number", NewRaw("\"1\""), json.Number("1"), nil}, - {"Number", NewRaw("\"1.1\""), json.Number("1.1"), nil}, - {"Number", NewRaw("\"0.x0\""), json.Number(""), nonEmptyErr}, - {"Number", NewRaw("{]"), json.Number(""), nonEmptyErr}, - {"Number", NewRaw("true"), json.Number("1"), nil}, - {"Number", NewRaw("false"), json.Number("0"), nil}, - {"Number", NewRaw("null"), json.Number("0"), nil}, - {"StrictNumber", NewRaw("true"), json.Number(""), ErrUnsupportType}, - {"StrictNumber", NewRaw("false"), json.Number(""), ErrUnsupportType}, - {"StrictNumber", Node{}, json.Number(""), ErrUnsupportType}, - {"StrictNumber", NewAny(json.Number("0")), json.Number("0"), nil}, - {"StrictNumber", NewRaw("0.0"), json.Number("0.0"), nil}, - {"StrictNumber", NewRaw("null"), json.Number(""), ErrUnsupportType}, - {"String", Node{}, "", ErrUnsupportType}, - {"String", NewAny(`\u263a`), `\u263a`, nil}, - {"String", NewRaw(`"\u263a"`), `☺`, nil}, - {"String", NewString(`\u263a`), `\u263a`, nil}, - {"String", NewRaw(`0.0`), "0.0", nil}, - {"String", NewRaw(`true`), "true", nil}, - {"String", NewRaw(`false`), "false", nil}, - {"String", NewRaw(`null`), "", nil}, - {"String", NewAny(false), "false", nil}, - {"String", NewAny(true), "true", nil}, - {"String", NewAny(int(1)), "1", nil}, - {"String", NewAny(int8(1)), "1", nil}, - {"String", NewAny(int16(1)), "1", nil}, - {"String", NewAny(int32(1)), "1", nil}, - {"String", NewAny(int64(1)), "1", nil}, - {"String", NewAny(uint(1)), "1", nil}, - {"String", NewAny(uint8(1)), "1", nil}, - {"String", NewAny(uint32(1)), "1", nil}, - {"String", NewAny(uint64(1)), "1", nil}, - {"String", NewAny(float32(1)), "1", nil}, - {"String", NewAny(float64(1)), "1", nil}, - {"String", NewAny("1.1"), "1.1", nil}, - {"String", NewAny("+1x.1"), "+1x.1", nil}, - {"String", NewAny(json.Number("0")), ("0"), nil}, - {"String", NewAny(json.Number("x")), ("x"), nil}, - {"String", NewAny([]byte{0}), (""), ErrUnsupportType}, - {"StrictString", Node{}, "", ErrUnsupportType}, - {"StrictString", NewAny(`\u263a`), `\u263a`, nil}, - {"StrictString", NewRaw(`"\u263a"`), `☺`, nil}, - {"StrictString", NewString(`\u263a`), `\u263a`, nil}, - {"StrictString", NewRaw(`0.0`), "", ErrUnsupportType}, - {"StrictString", NewRaw(`true`), "", ErrUnsupportType}, - {"StrictString", NewRaw(`false`), "", ErrUnsupportType}, - {"StrictString", NewRaw(`null`), "", ErrUnsupportType}, - {"Len", Node{}, 0, nil}, - {"Len", NewAny(0), 0, ErrUnsupportType}, - {"Len", NewNull(), 0, nil}, - {"Len", NewRaw(`"1"`), 1, nil}, - {"Len", NewRaw(`[1]`), 0, nil}, - {"Len", NewArray([]Node{NewNull()}), 1, nil}, - {"Len", lazyArray, 0, nil}, - {"Len", NewRaw(`{"a":1}`), 0, nil}, - {"Len", lazyObject, 0, nil}, - {"Cap", Node{}, 0, nil}, - {"Cap", NewAny(0), 0, ErrUnsupportType}, - {"Cap", NewNull(), 0, nil}, - {"Cap", NewRaw(`[1]`), _DEFAULT_NODE_CAP, nil}, - {"Cap", NewObject([]Pair{{"", NewNull()}}), 1, nil}, - {"Cap", NewRaw(`{"a":1}`), _DEFAULT_NODE_CAP, nil}, - } - lazyArray.skipAllIndex() - lazyObject.skipAllKey() - cases = append(cases, - tcase{"Len", lazyObject, 17, nil}, - tcase{"Len", lazyObject, 17, nil}, - tcase{"Cap", lazyObject, _DEFAULT_NODE_CAP * 2, nil}, - tcase{"Cap", lazyObject, _DEFAULT_NODE_CAP * 2, nil}, - ) - - for i, c := range cases { - fmt.Println(i, c) - rt := reflect.ValueOf(&c.node) - m := rt.MethodByName(c.method) - rets := m.Call([]reflect.Value{}) - if len(rets) != 2 { - t.Error(i, rets) - } - if !reflect.DeepEqual(rets[0].Interface(), c.exp) { - t.Error(i, rets[0].Interface(), c.exp) - } - v := rets[1].Interface() - if c.err == nonEmptyErr { - if reflect.ValueOf(v).IsNil() { - t.Error(i, v) - } - } else if v != c.err { - t.Error(i, v) - } - } -} - -func TestCheckError(t *testing.T) { - empty := Node{} - if !empty.Valid() || empty.Check() != nil || empty.Error() != "" { - t.Fatal() - } - - n := newRawNode("[hello]", types.V_ARRAY) - n.parseRaw(false) - if n.Check() != nil { - t.Fatal(n.Check()) - } - n = newRawNode("[hello]", types.V_ARRAY) - n.parseRaw(true) - p := NewParser("[hello]") - p.noLazy = true - p.skipValue = false - _, x := p.Parse() - if n.Error() != newSyntaxError(p.syntaxError(x)).Error() { - t.Fatal(n.Check()) - } - - s, err := NewParser(`{"a":{}, "b":talse, "c":{}}`).Parse() - if err != 0 { - t.Fatal(err) - } - - root := s.GetByPath() - // fmt.Println(root.Check()) - a := root.Get("a") - if a.Check() != nil { - t.Fatal(a.Check()) - } - c := root.Get("c") - if c.Check() == nil { - t.Fatal() - } - fmt.Println(c.Check()) - - _, e := a.Properties() - if e != nil { - t.Fatal(e) - } - exist, e := a.Set("d", newRawNode("x", types.V_OBJECT)) - if exist || e != nil { - t.Fatal(err) - } - if a.len() != 1 { - t.Fail() - } - d := a.Get("d").Get("") - if d.Check() == nil { - t.Fatal(d) - } - exist, e = a.Set("e", newRawNode("[}", types.V_ARRAY)) - if e != nil { - t.Fatal(e) - } - if a.len() != 2 { - t.Fail() - } - d = a.Index(1).Index(0) - if d.Check() == nil { - t.Fatal(d) - } - - it, e := root.Interface() - if e == nil { - t.Fatal(it) - } - fmt.Println(e) -} - -func TestIndex(t *testing.T) { - root, derr := NewParser(_TwitterJson).Parse() - if derr != 0 { - t.Fatalf("decode failed: %v", derr.Error()) - } - status := root.GetByPath("statuses", 0) - x, _ := status.Index(4).String() - y, _ := status.Get("id_str").String() - if x != y { - t.Fail() - } -} - -func TestUnset(t *testing.T) { - root, derr := NewParser(_TwitterJson).Parse() - if derr != 0 { - t.Fatalf("decode failed: %v", derr.Error()) - } - entities := root.GetByPath("statuses", 0, "entities") - if !entities.Exists() { - t.Fatal() - } - exist, err := entities.Unset("urls") - if !exist || err != nil { - t.Fatal() - } - e := entities.Get("urls") - if e.Exists() { - t.Fatal() - } - if entities.len() != 2 { - t.Fatal() - } - - entities.Set("urls", NewString("a")) - e = entities.Get("urls") - x, _ := e.String() - if !e.Exists() || x != "a" { - t.Fatal() - } - exist, err = entities.UnsetByIndex(entities.len() - 1) - if !exist || err != nil { - t.Fatal() - } - e = entities.Get("urls") - if e.Exists() { - t.Fatal() - } - - hashtags := entities.Get("hashtags").Index(0) - hashtags.Set("text2", newRawNode(`{}`, types.V_OBJECT)) - exist, err = hashtags.Unset("indices") - if !exist || err != nil || hashtags.len() != 2 { - t.Fatal() - } - y, _ := hashtags.Get("text").String() - if y != "freebandnames" { - t.Fatal() - } - if hashtags.Get("text2").Type() != V_OBJECT { - t.Fatal() - } - - ums := entities.Get("user_mentions") - ums.Add(NewNull()) - ums.Add(NewBool(true)) - ums.Add(NewBool(false)) - if ums.len() != 3 { - t.Fatal() - } - exist, err = ums.UnsetByIndex(1) - if !exist || err != nil { - t.Fatal() - } - v1, _ := ums.Index(0).Interface() - v2, _ := ums.Index(1).Interface() - if v1 != nil || v2 != false { - t.Fatal() - } - -} - -func TestUnsafeNode(t *testing.T) { - str, loop := getTestIteratorSample(_DEFAULT_NODE_CAP) - - root, err := NewSearcher(str).GetByPath("array") - if err != nil { - t.Fatal(err) - } - a, _ := root.UnsafeArray() - if len(a) != loop { - t.Fatalf("exp:%v, got:%v", loop, len(a)) - } - for i := int64(0); i < int64(loop); i++ { - in := a[i] - x, _ := in.Int64() - if x != i { - t.Fatalf("exp:%v, got:%v", i, x) - } - } - - root, err = NewSearcher(str).GetByPath("object") - if err != nil { - t.Fatal(err) - } - b, _ := root.UnsafeMap() - if len(b) != loop { - t.Fatalf("exp:%v, got:%v", loop, len(b)) - } - for i := int64(0); i < int64(loop); i++ { - k := `k` + strconv.Itoa(int(i)) - if k != b[i].Key { - t.Fatalf("unexpected element: %#v", b[i]) - } - x, _ := b[i].Value.Int64() - if x != i { - t.Fatalf("exp:%v, got:%v", i, x) - } - } -} - -func TestUseNode(t *testing.T) { - str, loop := getTestIteratorSample(_DEFAULT_NODE_CAP) - root, e := NewParser(str).Parse() - if e != 0 { - t.Fatal(e) - } - _, er := root.InterfaceUseNode() - if er != nil { - t.Fatal(er) - } - - root, err := NewSearcher(str).GetByPath("array") - if err != nil { - t.Fatal(err) - } - a, _ := root.ArrayUseNode() - if len(a) != loop { - t.Fatalf("exp:%v, got:%v", loop, len(a)) - } - for i := int64(0); i < int64(loop); i++ { - in := a[i] - a, _ := in.Int64() - if a != i { - t.Fatalf("exp:%v, got:%v", i, a) - } - } - - root, err = NewSearcher(str).GetByPath("array") - if err != nil { - t.Fatal(err) - } - x, _ := root.InterfaceUseNode() - a = x.([]Node) - if len(a) != loop { - t.Fatalf("exp:%v, got:%v", loop, len(a)) - } - for i := int64(0); i < int64(loop); i++ { - in := a[i] - a, _ := in.Int64() - if a != i { - t.Fatalf("exp:%v, got:%v", i, a) - } - } - - root, err = NewSearcher(str).GetByPath("object") - if err != nil { - t.Fatal(err) - } - b, _ := root.MapUseNode() - if len(b) != loop { - t.Fatalf("exp:%v, got:%v", loop, len(b)) - } - for i := int64(0); i < int64(loop); i++ { - k := `k` + strconv.Itoa(int(i)) - xn, ok := b[k] - if !ok { - t.Fatalf("unexpected element: %#v", xn) - } - a, _ := xn.Int64() - if a != i { - t.Fatalf("exp:%v, got:%v", i, a) - } - } - - root, err = NewSearcher(str).GetByPath("object") - if err != nil { - t.Fatal(err) - } - x, _ = root.InterfaceUseNode() - b = x.(map[string]Node) - if len(b) != loop { - t.Fatalf("exp:%v, got:%v", loop, len(b)) - } - for i := int64(0); i < int64(loop); i++ { - k := `k` + strconv.Itoa(int(i)) - xn, ok := b[k] - if !ok { - t.Fatalf("unexpected element: %#v", xn) - } - a, _ := xn.Int64() - if a != i { - t.Fatalf("exp:%v, got:%v", i, a) - } - } -} - -func TestUseNumber(t *testing.T) { - str, _ := getTestIteratorSample(_DEFAULT_NODE_CAP) - root, e := NewParser(str).Parse() - if e != 0 { - t.Fatal(e) - } - _, er := root.InterfaceUseNumber() - if er != nil { - t.Fatal(er) - } - - node, err := NewParser("1061346755812312312313").Parse() - if err != 0 { - t.Fatal(err) - } - if node.Type() != V_NUMBER { - t.Fatalf("wrong type: %v", node.Type()) - } - x, _ := node.InterfaceUseNumber() - iv := x.(json.Number) - if iv.String() != "1061346755812312312313" { - t.Fatalf("exp:%#v, got:%#v", "1061346755812312312313", iv.String()) - } - x, _ = node.Interface() - ix := x.(float64) - if ix != float64(1061346755812312312313) { - t.Fatalf("exp:%#v, got:%#v", float64(1061346755812312312313), ix) - } - xj, _ := node.Number() - ij, _ := xj.Int64() - jj, _ := json.Number("1061346755812312312313").Int64() - if ij != jj { - t.Fatalf("exp:%#v, got:%#v", jj, ij) - } -} - -func TestMap(t *testing.T) { - node, err := NewParser(`{"a":-0, "b":1, "c":-1.2, "d":-1.2e-10}`).Parse() - if err != 0 { - t.Fatal(err) - } - m, _ := node.Map() - assert.Equal(t, m, map[string]interface{}{ - "a": float64(0), - "b": float64(1), - "c": -1.2, - "d": -1.2e-10, - }) - m1, _ := node.MapUseNumber() - assert.Equal(t, m1, map[string]interface{}{ - "a": json.Number("-0"), - "b": json.Number("1"), - "c": json.Number("-1.2"), - "d": json.Number("-1.2e-10"), - }) -} - -func TestArray(t *testing.T) { - node, err := NewParser(`[-0, 1, -1.2, -1.2e-10]`).Parse() - if err != 0 { - t.Fatal(err) - } - m, _ := node.Array() - assert.Equal(t, m, []interface{}{ - float64(0), - float64(1), - -1.2, - -1.2e-10, - }) - m1, _ := node.ArrayUseNumber() - assert.Equal(t, m1, []interface{}{ - json.Number("-0"), - json.Number("1"), - json.Number("-1.2"), - json.Number("-1.2e-10"), - }) -} - -func TestNodeRaw(t *testing.T) { - root, derr := NewSearcher(_TwitterJson).GetByPath("search_metadata") - if derr != nil { - t.Fatalf("decode failed: %v", derr.Error()) - } - val, _ := root.Raw() - var comp = `{ - "max_id": 250126199840518145, - "since_id": 24012619984051000, - "refresh_url": "?since_id=250126199840518145&q=%23freebandnames&result_type=mixed&include_entities=1", - "next_results": "?max_id=249279667666817023&q=%23freebandnames&count=4&include_entities=1&result_type=mixed", - "count": 4, - "completed_in": 0.035, - "since_id_str": "24012619984051000", - "query": "%23freebandnames", - "max_id_str": "250126199840518145" - }` - if val != comp { - t.Fatalf("exp: %+v, got: %+v", comp, val) - } - - root, derr = NewSearcher(_TwitterJson).GetByPath("statuses", 0, "entities", "hashtags") - if derr != nil { - t.Fatalf("decode failed: %v", derr.Error()) - } - val, _ = root.Raw() - comp = `[ - { - "text": "freebandnames", - "indices": [ - 20, - 34 - ] - } - ]` - if val != comp { - t.Fatalf("exp: \n%s\n, got: \n%s\n", comp, val) - } - - var data = `{"k1" : { "a" : "bc"} , "k2" : [1,2 ] , "k3":{} , "k4":[]}` - root, derr = NewSearcher(data).GetByPath("k1") - if derr != nil { - t.Fatalf("decode failed: %v", derr.Error()) - } - val, _ = root.Raw() - comp = `{ "a" : "bc"}` - if val != comp { - t.Fatalf("exp: %+v, got: %+v", comp, val) - } - - root, derr = NewSearcher(data).GetByPath("k2") - if derr != nil { - t.Fatalf("decode failed: %v", derr.Error()) - } - val, _ = root.Raw() - comp = `[1,2 ]` - if val != comp { - t.Fatalf("exp: %+v, got: %+v", comp, val) - } - - root, derr = NewSearcher(data).GetByPath("k3") - if derr != nil { - t.Fatalf("decode failed: %v", derr.Error()) - } - val, _ = root.Raw() - comp = `{}` - if val != comp { - t.Fatalf("exp: %+v, got: %+v", comp, val) - } - - root, derr = NewSearcher(data).GetByPath("k4") - if derr != nil { - t.Fatalf("decode failed: %v", derr.Error()) - } - val, _ = root.Raw() - comp = `[]` - if val != comp { - t.Fatalf("exp: %+v, got: %+v", comp, val) - } -} - -func TestNodeGet(t *testing.T) { - root, derr := NewParser(_TwitterJson).Parse() - if derr != 0 { - t.Fatalf("decode failed: %v", derr.Error()) - } - val, _ := root.Get("search_metadata").Get("max_id").Int64() - if val != int64(250126199840518145) { - t.Fatalf("exp: %+v, got: %+v", 250126199840518145, val) - } -} - -func TestNodeIndex(t *testing.T) { - root, derr := NewParser(_TwitterJson).Parse() - if derr != 0 { - t.Fatalf("decode failed: %v", derr.Error()) - } - val, _ := root.Get("statuses").Index(3).Get("id_str").String() - if val != "249279667666817024" { - t.Fatalf("exp: %+v, got: %+v", "249279667666817024", val) - } -} - -func TestNodeGetByPath(t *testing.T) { - root, derr := NewParser(_TwitterJson).Parse() - if derr != 0 { - t.Fatalf("decode failed: %v", derr.Error()) - } - val, _ := root.GetByPath("statuses", 3, "id_str").String() - if val != "249279667666817024" { - t.Fatalf("exp: %+v, got: %+v", "249279667666817024", val) - } -} - -func TestNodeSet(t *testing.T) { - empty := Node{} - err := empty.Add(Node{}) - if err != nil { - t.Fatal(err) - } - empty2 := empty.Index(0) - if empty2.Check() != nil { - t.Fatal(err) - } - exist, err := empty2.SetByIndex(1, Node{}) - if exist || err == nil { - t.Fatal(exist, err) - } - empty3 := empty.Index(0) - if empty3.Check() != nil { - t.Fatal(err) - } - exist, err = empty3.Set("a", NewNumber("-1")) - if exist || err != nil { - t.Fatal(exist, err) - } - if n, e := empty.Index(0).Get("a").Int64(); e != nil || n != -1 { - t.Fatal(n, e) - } - - empty = NewNull() - err = empty.Add(NewNull()) - if err != nil { - t.Fatal(err) - } - empty2 = empty.Index(0) - if empty2.Check() != nil { - t.Fatal(err) - } - exist, err = empty2.SetByIndex(1, NewNull()) - if exist || err == nil { - t.Fatal(exist, err) - } - empty3 = empty.Index(0) - if empty3.Check() != nil { - t.Fatal(err) - } - exist, err = empty3.Set("a", NewNumber("-1")) - if exist || err != nil { - t.Fatal(exist, err) - } - if n, e := empty.Index(0).Get("a").Int64(); e != nil || n != -1 { - t.Fatal(n, e) - } - - root, derr := NewParser(_TwitterJson).Parse() - if derr != 0 { - t.Fatalf("decode failed: %v", derr.Error()) - } - app, _ := NewParser("111").Parse() - root.GetByPath("statuses", 3).Set("id_str", app) - val, _ := root.GetByPath("statuses", 3, "id_str").Int64() - if val != 111 { - t.Fatalf("exp: %+v, got: %+v", 111, val) - } - for i := root.GetByPath("statuses", 3).cap(); i >= 0; i-- { - root.GetByPath("statuses", 3).Set("id_str"+strconv.Itoa(i), app) - } - val, _ = root.GetByPath("statuses", 3, "id_str0").Int64() - if val != 111 { - t.Fatalf("exp: %+v, got: %+v", 111, val) - } - - nroot, derr := NewParser(`{"a":[0.1,true,0,"name",{"b":"c"}]}`).Parse() - if derr != 0 { - t.Fatalf("decode failed: %v", derr.Error()) - } - root.GetByPath("statuses", 3).Set("id_str2", nroot) - val2, _ := root.GetByPath("statuses", 3, "id_str2", "a", 4, "b").String() - if val2 != "c" { - t.Fatalf("exp:%+v, got:%+v", "c", val2) - } -} - -func TestNodeAny(t *testing.T) { - empty := Node{} - _, err := empty.SetAny("any", map[string]interface{}{"a": []int{0}}) - if err != nil { - t.Fatal(err) - } - if m, err := empty.Get("any").Interface(); err != nil { - t.Fatal(err) - } else if v, ok := m.(map[string]interface{}); !ok { - t.Fatal(v) - } - if buf, err := empty.MarshalJSON(); err != nil { - t.Fatal(err) - } else if string(buf) != `{"any":{"a":[0]}}` { - t.Fatal(string(buf)) - } - if _, err := empty.Set("any2", Node{}); err != nil { - t.Fatal(err) - } - if err := empty.Get("any2").AddAny(nil); err != nil { - t.Fatal(err) - } - if buf, err := empty.MarshalJSON(); err != nil { - t.Fatal(err) - } else if string(buf) != `{"any":{"a":[0]},"any2":[null]}` { - t.Fatal(string(buf)) - } - if _, err := empty.Get("any2").SetAnyByIndex(0, NewNumber("-0.0")); err != nil { - t.Fatal(err) - } - if buf, err := empty.MarshalJSON(); err != nil { - t.Fatal(err) - } else if string(buf) != `{"any":{"a":[0]},"any2":[-0.0]}` { - t.Fatal(string(buf)) - } -} - -func TestNodeSetByIndex(t *testing.T) { - root, derr := NewParser(_TwitterJson).Parse() - if derr != 0 { - t.Fatalf("decode failed: %v", derr.Error()) - } - app, _ := NewParser("111").Parse() - st := root.GetByPath("statuses") - st.SetByIndex(0, app) - st = root.GetByPath("statuses") - val := st.Index(0) - x, _ := val.Int64() - if x != 111 { - t.Fatalf("exp: %+v, got: %+v", 111, val) - } - - nroot, derr := NewParser(`{"a":[0.1,true,0,"name",{"b":"c"}]}`).Parse() - if derr != 0 { - t.Fatalf("decode failed: %v", derr.Error()) - } - root.GetByPath("statuses").SetByIndex(0, nroot) - val2, _ := root.GetByPath("statuses", 0, "a", 4, "b").String() - if val2 != "c" { - t.Fatalf("exp:%+v, got:%+v", "c", val2) - } -} - -func TestNodeAdd(t *testing.T) { - root, derr := NewParser(_TwitterJson).Parse() - if derr != 0 { - t.Fatalf("decode failed: %v", derr.Error()) - } - app, _ := NewParser("111").Parse() - - for i := root.GetByPath("statuses").cap(); i >= 0; i-- { - root.GetByPath("statuses").Add(app) - } - val, _ := root.GetByPath("statuses", 4).Int64() - if val != 111 { - t.Fatalf("exp: %+v, got: %+v", 111, val) - } - val, _ = root.GetByPath("statuses", root.GetByPath("statuses").len()-1).Int64() - if val != 111 { - t.Fatalf("exp: %+v, got: %+v", 111, val) - } - - nroot, derr := NewParser(`{"a":[0.1,true,0,"name",{"b":"c"}]}`).Parse() - if derr != 0 { - t.Fatalf("decode failed: %v", derr.Error()) - } - root.GetByPath("statuses").Add(nroot) - val2, _ := root.GetByPath("statuses", root.GetByPath("statuses").len()-1, "a", 4, "b").String() - if val2 != "c" { - t.Fatalf("exp:%+v, got:%+v", "c", val2) - } -} - -func BenchmarkLoadNode(b *testing.B) { - b.Run("Interface()", func(b *testing.B) { - b.SetBytes(int64(len(_TwitterJson))) - b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - root, err := NewSearcher(_TwitterJson).GetByPath("statuses", 0) - if err != nil { - b.Fatal(err) - } - _, _ = root.Interface() - } - }) - }) - - b.Run("LoadAll()", func(b *testing.B) { - b.SetBytes(int64(len(_TwitterJson))) - b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - root, err := NewSearcher(_TwitterJson).GetByPath("statuses", 0) - if err != nil { - b.Fatal(err) - } - _ = root.LoadAll() - } - }) - }) - - b.Run("InterfaceUseNode()", func(b *testing.B) { - b.SetBytes(int64(len(_TwitterJson))) - b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - root, err := NewSearcher(_TwitterJson).GetByPath("statuses", 0) - if err != nil { - b.Fatal(err) - } - _, _ = root.InterfaceUseNode() - } - }) - }) - - b.Run("Load()", func(b *testing.B) { - b.SetBytes(int64(len(_TwitterJson))) - b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - root, err := NewSearcher(_TwitterJson).GetByPath("statuses", 0) - if err != nil { - b.Fatal(err) - } - _ = root.Load() - } - }) - }) -} - -func BenchmarkNodeGetByPath(b *testing.B) { - root, derr := NewParser(_TwitterJson).Parse() - if derr != 0 { - b.Fatalf("decode failed: %v", derr.Error()) - } - _, _ = root.GetByPath("statuses", 3, "entities", "hashtags", 0, "text").String() - b.ResetTimer() - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - _, _ = root.GetByPath("statuses", 3, "entities", "hashtags", 0, "text").String() - } - }) -} - -func BenchmarkStructGetByPath(b *testing.B) { - var root = _TwitterStruct{} - err := json.Unmarshal([]byte(_TwitterJson), &root) - if err != nil { - b.Fatalf("unmarshal failed: %v", err) - } - - b.ResetTimer() - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - _ = root.Statuses[3].Entities.Hashtags[0].Text - } - }) -} - -func BenchmarkNodeIndex(b *testing.B) { - root, derr := NewParser(_TwitterJson).Parse() - if derr != 0 { - b.Fatalf("decode failed: %v", derr.Error()) - } - node := root.Get("statuses").Index(3).Get("entities").Get("hashtags").Index(0) - node.Set("test1", NewNumber("1")) - node.Set("test2", NewNumber("2")) - node.Set("test3", NewNumber("3")) - node.Set("test4", NewNumber("4")) - node.Set("test5", NewNumber("5")) - b.ResetTimer() - for i := 0; i < b.N; i++ { - node.Index(2) - } -} - -func BenchmarkStructIndex(b *testing.B) { - type T struct { - A Node - B Node - C Node - D Node - E Node - } - var obj = new(T) - b.ResetTimer() - for i := 0; i < b.N; i++ { - _ = obj.C - } -} - -func BenchmarkSliceIndex(b *testing.B) { - var obj = []Node{Node{}, Node{}, Node{}, Node{}, Node{}} - b.ResetTimer() - for i := 0; i < b.N; i++ { - _ = obj[2] - } -} - -func BenchmarkMapIndex(b *testing.B) { - var obj = map[string]interface{}{"test1": Node{}, "test2": Node{}, "test3": Node{}, "test4": Node{}, "test5": Node{}} - b.ResetTimer() - for i := 0; i < b.N; i++ { - for k := range obj { - if k == "test3" { - break - } - } - } -} - -func BenchmarkNodeGet(b *testing.B) { - var N = 5 - var half = "test" + strconv.Itoa(N/2+1) - root, derr := NewParser(_TwitterJson).Parse() - if derr != 0 { - b.Fatalf("decode failed: %v", derr.Error()) - } - node := root.Get("statuses").Index(3).Get("entities").Get("hashtags").Index(0) - for i := 0; i < N; i++ { - node.Set("test"+strconv.Itoa(i), NewNumber(strconv.Itoa(i))) - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - _ = node.Get(half) - } -} - -func BenchmarkSliceGet(b *testing.B) { - var obj = []string{"test1", "test2", "test3", "test4", "test5"} - str := "test3" - b.ResetTimer() - for i := 0; i < b.N; i++ { - for _, k := range obj { - if k == str { - break - } - } - } -} - -func BenchmarkMapGet(b *testing.B) { - root, derr := NewParser(_TwitterJson).Parse() - if derr != 0 { - b.Fatalf("decode failed: %v", derr.Error()) - } - node := root.Get("statuses").Index(3).Get("entities").Get("hashtags").Index(0) - node.Set("test1", NewNumber("1")) - node.Set("test2", NewNumber("2")) - node.Set("test3", NewNumber("3")) - node.Set("test4", NewNumber("4")) - node.Set("test5", NewNumber("5")) - m, _ := node.Map() - b.ResetTimer() - for i := 0; i < b.N; i++ { - _ = m["test3"] - } -} - -func BenchmarkNodeSet(b *testing.B) { - root, derr := NewParser(_TwitterJson).Parse() - if derr != 0 { - b.Fatalf("decode failed: %v", derr.Error()) - } - node := root.Get("statuses").Index(3).Get("entities").Get("hashtags").Index(0) - node.Set("test1", NewNumber("1")) - node.Set("test2", NewNumber("2")) - node.Set("test3", NewNumber("3")) - node.Set("test4", NewNumber("4")) - node.Set("test5", NewNumber("5")) - n := NewNull() - b.ResetTimer() - for i := 0; i < b.N; i++ { - node.Set("test3", n) - } -} - -func BenchmarkMapSet(b *testing.B) { - root, derr := NewParser(_TwitterJson).Parse() - if derr != 0 { - b.Fatalf("decode failed: %v", derr.Error()) - } - node := root.Get("statuses").Index(3).Get("entities").Get("hashtags").Index(0) - node.Set("test1", NewNumber("1")) - node.Set("test2", NewNumber("2")) - node.Set("test3", NewNumber("3")) - node.Set("test4", NewNumber("4")) - node.Set("test5", NewNumber("5")) - m, _ := node.Map() - n := NewNull() - b.ResetTimer() - for i := 0; i < b.N; i++ { - m["test3"] = n - } -} - -func BenchmarkNodeSetByIndex(b *testing.B) { - root, derr := NewParser(_TwitterJson).Parse() - if derr != 0 { - b.Fatalf("decode failed: %v", derr.Error()) - } - node := root.Get("statuses").Index(3).Get("entities").Get("hashtags") - node.Add(NewNumber("1")) - node.Add(NewNumber("2")) - node.Add(NewNumber("3")) - node.Add(NewNumber("4")) - node.Add(NewNumber("5")) - n := NewNull() - b.ResetTimer() - for i := 0; i < b.N; i++ { - node.SetByIndex(2, n) - } -} - -func BenchmarkSliceSetByIndex(b *testing.B) { - root, derr := NewParser(_TwitterJson).Parse() - if derr != 0 { - b.Fatalf("decode failed: %v", derr.Error()) - } - node := root.Get("statuses").Index(3).Get("entities").Get("hashtags") - node.Add(NewNumber("1")) - node.Add(NewNumber("2")) - node.Add(NewNumber("3")) - node.Add(NewNumber("4")) - node.Add(NewNumber("5")) - m, _ := node.Array() - n := NewNull() - b.ResetTimer() - for i := 0; i < b.N; i++ { - m[2] = n - } -} - -func BenchmarkStructSetByIndex(b *testing.B) { - type T struct { - A Node - B Node - C Node - D Node - E Node - } - var obj = new(T) - n := NewNull() - b.ResetTimer() - for i := 0; i < b.N; i++ { - obj.C = n - } -} - -func BenchmarkNodeUnset(b *testing.B) { - root, derr := NewParser(_TwitterJson).Parse() - if derr != 0 { - b.Fatalf("decode failed: %v", derr.Error()) - } - node := root.Get("statuses").Index(3).Get("entities").Get("hashtags").Index(0) - node.Set("test1", NewNumber("1")) - node.Set("test2", NewNumber("2")) - node.Set("test3", NewNumber("3")) - node.Set("test4", NewNumber("4")) - node.Set("test5", NewNumber("5")) - b.ResetTimer() - for i := 0; i < b.N; i++ { - node.Unset("test3") - } -} - -func BenchmarkMapUnset(b *testing.B) { - root, derr := NewParser(_TwitterJson).Parse() - if derr != 0 { - b.Fatalf("decode failed: %v", derr.Error()) - } - node := root.Get("statuses").Index(3).Get("entities").Get("hashtags").Index(0) - node.Set("test1", NewNumber("1")) - node.Set("test2", NewNumber("2")) - node.Set("test3", NewNumber("3")) - node.Set("test4", NewNumber("4")) - node.Set("test5", NewNumber("5")) - m, _ := node.Map() - b.ResetTimer() - for i := 0; i < b.N; i++ { - delete(m, "test3") - } -} - -func BenchmarkNodUnsetByIndex(b *testing.B) { - root, derr := NewParser(_TwitterJson).Parse() - if derr != 0 { - b.Fatalf("decode failed: %v", derr.Error()) - } - node := root.Get("statuses").Index(3).Get("entities").Get("hashtags") - node.Add(NewNumber("1")) - node.Add(NewNumber("2")) - node.Add(NewNumber("3")) - node.Add(NewNumber("4")) - node.Add(NewNumber("5")) - b.ResetTimer() - for i := 0; i < b.N; i++ { - node.UnsetByIndex(2) - } -} - -func BenchmarkSliceUnsetByIndex(b *testing.B) { - root, derr := NewParser(_TwitterJson).Parse() - if derr != 0 { - b.Fatalf("decode failed: %v", derr.Error()) - } - node := root.Get("statuses").Index(3).Get("entities").Get("hashtags") - node.Add(NewNumber("1")) - node.Add(NewNumber("2")) - node.Add(NewNumber("3")) - node.Add(NewNumber("4")) - node.Add(NewNumber("5")) - m, _ := node.Array() - b.ResetTimer() - for i := 0; i < b.N; i++ { - for i := 3; i < 5; i++ { - m[i-1] = m[i] - } - } -} - -func BenchmarkNodeAdd(b *testing.B) { - n := NewObject([]Pair{{"test", NewNumber("1")}}) - b.ResetTimer() - for i := 0; i < b.N; i++ { - node := NewArray([]Node{}) - node.Add(n) - } -} - -func BenchmarkSliceAdd(b *testing.B) { - n := NewObject([]Pair{{"test", NewNumber("1")}}) - b.ResetTimer() - for i := 0; i < b.N; i++ { - node := []Node{} - node = append(node, n) - } -} - -func BenchmarkMapAdd(b *testing.B) { - n := NewObject([]Pair{{"test", NewNumber("1")}}) - b.ResetTimer() - for i := 0; i < b.N; i++ { - node := map[string]Node{} - node["test3"] = n - } -} diff --git a/internal/json/parser.go b/internal/json/parser.go deleted file mode 100644 index fa6ce0cf..00000000 --- a/internal/json/parser.go +++ /dev/null @@ -1,632 +0,0 @@ -/** - * Copyright 2023 CloudWeGo Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package json - -import ( - "fmt" - - "github.com/cloudwego/dynamicgo/internal/native/types" - "github.com/cloudwego/dynamicgo/internal/rt" -) - -const _DEFAULT_NODE_CAP int = 16 - -const ( - _ERR_NOT_FOUND types.ParsingError = 33 - _ERR_UNSUPPORT_TYPE types.ParsingError = 34 -) - -var ( - ErrNotExist error = newError(_ERR_NOT_FOUND, "value not exists") - ErrUnsupportType error = newError(_ERR_UNSUPPORT_TYPE, "unsupported type") -) - -type Parser struct { - p int - s string - noLazy bool - skipValue bool -} - -/** Parser Private Methods **/ - -func (self *Parser) delim() types.ParsingError { - n := len(self.s) - p := self.lspace(self.p) - - /* check for EOF */ - if p >= n { - return types.ERR_EOF - } - - /* check for the delimtier */ - if self.s[p] != ':' { - return types.ERR_INVALID_CHAR - } - - /* update the read pointer */ - self.p = p + 1 - return 0 -} - -func (self *Parser) object() types.ParsingError { - n := len(self.s) - p := self.lspace(self.p) - - /* check for EOF */ - if p >= n { - return types.ERR_EOF - } - - /* check for the delimtier */ - if self.s[p] != '{' { - return types.ERR_INVALID_CHAR - } - - /* update the read pointer */ - self.p = p + 1 - return 0 -} - -func (self *Parser) array() types.ParsingError { - n := len(self.s) - p := self.lspace(self.p) - - /* check for EOF */ - if p >= n { - return types.ERR_EOF - } - - /* check for the delimtier */ - if self.s[p] != '[' { - return types.ERR_INVALID_CHAR - } - - /* update the read pointer */ - self.p = p + 1 - return 0 -} - -func (self *Parser) lspace(sp int) int { - ns := len(self.s) - for ; sp < ns && IsSpace(self.s[sp]); sp += 1 { - } - - return sp -} - -func (self *Parser) decodeArray(ret []Node) (Node, types.ParsingError) { - sp := self.p - ns := len(self.s) - - /* check for EOF */ - if self.p = self.lspace(sp); self.p >= ns { - return Node{}, types.ERR_EOF - } - - /* check for empty array */ - if self.s[self.p] == ']' { - self.p++ - return emptyArrayNode, 0 - } - - /* allocate array space and parse every element */ - for { - var val Node - var err types.ParsingError - - if self.skipValue { - /* skip the value */ - var start int - if start, err = self.skip(); err != 0 { - return Node{}, err - } - if self.p > ns { - return Node{}, types.ERR_EOF - } - t := switchRawType(self.s[start]) - if t == _V_NONE { - return Node{}, types.ERR_INVALID_CHAR - } - val = newRawNode(self.s[start:self.p], t) - } else { - /* decode the value */ - if val, err = self.Parse(); err != 0 { - return Node{}, err - } - } - - /* add the value to result */ - ret = append(ret, val) - self.p = self.lspace(self.p) - - /* check for EOF */ - if self.p >= ns { - return Node{}, types.ERR_EOF - } - - /* check for the next character */ - switch self.s[self.p] { - case ',': - self.p++ - case ']': - self.p++ - return NewArray(ret), 0 - default: - if val.isLazy() { - return newLazyArray(self, ret), 0 - } - return Node{}, types.ERR_INVALID_CHAR - } - } -} - -func (self *Parser) decodeObject(ret []Pair) (Node, types.ParsingError) { - sp := self.p - ns := len(self.s) - - /* check for EOF */ - if self.p = self.lspace(sp); self.p >= ns { - return Node{}, types.ERR_EOF - } - - /* check for empty object */ - if self.s[self.p] == '}' { - self.p++ - return emptyObjectNode, 0 - } - - /* decode each pair */ - for { - var val Node - var njs types.JsonState - var err types.ParsingError - - /* decode the key */ - if njs = self.decodeValue(); njs.Vt != types.V_STRING { - return Node{}, types.ERR_INVALID_CHAR - } - - /* extract the key */ - idx := self.p - 1 - key := self.s[njs.Iv:idx] - - /* check for escape sequence */ - if njs.Ep != -1 { - if key, err = unquote(key); err != 0 { - return Node{}, err - } - } - - /* expect a ':' delimiter */ - if err = self.delim(); err != 0 { - return Node{}, err - } - - if self.skipValue { - /* skip the value */ - var start int - if start, err = self.skip(); err != 0 { - return Node{}, err - } - if self.p > ns { - return Node{}, types.ERR_EOF - } - t := switchRawType(self.s[start]) - if t == _V_NONE { - return Node{}, types.ERR_INVALID_CHAR - } - val = newRawNode(self.s[start:self.p], t) - } else { - /* decode the value */ - if val, err = self.Parse(); err != 0 { - return Node{}, err - } - } - - /* add the value to result */ - ret = append(ret, Pair{Key: key, Value: val}) - self.p = self.lspace(self.p) - - /* check for EOF */ - if self.p >= ns { - return Node{}, types.ERR_EOF - } - - /* check for the next character */ - switch self.s[self.p] { - case ',': - self.p++ - case '}': - self.p++ - return NewObject(ret), 0 - default: - if val.isLazy() { - return newLazyObject(self, ret), 0 - } - return Node{}, types.ERR_INVALID_CHAR - } - } -} - -func (self *Parser) decodeString(iv int64, ep int) (Node, types.ParsingError) { - p := self.p - 1 - s := self.s[iv:p] - - /* fast path: no escape sequence */ - if ep == -1 { - return NewString(s), 0 - } - - /* unquote the string */ - out, err := unquote(s) - - /* check for errors */ - if err != 0 { - return Node{}, err - } else { - return newBytes(rt.Str2Mem(out)), 0 - } -} - -/** Parser Interface **/ - -func (self *Parser) Pos() int { - return self.p -} - -func (self *Parser) Parse() (Node, types.ParsingError) { - switch val := self.decodeValue(); val.Vt { - case types.V_EOF: - return Node{}, types.ERR_EOF - case types.V_NULL: - return nullNode, 0 - case types.V_TRUE: - return trueNode, 0 - case types.V_FALSE: - return falseNode, 0 - case types.V_STRING: - return self.decodeString(val.Iv, int(val.Ep)) - case types.V_ARRAY: - if self.noLazy { - return self.decodeArray(make([]Node, 0, _DEFAULT_NODE_CAP)) - } - return newLazyArray(self, make([]Node, 0, _DEFAULT_NODE_CAP)), 0 - case types.V_OBJECT: - if self.noLazy { - return self.decodeObject(make([]Pair, 0, _DEFAULT_NODE_CAP)) - } - return newLazyObject(self, make([]Pair, 0, _DEFAULT_NODE_CAP)), 0 - case types.V_DOUBLE: - return NewNumber(self.s[val.Ep:self.p]), 0 - case types.V_INTEGER: - return NewNumber(self.s[val.Ep:self.p]), 0 - default: - return Node{}, types.ParsingError(-val.Vt) - } -} - -func (self *Parser) searchKey(match string) types.ParsingError { - ns := len(self.s) - if err := self.object(); err != 0 { - return err - } - - /* check for EOF */ - if self.p = self.lspace(self.p); self.p >= ns { - return types.ERR_EOF - } - - /* check for empty object */ - if self.s[self.p] == '}' { - self.p++ - return _ERR_NOT_FOUND - } - - var njs types.JsonState - var err types.ParsingError - /* decode each pair */ - for { - - /* decode the key */ - if njs = self.decodeValue(); njs.Vt != types.V_STRING { - return types.ERR_INVALID_CHAR - } - - /* extract the key */ - idx := self.p - 1 - key := self.s[njs.Iv:idx] - - /* check for escape sequence */ - if njs.Ep != -1 { - if key, err = unquote(key); err != 0 { - return err - } - } - - /* expect a ':' delimiter */ - if err = self.delim(); err != 0 { - return err - } - - /* skip value */ - if key != match { - if _, err = self.skip(); err != 0 { - return err - } - } else { - return 0 - } - - /* check for EOF */ - self.p = self.lspace(self.p) - if self.p >= ns { - return types.ERR_EOF - } - - /* check for the next character */ - switch self.s[self.p] { - case ',': - self.p++ - case '}': - self.p++ - return _ERR_NOT_FOUND - default: - return types.ERR_INVALID_CHAR - } - } -} - -func (self *Parser) searchIndex(idx int) types.ParsingError { - ns := len(self.s) - if err := self.array(); err != 0 { - return err - } - - /* check for EOF */ - if self.p = self.lspace(self.p); self.p >= ns { - return types.ERR_EOF - } - - /* check for empty array */ - if self.s[self.p] == ']' { - self.p++ - return _ERR_NOT_FOUND - } - - var err types.ParsingError - /* allocate array space and parse every element */ - for i := 0; i < idx; i++ { - - /* decode the value */ - if _, err = self.skip(); err != 0 { - return err - } - - /* check for EOF */ - self.p = self.lspace(self.p) - if self.p >= ns { - return types.ERR_EOF - } - - /* check for the next character */ - switch self.s[self.p] { - case ',': - self.p++ - case ']': - self.p++ - return _ERR_NOT_FOUND - default: - return types.ERR_INVALID_CHAR - } - } - - return 0 -} - -func (self *Node) skipNextNode() *Node { - if !self.isLazy() { - return nil - } - - parser, stack := self.getParserAndArrayStack() - ret := stack.v - sp := parser.p - ns := len(parser.s) - - /* check for EOF */ - if parser.p = parser.lspace(sp); parser.p >= ns { - return newSyntaxError(parser.syntaxError(types.ERR_EOF)) - } - - /* check for empty array */ - if parser.s[parser.p] == ']' { - parser.p++ - self.setArray(ret) - return nil - } - - var val Node - /* skip the value */ - if start, err := parser.skip(); err != 0 { - return newSyntaxError(parser.syntaxError(err)) - } else { - t := switchRawType(parser.s[start]) - if t == _V_NONE { - return newSyntaxError(parser.syntaxError(types.ERR_INVALID_CHAR)) - } - val = newRawNode(parser.s[start:parser.p], t) - } - - /* add the value to result */ - ret = append(ret, val) - parser.p = parser.lspace(parser.p) - - /* check for EOF */ - if parser.p >= ns { - return newSyntaxError(parser.syntaxError(types.ERR_EOF)) - } - - /* check for the next character */ - switch parser.s[parser.p] { - case ',': - parser.p++ - self.setLazyArray(parser, ret) - return &ret[len(ret)-1] - case ']': - parser.p++ - self.setArray(ret) - return &ret[len(ret)-1] - default: - return newSyntaxError(parser.syntaxError(types.ERR_INVALID_CHAR)) - } -} - -func (self *Node) skipNextPair() *Pair { - if !self.isLazy() { - return nil - } - - parser, stack := self.getParserAndObjectStack() - ret := stack.v - sp := parser.p - ns := len(parser.s) - - /* check for EOF */ - if parser.p = parser.lspace(sp); parser.p >= ns { - return &Pair{"", *newSyntaxError(parser.syntaxError(types.ERR_EOF))} - } - - /* check for empty object */ - if parser.s[parser.p] == '}' { - parser.p++ - self.setObject(ret) - return nil - } - - /* decode one pair */ - var val Node - var njs types.JsonState - var err types.ParsingError - - /* decode the key */ - if njs = parser.decodeValue(); njs.Vt != types.V_STRING { - return &Pair{"", *newSyntaxError(parser.syntaxError(types.ERR_INVALID_CHAR))} - } - - /* extract the key */ - idx := parser.p - 1 - key := parser.s[njs.Iv:idx] - - /* check for escape sequence */ - if njs.Ep != -1 { - if key, err = unquote(key); err != 0 { - return &Pair{key, *newSyntaxError(parser.syntaxError(err))} - } - } - - /* expect a ':' delimiter */ - if err = parser.delim(); err != 0 { - return &Pair{key, *newSyntaxError(parser.syntaxError(err))} - } - - /* skip the value */ - if start, err := parser.skip(); err != 0 { - return &Pair{key, *newSyntaxError(parser.syntaxError(err))} - } else { - t := switchRawType(parser.s[start]) - if t == _V_NONE { - return &Pair{key, *newSyntaxError(parser.syntaxError(types.ERR_INVALID_CHAR))} - } - val = newRawNode(parser.s[start:parser.p], t) - } - - /* add the value to result */ - ret = append(ret, Pair{Key: key, Value: val}) - parser.p = parser.lspace(parser.p) - - /* check for EOF */ - if parser.p >= ns { - return &Pair{key, *newSyntaxError(parser.syntaxError(types.ERR_EOF))} - } - - /* check for the next character */ - switch parser.s[parser.p] { - case ',': - parser.p++ - self.setLazyObject(parser, ret) - return &ret[len(ret)-1] - case '}': - parser.p++ - self.setObject(ret) - return &ret[len(ret)-1] - default: - return &Pair{key, *newSyntaxError(parser.syntaxError(types.ERR_INVALID_CHAR))} - } -} - -/** Parser Factory **/ - -// Loads parse all json into interface{} -func Loads(src string) (int, interface{}, error) { - ps := &Parser{s: src} - np, err := ps.Parse() - - /* check for errors */ - if err != 0 { - return 0, nil, ps.ExportError(err) - } else { - x, err := np.Interface() - if err != nil { - return 0, nil, err - } - return ps.Pos(), x, nil - } -} - -// LoadsUseNumber parse all json into interface{}, with numeric nodes casted to json.Number -func LoadsUseNumber(src string) (int, interface{}, error) { - ps := &Parser{s: src} - np, err := ps.Parse() - - /* check for errors */ - if err != 0 { - return 0, nil, err - } else { - x, err := np.InterfaceUseNumber() - if err != nil { - return 0, nil, err - } - return ps.Pos(), x, nil - } -} - -func NewParser(src string) *Parser { - return &Parser{s: src} -} - -// ExportError converts types.ParsingError to std Error -func (self *Parser) ExportError(err types.ParsingError) error { - if err == _ERR_NOT_FOUND { - return ErrNotExist - } - return fmt.Errorf("%q", SyntaxError{ - Pos: self.p, - Src: self.s, - Code: err, - }.Description()) -} diff --git a/internal/json/parser_test.go b/internal/json/parser_test.go deleted file mode 100644 index 6d73e533..00000000 --- a/internal/json/parser_test.go +++ /dev/null @@ -1,343 +0,0 @@ -/** - * Copyright 2023 CloudWeGo Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package json - -import ( - "encoding/json" - "os" - "runtime" - "runtime/debug" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -var ( - debugSyncGC = os.Getenv("SONIC_SYNC_GC") != "" - debugAsyncGC = os.Getenv("SONIC_NO_ASYNC_GC") == "" -) - -func TestMain(m *testing.M) { - go func() { - if !debugAsyncGC { - return - } - println("Begin GC looping...") - for { - runtime.GC() - debug.FreeOSMemory() - } - println("stop GC looping!") - }() - time.Sleep(time.Millisecond) - m.Run() -} - -func TestGC_Parse(t *testing.T) { - if debugSyncGC { - return - } - _, _, err := Loads(_TwitterJson) - if err != nil { - t.Fatal(err) - } - wg := &sync.WaitGroup{} - N := 1000 - for i := 0; i < N; i++ { - wg.Add(1) - go func(wg *sync.WaitGroup) { - defer wg.Done() - _, _, err := Loads(_TwitterJson) - if err != nil { - t.Fatal(err) - } - runtime.GC() - }(wg) - } - wg.Wait() -} - -func runDecoderTest(t *testing.T, src string, expect interface{}) { - vv, err := NewParser(src).Parse() - if err != 0 { - panic(err) - } - x, _ := vv.Interface() - assert.Equal(t, expect, x) -} - -func runDecoderTestUseNumber(t *testing.T, src string, expect interface{}) { - vv, err := NewParser(src).Parse() - if err != 0 { - panic(err) - } - vvv, _ := vv.InterfaceUseNumber() - switch vvv.(type) { - case json.Number: - assert.Equal(t, expect, n2f64(vvv.(json.Number))) - case []interface{}: - x := vvv.([]interface{}) - for i, e := range x { - if ev, ok := e.(json.Number); ok { - x[i] = n2f64(ev) - } - } - assert.Equal(t, expect, x) - case map[string]interface{}: - x := vvv.(map[string]interface{}) - for k, v := range x { - if ev, ok := v.(json.Number); ok { - x[k] = n2f64(ev) - } - } - assert.Equal(t, expect, x) - } -} - -func n2f64(i json.Number) float64 { - x, err := i.Float64() - if err != nil { - panic(err) - } - return x -} - -func TestParser_Basic(t *testing.T) { - runDecoderTest(t, `null`, nil) - runDecoderTest(t, `true`, true) - runDecoderTest(t, `false`, false) - runDecoderTest(t, `"hello, world \\ \/ \b \f \n \r \t \u666f ζ΅‹θ―•δΈ­ζ–‡"`, "hello, world \\ / \b \f \n \r \t \u666f ζ΅‹θ―•δΈ­ζ–‡") - runDecoderTest(t, `"\ud83d\ude00"`, "πŸ˜€") - runDecoderTest(t, `0`, float64(0)) - runDecoderTest(t, `-0`, float64(0)) - runDecoderTest(t, `123456`, float64(123456)) - runDecoderTest(t, `-12345`, float64(-12345)) - runDecoderTest(t, `0.2`, 0.2) - runDecoderTest(t, `1.2`, 1.2) - runDecoderTest(t, `-0.2`, -0.2) - runDecoderTest(t, `-1.2`, -1.2) - runDecoderTest(t, `0e12`, 0e12) - runDecoderTest(t, `0e+12`, 0e+12) - runDecoderTest(t, `0e-12`, 0e-12) - runDecoderTest(t, `-0e12`, -0e12) - runDecoderTest(t, `-0e+12`, -0e+12) - runDecoderTest(t, `-0e-12`, -0e-12) - runDecoderTest(t, `2e12`, 2e12) - runDecoderTest(t, `2E12`, 2e12) - runDecoderTest(t, `2e+12`, 2e+12) - runDecoderTest(t, `2e-12`, 2e-12) - runDecoderTest(t, `-2e12`, -2e12) - runDecoderTest(t, `-2e+12`, -2e+12) - runDecoderTest(t, `-2e-12`, -2e-12) - runDecoderTest(t, `0.2e12`, 0.2e12) - runDecoderTest(t, `0.2e+12`, 0.2e+12) - runDecoderTest(t, `0.2e-12`, 0.2e-12) - runDecoderTest(t, `-0.2e12`, -0.2e12) - runDecoderTest(t, `-0.2e+12`, -0.2e+12) - runDecoderTest(t, `-0.2e-12`, -0.2e-12) - runDecoderTest(t, `1.2e12`, 1.2e12) - runDecoderTest(t, `1.2e+12`, 1.2e+12) - runDecoderTest(t, `1.2e-12`, 1.2e-12) - runDecoderTest(t, `-1.2e12`, -1.2e12) - runDecoderTest(t, `-1.2e+12`, -1.2e+12) - runDecoderTest(t, `-1.2e-12`, -1.2e-12) - runDecoderTest(t, `-1.2E-12`, -1.2e-12) - runDecoderTest(t, `[]`, []interface{}{}) - runDecoderTest(t, `{}`, map[string]interface{}{}) - runDecoderTest(t, `["asd", "123", true, false, null, 2.4, 1.2e15]`, []interface{}{"asd", "123", true, false, nil, 2.4, 1.2e15}) - runDecoderTest(t, `{"asdf": "qwer", "zxcv": true}`, map[string]interface{}{"asdf": "qwer", "zxcv": true}) - runDecoderTest(t, `{"a": "123", "b": true, "c": false, "d": null, "e": 2.4, "f": 1.2e15, "g": 1}`, map[string]interface{}{"a": "123", "b": true, "c": false, "d": nil, "e": 2.4, "f": 1.2e15, "g": float64(1)}) - - runDecoderTestUseNumber(t, `null`, nil) - runDecoderTestUseNumber(t, `true`, true) - runDecoderTestUseNumber(t, `false`, false) - runDecoderTestUseNumber(t, `"hello, world \\ \/ \b \f \n \r \t \u666f ζ΅‹θ―•δΈ­ζ–‡"`, "hello, world \\ / \b \f \n \r \t \u666f ζ΅‹θ―•δΈ­ζ–‡") - runDecoderTestUseNumber(t, `"\ud83d\ude00"`, "πŸ˜€") - runDecoderTestUseNumber(t, `0`, float64(0)) - runDecoderTestUseNumber(t, `-0`, float64(0)) - runDecoderTestUseNumber(t, `123456`, float64(123456)) - runDecoderTestUseNumber(t, `-12345`, float64(-12345)) - runDecoderTestUseNumber(t, `0.2`, 0.2) - runDecoderTestUseNumber(t, `1.2`, 1.2) - runDecoderTestUseNumber(t, `-0.2`, -0.2) - runDecoderTestUseNumber(t, `-1.2`, -1.2) - runDecoderTestUseNumber(t, `0e12`, 0e12) - runDecoderTestUseNumber(t, `0e+12`, 0e+12) - runDecoderTestUseNumber(t, `0e-12`, 0e-12) - runDecoderTestUseNumber(t, `-0e12`, -0e12) - runDecoderTestUseNumber(t, `-0e+12`, -0e+12) - runDecoderTestUseNumber(t, `-0e-12`, -0e-12) - runDecoderTestUseNumber(t, `2e12`, 2e12) - runDecoderTestUseNumber(t, `2E12`, 2e12) - runDecoderTestUseNumber(t, `2e+12`, 2e+12) - runDecoderTestUseNumber(t, `2e-12`, 2e-12) - runDecoderTestUseNumber(t, `-2e12`, -2e12) - runDecoderTestUseNumber(t, `-2e+12`, -2e+12) - runDecoderTestUseNumber(t, `-2e-12`, -2e-12) - runDecoderTestUseNumber(t, `0.2e12`, 0.2e12) - runDecoderTestUseNumber(t, `0.2e+12`, 0.2e+12) - runDecoderTestUseNumber(t, `0.2e-12`, 0.2e-12) - runDecoderTestUseNumber(t, `-0.2e12`, -0.2e12) - runDecoderTestUseNumber(t, `-0.2e+12`, -0.2e+12) - runDecoderTestUseNumber(t, `-0.2e-12`, -0.2e-12) - runDecoderTestUseNumber(t, `1.2e12`, 1.2e12) - runDecoderTestUseNumber(t, `1.2e+12`, 1.2e+12) - runDecoderTestUseNumber(t, `1.2e-12`, 1.2e-12) - runDecoderTestUseNumber(t, `-1.2e12`, -1.2e12) - runDecoderTestUseNumber(t, `-1.2e+12`, -1.2e+12) - runDecoderTestUseNumber(t, `-1.2e-12`, -1.2e-12) - runDecoderTestUseNumber(t, `-1.2E-12`, -1.2e-12) - runDecoderTestUseNumber(t, `["asd", "123", true, false, null, 2.4, 1.2e15, 1]`, []interface{}{"asd", "123", true, false, nil, 2.4, 1.2e15, float64(1)}) - runDecoderTestUseNumber(t, `{"a": "123", "b": true, "c": false, "d": null, "e": 2.4, "f": 1.2e15, "g": 1}`, map[string]interface{}{"a": "123", "b": true, "c": false, "d": nil, "e": 2.4, "f": 1.2e15, "g": float64(1)}) -} - -func TestLoads(t *testing.T) { - _, i, e := Loads(`{"a": "123", "b": true, "c": false, "d": null, "e": 2.4, "f": 1.2e15, "g": 1}`) - if e != nil { - t.Fatal(e) - } - assert.Equal(t, map[string]interface{}{"a": "123", "b": true, "c": false, "d": nil, "e": 2.4, "f": 1.2e15, "g": float64(1)}, i) - _, i, e = LoadsUseNumber(`{"a": "123", "b": true, "c": false, "d": null, "e": 2.4, "f": 1.2e15, "g": 1}`) - if e != nil { - t.Fatal(e) - } - assert.Equal(t, map[string]interface{}{"a": "123", "b": true, "c": false, "d": nil, "e": json.Number("2.4"), "f": json.Number("1.2e15"), "g": json.Number("1")}, i) -} - -func TestParsehNotExist(t *testing.T) { - s, err := NewParser(` { "xx" : [ 0, "" ] ,"yy" :{ "2": "" } } `).Parse() - if err != 0 { - t.Fatal(err) - } - node := s.GetByPath("xx", 2) - if node.Exists() { - t.Fatalf("node: %v", node) - } - node = s.GetByPath("xx", 1) - if !node.Exists() { - t.Fatalf("node: %v", nil) - } - node = s.GetByPath("yy", "3") - if node.Exists() { - t.Fatalf("node: %v", node) - } - node = s.GetByPath("yy", "2") - if !node.Exists() { - t.Fatalf("node: %v", nil) - } -} - -func BenchmarkParser_Sonic(b *testing.B) { - r, err := NewParser(_TwitterJson).Parse() - if err != 0 { - b.Fatal(err) - } - if err := r.LoadAll(); err != nil { - b.Fatal(err) - } - b.SetBytes(int64(len(_TwitterJson))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - r, _ = NewParser(_TwitterJson).Parse() - _ = r.LoadAll() - } -} - -func BenchmarkParser_Parallel_Sonic(b *testing.B) { - r, _ := NewParser(_TwitterJson).Parse() - if err := r.LoadAll(); err != nil { - b.Fatal(err) - } - b.SetBytes(int64(len(_TwitterJson))) - b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - r, _ := NewParser(_TwitterJson).Parse() - _ = r.LoadAll() - } - }) -} - -func BenchmarkParseOne_Sonic(b *testing.B) { - ast, _ := NewParser(_TwitterJson).Parse() - node, _ := ast.Get("statuses").Index(2).Get("id").Int64() - if node != 249289491129438208 { - b.Fail() - } - b.SetBytes(int64(len(_TwitterJson))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - ast, _ := NewParser(_TwitterJson).Parse() - _, _ = ast.Get("statuses").Index(2).Get("id").Int64() - } -} - -func BenchmarkParseOne_Parallel_Sonic(b *testing.B) { - ast, _ := NewParser(_TwitterJson).Parse() - node, _ := ast.Get("statuses").Index(2).Get("id").Int64() - if node != 249289491129438208 { - b.Fail() - } - b.SetBytes(int64(len(_TwitterJson))) - b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - ast, _ := NewParser(_TwitterJson).Parse() - _, _ = ast.Get("statuses").Index(2).Get("id").Int64() - } - }) -} - -func BenchmarkParseSeven_Sonic(b *testing.B) { - b.SetBytes(int64(len(_TwitterJson))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - ast, _ := NewParser(_TwitterJson).Parse() - node := ast.GetByPath("statuses", 3, "id") - node = ast.GetByPath("statuses", 3, "user", "entities", "description") - node = ast.GetByPath("statuses", 3, "user", "entities", "url", "urls") - node = ast.GetByPath("statuses", 3, "user", "entities", "url") - node = ast.GetByPath("statuses", 3, "user", "created_at") - node = ast.GetByPath("statuses", 3, "user", "name") - node = ast.GetByPath("statuses", 3, "text") - if node.Check() != nil { - b.Fail() - } - } -} - -func BenchmarkParseSeven_Parallel_Sonic(b *testing.B) { - b.SetBytes(int64(len(_TwitterJson))) - b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - ast, _ := NewParser(_TwitterJson).Parse() - node := ast.GetByPath("statuses", 3, "id") - node = ast.GetByPath("statuses", 3, "user", "entities", "description") - node = ast.GetByPath("statuses", 3, "user", "entities", "url", "urls") - node = ast.GetByPath("statuses", 3, "user", "entities", "url") - node = ast.GetByPath("statuses", 3, "user", "created_at") - node = ast.GetByPath("statuses", 3, "user", "name") - node = ast.GetByPath("statuses", 3, "text") - if node.Check() != nil { - b.Fail() - } - } - }) -} diff --git a/internal/json/search.go b/internal/json/search.go deleted file mode 100644 index c9a37bcf..00000000 --- a/internal/json/search.go +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Copyright 2023 CloudWeGo Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package json - -import ( - "fmt" - - "github.com/cloudwego/dynamicgo/internal/native/types" -) - -type Searcher struct { - parser Parser -} - -func NewSearcher(str string) *Searcher { - return &Searcher{ - parser: Parser{ - s: str, - noLazy: false, - }, - } -} - -func (self *Searcher) GetByPath(path ...interface{}) (Node, error) { - self.parser.p = 0 - - var err types.ParsingError - for _, p := range path { - switch p := p.(type) { - case int: - if err = self.parser.searchIndex(p); err != 0 { - return Node{}, self.parser.ExportError(err) - } - case string: - if err = self.parser.searchKey(p); err != 0 { - return Node{}, self.parser.ExportError(err) - } - default: - panic("path must be either int or string") - } - } - - var start = self.parser.p - if start, err = self.parser.skip(); err != 0 { - return Node{}, self.parser.ExportError(err) - } - ns := len(self.parser.s) - if self.parser.p > ns || start >= ns || start >= self.parser.p { - return Node{}, fmt.Errorf("skip %d char out of json boundary", start) - } - - t := switchRawType(self.parser.s[start]) - if t == _V_NONE { - return Node{}, self.parser.ExportError(err) - } - - return newRawNode(self.parser.s[start:self.parser.p], t), nil -} diff --git a/internal/json/search_test.go b/internal/json/search_test.go deleted file mode 100644 index 88bd72b7..00000000 --- a/internal/json/search_test.go +++ /dev/null @@ -1,261 +0,0 @@ -/** - * Copyright 2023 CloudWeGo Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package json - -import ( - "math" - "runtime" - "strconv" - "strings" - "sync" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestGC_Search(t *testing.T) { - if debugSyncGC { - return - } - _, err := NewSearcher(_TwitterJson).GetByPath("statuses", 0, "id") - if err != nil { - t.Fatal(err) - } - wg := &sync.WaitGroup{} - // A limitation of the race detecting is 8128. - // See https://github.com/golang/go/issues/43898 - N := 5000 - for i := 0; i < N; i++ { - wg.Add(1) - go func(wg *sync.WaitGroup) { - defer wg.Done() - _, err := NewSearcher(_TwitterJson).GetByPath("statuses", 0, "id") - if err != nil { - t.Fatal(err) - } - runtime.GC() - }(wg) - } - wg.Wait() -} - -func TestExportError(t *testing.T) { - data := `{"a":]` - p := NewSearcher(data) - _, err := p.GetByPath("a") - if err == nil { - t.Fatal() - } - if strings.Index(err.Error(), `"Syntax error at `) != 0 { - t.Fatal(err) - } - - data = `:"b"]` - p = NewSearcher(data) - _, err = p.GetByPath("a") - if err == nil { - t.Fatal() - } - if err.Error() != `"Syntax error at index 0: invalid char\n\n\t:\"b\"]\n\t^....\n"` { - t.Fatal(err) - } - - data = `{:"b"]` - p = NewSearcher(data) - _, err = p.GetByPath("a") - if err == nil { - t.Fatal() - } - if err.Error() != `"Syntax error at index 1: invalid char\n\n\t{:\"b\"]\n\t.^....\n"` { - t.Fatal(err) - } -} - -func TestSearcher_GetByPath(t *testing.T) { - s := NewSearcher(` { "xx" : [] ,"yy" :{ }, "test" : [ true , 0.1 , "abc", ["h"], {"a":"bc"} ] } `) - - node, e := s.GetByPath("test", 0) - a, _ := node.Bool() - if e != nil || a != true { - t.Fatalf("node: %v, err: %v", node, e) - } - - node, e = s.GetByPath("test", 1) - b, _ := node.Float64() - if e != nil || b != 0.1 { - t.Fatalf("node: %v, err: %v", node, e) - } - - node, e = s.GetByPath("test", 2) - c, _ := node.String() - if e != nil || c != "abc" { - t.Fatalf("node: %v, err: %v", node, e) - } - - node, e = s.GetByPath("test", 3) - arr, _ := node.Array() - if e != nil || arr[0] != "h" { - t.Fatalf("node: %v, err: %v", node, e) - } - - node, e = s.GetByPath("test", 4, "a") - d, _ := node.String() - if e != nil || d != "bc" { - t.Fatalf("node: %v, err: %v", node, e) - } -} - -func TestSearcher_GetByPathErr(t *testing.T) { - s := NewSearcher(` { "xx" : [] ,"yy" :{ }, "test" : [ true , 0.1 , "abc", ["h"], {"a":"bc"} ], "err1":[a, ] , "err2":{ ,"x":"xx"} } `) - node, e := s.GetByPath("zz") - if e == nil { - t.Fatalf("node: %v, err: %v", node, e) - } - s.parser.p = 0 - node, e = s.GetByPath("xx", 4) - if e == nil { - t.Fatalf("node: %v, err: %v", node, e) - } - s.parser.p = 0 - node, e = s.GetByPath("yy", "a") - if e == nil { - t.Fatalf("node: %v, err: %v", node, e) - } - s.parser.p = 0 - node, e = s.GetByPath("test", 2, "x") - if e == nil { - t.Fatalf("node: %v, err: %v", node, e) - } - s.parser.p = 0 - node, e = s.GetByPath("err1", 0) - if e == nil { - t.Fatalf("node: %v, err: %v", node, e) - } - s.parser.p = 0 - node, e = s.GetByPath("err2", "x") - if e == nil { - t.Fatalf("node: %v, err: %v", node, e) - } -} - -func TestLoadIndex(t *testing.T) { - node, err := NewSearcher(`{"a":[-0, 1, -1.2, -1.2e-10]}`).GetByPath("a") - if err != nil { - t.Fatal(err) - } - a, _ := node.Index(3).Float64() - assert.Equal(t, -1.2e-10, a) - m, _ := node.Array() - assert.Equal(t, m, []interface{}{ - float64(0), - float64(1), - -1.2, - -1.2e-10, - }) -} - -func TestSearchNotExist(t *testing.T) { - s := NewSearcher(` { "xx" : [ 0, "" ] ,"yy" :{ "2": "" } } `) - node, e := s.GetByPath("xx", 2) - if node.Exists() { - t.Fatalf("node: %v, err: %v", node, e) - } - node, e = s.GetByPath("xx", 1) - if e != nil || !node.Exists() { - t.Fatalf("node: %v, err: %v", node, e) - } - node, e = s.GetByPath("yy", "3") - if node.Exists() { - t.Fatalf("node: %v, err: %v", node, e) - } - node, e = s.GetByPath("yy", "2") - if e != nil || !node.Exists() { - t.Fatalf("node: %v, err: %v", node, e) - } -} - -func BenchmarkGetOne_Sonic(b *testing.B) { - b.SetBytes(int64(len(_TwitterJson))) - ast := NewSearcher(_TwitterJson) - for i := 0; i < b.N; i++ { - node, err := ast.GetByPath("statuses", 3, "id") - if err != nil { - b.Fatal(err) - } - x, _ := node.Int64() - if x != 249279667666817024 { - b.Fatal(node.Interface()) - } - } -} - -func BenchmarkGetOne_Parallel_Sonic(b *testing.B) { - b.SetBytes(int64(len(_TwitterJson))) - b.RunParallel(func(pb *testing.PB) { - ast := NewSearcher(_TwitterJson) - for pb.Next() { - node, err := ast.GetByPath("statuses", 3, "id") - if err != nil { - b.Fatal(err) - } - x, _ := node.Int64() - if x != 249279667666817024 { - b.Fatal(node.Interface()) - } - } - }) -} - -func BenchmarkSetOne_Sonic(b *testing.B) { - node, err := NewSearcher(_TwitterJson).GetByPath("statuses", 3) - if err != nil { - b.Fatal(err) - } - n := NewNumber(strconv.Itoa(math.MaxInt32)) - _, err = node.Set("id", n) - if err != nil { - b.Fatal(err) - } - b.SetBytes(int64(len(_TwitterJson))) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - node, _ := NewSearcher(_TwitterJson).GetByPath("statuses", 3) - _, _ = node.Set("id", n) - } -} - -func BenchmarkSetOne_Parallel_Sonic(b *testing.B) { - node, err := NewSearcher(_TwitterJson).GetByPath("statuses", 3) - if err != nil { - b.Fatal(err) - } - n := NewNumber(strconv.Itoa(math.MaxInt32)) - _, err = node.Set("id", n) - if err != nil { - b.Fatal(err) - } - b.SetBytes(int64(len(_TwitterJson))) - b.ReportAllocs() - b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - node, _ := NewSearcher(_TwitterJson).GetByPath("statuses", 3) - _, _ = node.Set("id", n) - } - }) -} diff --git a/internal/json/sort.go b/internal/json/sort.go deleted file mode 100644 index 499f7c81..00000000 --- a/internal/json/sort.go +++ /dev/null @@ -1,206 +0,0 @@ -/** - * Copyright 2023 CloudWeGo Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package json - -// Algorithm 3-way Radix Quicksort, d means the radix. -// Reference: https://algs4.cs.princeton.edu/51radix/Quick3string.java.html -func radixQsort(kvs PairSlice, d, maxDepth int) { - for len(kvs) > 11 { - // To avoid the worst case of quickSort (time: O(n^2)), use introsort here. - // Reference: https://en.wikipedia.org/wiki/Introsort and - // https://github.com/golang/go/issues/467 - if maxDepth == 0 { - heapSort(kvs, 0, len(kvs)) - return - } - maxDepth-- - - p := pivot(kvs, d) - lt, i, gt := 0, 0, len(kvs) - for i < gt { - c := byteAt(kvs[i].Key, d) - if c < p { - swap(kvs, lt, i) - i++ - lt++ - } else if c > p { - gt-- - swap(kvs, i, gt) - } else { - i++ - } - } - - // kvs[0:lt] < v = kvs[lt:gt] < kvs[gt:len(kvs)] - // Native implemention: - // radixQsort(kvs[:lt], d, maxDepth) - // if p > -1 { - // radixQsort(kvs[lt:gt], d+1, maxDepth) - // } - // radixQsort(kvs[gt:], d, maxDepth) - // Optimize as follows: make recursive calls only for the smaller parts. - // Reference: https://www.geeksforgeeks.org/quicksort-tail-call-optimization-reducing-worst-case-space-log-n/ - if p == -1 { - if lt > len(kvs)-gt { - radixQsort(kvs[gt:], d, maxDepth) - kvs = kvs[:lt] - } else { - radixQsort(kvs[:lt], d, maxDepth) - kvs = kvs[gt:] - } - } else { - ml := maxThree(lt, gt-lt, len(kvs)-gt) - if ml == lt { - radixQsort(kvs[lt:gt], d+1, maxDepth) - radixQsort(kvs[gt:], d, maxDepth) - kvs = kvs[:lt] - } else if ml == gt-lt { - radixQsort(kvs[:lt], d, maxDepth) - radixQsort(kvs[gt:], d, maxDepth) - kvs = kvs[lt:gt] - d += 1 - } else { - radixQsort(kvs[:lt], d, maxDepth) - radixQsort(kvs[lt:gt], d+1, maxDepth) - kvs = kvs[gt:] - } - } - } - insertRadixSort(kvs, d) -} - -func insertRadixSort(kvs PairSlice, d int) { - for i := 1; i < len(kvs); i++ { - for j := i; j > 0 && lessFrom(kvs[j].Key, kvs[j-1].Key, d); j-- { - swap(kvs, j, j-1) - } - } -} - -func pivot(kvs PairSlice, d int) int { - m := len(kvs) >> 1 - if len(kvs) > 40 { - // Tukey's ``Ninther,'' median of three mediankvs of three. - t := len(kvs) / 8 - return medianThree( - medianThree(byteAt(kvs[0].Key, d), byteAt(kvs[t].Key, d), byteAt(kvs[2*t].Key, d)), - medianThree(byteAt(kvs[m].Key, d), byteAt(kvs[m-t].Key, d), byteAt(kvs[m+t].Key, d)), - medianThree(byteAt(kvs[len(kvs)-1].Key, d), - byteAt(kvs[len(kvs)-1-t].Key, d), - byteAt(kvs[len(kvs)-1-2*t].Key, d))) - } - return medianThree(byteAt(kvs[0].Key, d), byteAt(kvs[m].Key, d), byteAt(kvs[len(kvs)-1].Key, d)) -} - -func medianThree(i, j, k int) int { - if i > j { - i, j = j, i - } // i < j - if k < i { - return i - } - if k > j { - return j - } - return k -} - -func maxThree(i, j, k int) int { - max := i - if max < j { - max = j - } - if max < k { - max = k - } - return max -} - -// maxDepth returns a threshold at which quicksort should switch -// to heapsort. It returnkvs 2*ceil(lg(n+1)). -func maxDepth(n int) int { - var depth int - for i := n; i > 0; i >>= 1 { - depth++ - } - return depth * 2 -} - -// siftDown implements the heap property on kvs[lo:hi]. -// first is an offset into the array where the root of the heap lies. -func siftDown(kvs PairSlice, lo, hi, first int) { - root := lo - for { - child := 2*root + 1 - if child >= hi { - break - } - if child+1 < hi && kvs[first+child].Key < kvs[first+child+1].Key { - child++ - } - if kvs[first+root].Key >= kvs[first+child].Key { - return - } - swap(kvs, first+root, first+child) - root = child - } -} - -func heapSort(kvs PairSlice, a, b int) { - first := a - lo := 0 - hi := b - a - - // Build heap with the greatest element at top. - for i := (hi - 1) / 2; i >= 0; i-- { - siftDown(kvs, i, hi, first) - } - - // Pop elements, the largest first, into end of kvs. - for i := hi - 1; i >= 0; i-- { - swap(kvs, first, first+i) - siftDown(kvs, lo, i, first) - } -} - -// Note that Pair.Key is NOT pointed to Pair.m when map key is integer after swap -func swap(kvs PairSlice, a, b int) { - kvs[a].Key, kvs[b].Key = kvs[b].Key, kvs[a].Key - kvs[a].Value, kvs[b].Value = kvs[b].Value, kvs[a].Value -} - -// Compare two strings from the pos d. -func lessFrom(a, b string, d int) bool { - l := len(a) - if l > len(b) { - l = len(b) - } - for i := d; i < l; i++ { - if a[i] == b[i] { - continue - } - return a[i] < b[i] - } - return len(a) < len(b) -} - -func byteAt(b string, p int) int { - if p < len(b) { - return int(b[p]) - } - return -1 -} diff --git a/internal/json/stubs.go b/internal/json/stubs.go deleted file mode 100644 index 8bc09ae9..00000000 --- a/internal/json/stubs.go +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Copyright 2023 CloudWeGo Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package json - -import ( - "reflect" - "unicode/utf8" - "unsafe" - - "github.com/cloudwego/dynamicgo/internal/rt" -) - -var ( - byteType = rt.UnpackType(reflect.TypeOf(byte(0))) -) - -//go:noescape -//go:linkname memmove runtime.memmove -//goland:noinspection GoUnusedParameter -func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr) - -//go:linkname unsafe_NewArray reflect.unsafe_NewArray -//goland:noinspection GoUnusedParameter -func unsafe_NewArray(typ *rt.GoType, n int) unsafe.Pointer - -//go:nosplit -func mem2ptr(s []byte) unsafe.Pointer { - return (*rt.GoSlice)(unsafe.Pointer(&s)).Ptr -} - -//go:nosplit -func ptr2slice(s unsafe.Pointer, l int, c int) unsafe.Pointer { - slice := &rt.GoSlice{ - Ptr: s, - Len: l, - Cap: c, - } - return unsafe.Pointer(slice) -} - -//go:nosplit -func str2ptr(s string) unsafe.Pointer { - return (*rt.GoString)(unsafe.Pointer(&s)).Ptr -} - -//go:nosplit -func addr2str(p unsafe.Pointer, n int64) (s string) { - (*rt.GoString)(unsafe.Pointer(&s)).Ptr = p - (*rt.GoString)(unsafe.Pointer(&s)).Len = int(n) - return -} - -var ( - //go:linkname safeSet encoding/json.safeSet - safeSet [utf8.RuneSelf]bool - - //go:linkname hex encoding/json.hex - hex string -) - -//go:linkname unquoteBytes encoding/json.unquoteBytes -func unquoteBytes(s []byte) (t []byte, ok bool) diff --git a/internal/json/testdata_test.go b/internal/json/testdata_test.go deleted file mode 100644 index 054c2e50..00000000 --- a/internal/json/testdata_test.go +++ /dev/null @@ -1,531 +0,0 @@ -/** - * Copyright 2023 CloudWeGo Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package json - -const _TwitterJson = `{ - "statuses": [ - { - "coordinates": null, - "favorited": false, - "truncated": false, - "created_at": "Mon Sep 24 03:35:21 +0000 2012", - "id_str": "250075927172759552", - "entities": { - "urls": [ - - ], - "hashtags": [ - { - "text": "freebandnames", - "indices": [ - 20, - 34 - ] - } - ], - "user_mentions": [ - - ] - }, - "in_reply_to_user_id_str": null, - "contributors": null, - "text": "Aggressive Ponytail #freebandnames", - "metadata": { - "iso_language_code": "en", - "result_type": "recent" - }, - "retweet_count": 0, - "in_reply_to_status_id_str": null, - "id": 250075927172759552, - "geo": null, - "retweeted": false, - "in_reply_to_user_id": null, - "place": null, - "user": { - "profile_sidebar_fill_color": "DDEEF6", - "profile_sidebar_border_color": "C0DEED", - "profile_background_tile": false, - "name": "Sean Cummings", - "profile_image_url": "http://a0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg", - "created_at": "Mon Apr 26 06:01:55 +0000 2010", - "location": "LA, CA", - "follow_request_sent": null, - "profile_link_color": "0084B4", - "is_translator": false, - "id_str": "137238150", - "entities": { - "url": { - "urls": [ - { - "expanded_url": null, - "url": "", - "indices": [ - 0, - 0 - ] - } - ] - }, - "description": { - "urls": [ - - ] - } - }, - "default_profile": true, - "contributors_enabled": false, - "favourites_count": 0, - "url": null, - "profile_image_url_https": "https://si0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg", - "utc_offset": -28800, - "id": 137238150, - "profile_use_background_image": true, - "listed_count": 2, - "profile_text_color": "333333", - "lang": "en", - "followers_count": 70, - "protected": false, - "notifications": null, - "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", - "profile_background_color": "C0DEED", - "verified": false, - "geo_enabled": true, - "time_zone": "Pacific Time (US & Canada)", - "description": "Born 330 Live 310", - "default_profile_image": false, - "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", - "statuses_count": 579, - "friends_count": 110, - "following": null, - "show_all_inline_media": false, - "screen_name": "sean_cummings" - }, - "in_reply_to_screen_name": null, - "source": "Twitter for Mac", - "in_reply_to_status_id": null - }, - { - "coordinates": null, - "favorited": false, - "truncated": false, - "created_at": "Fri Sep 21 23:40:54 +0000 2012", - "id_str": "249292149810667520", - "entities": { - "urls": [ - - ], - "hashtags": [ - { - "text": "FreeBandNames", - "indices": [ - 20, - 34 - ] - } - ], - "user_mentions": [ - - ] - }, - "in_reply_to_user_id_str": null, - "contributors": null, - "text": "Thee Namaste Nerdz. #FreeBandNames", - "metadata": { - "iso_language_code": "pl", - "result_type": "recent" - }, - "retweet_count": 0, - "in_reply_to_status_id_str": null, - "id": 249292149810667520, - "geo": null, - "retweeted": false, - "in_reply_to_user_id": null, - "place": null, - "user": { - "profile_sidebar_fill_color": "DDFFCC", - "profile_sidebar_border_color": "BDDCAD", - "profile_background_tile": true, - "name": "Chaz Martenstein", - "profile_image_url": "http://a0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg", - "created_at": "Tue Apr 07 19:05:07 +0000 2009", - "location": "Durham, NC", - "follow_request_sent": null, - "profile_link_color": "0084B4", - "is_translator": false, - "id_str": "29516238", - "entities": { - "url": { - "urls": [ - { - "expanded_url": null, - "url": "http://bullcityrecords.com/wnng/", - "indices": [ - 0, - 32 - ] - } - ] - }, - "description": { - "urls": [ - - ] - } - }, - "default_profile": false, - "contributors_enabled": false, - "favourites_count": 8, - "url": "http://bullcityrecords.com/wnng/", - "profile_image_url_https": "https://si0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg", - "utc_offset": -18000, - "id": 29516238, - "profile_use_background_image": true, - "listed_count": 118, - "profile_text_color": "333333", - "lang": "en", - "followers_count": 2052, - "protected": false, - "notifications": null, - "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/9423277/background_tile.bmp", - "profile_background_color": "9AE4E8", - "verified": false, - "geo_enabled": false, - "time_zone": "Eastern Time (US & Canada)", - "description": "You will come to Durham, North Carolina. I will sell you some records then, here in Durham, North Carolina. Fun will happen.", - "default_profile_image": false, - "profile_background_image_url": "http://a0.twimg.com/profile_background_images/9423277/background_tile.bmp", - "statuses_count": 7579, - "friends_count": 348, - "following": null, - "show_all_inline_media": true, - "screen_name": "bullcityrecords" - }, - "in_reply_to_screen_name": null, - "source": "web", - "in_reply_to_status_id": null - }, - { - "coordinates": null, - "favorited": false, - "truncated": false, - "created_at": "Fri Sep 21 23:30:20 +0000 2012", - "id_str": "249289491129438208", - "entities": { - "urls": [ - - ], - "hashtags": [ - { - "text": "freebandnames", - "indices": [ - 29, - 43 - ] - } - ], - "user_mentions": [ - - ] - }, - "in_reply_to_user_id_str": null, - "contributors": null, - "text": "Mexican Heaven, Mexican Hell #freebandnames", - "metadata": { - "iso_language_code": "en", - "result_type": "recent" - }, - "retweet_count": 0, - "in_reply_to_status_id_str": null, - "id": 249289491129438208, - "geo": null, - "retweeted": false, - "in_reply_to_user_id": null, - "place": null, - "user": { - "profile_sidebar_fill_color": "99CC33", - "profile_sidebar_border_color": "829D5E", - "profile_background_tile": false, - "name": "Thomas John Wakeman", - "profile_image_url": "http://a0.twimg.com/profile_images/2219333930/Froggystyle_normal.png", - "created_at": "Tue Sep 01 21:21:35 +0000 2009", - "location": "Kingston New York", - "follow_request_sent": null, - "profile_link_color": "D02B55", - "is_translator": false, - "id_str": "70789458", - "entities": { - "url": { - "urls": [ - { - "expanded_url": null, - "url": "", - "indices": [ - 0, - 0 - ] - } - ] - }, - "description": { - "urls": [ - - ] - } - }, - "default_profile": false, - "contributors_enabled": false, - "favourites_count": 19, - "url": null, - "profile_image_url_https": "https://si0.twimg.com/profile_images/2219333930/Froggystyle_normal.png", - "utc_offset": -18000, - "id": 70789458, - "profile_use_background_image": true, - "listed_count": 1, - "profile_text_color": "3E4415", - "lang": "en", - "followers_count": 63, - "protected": false, - "notifications": null, - "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme5/bg.gif", - "profile_background_color": "352726", - "verified": false, - "geo_enabled": false, - "time_zone": "Eastern Time (US & Canada)", - "description": "Science Fiction Writer, sort of. Likes Superheroes, Mole People, Alt. Timelines.", - "default_profile_image": false, - "profile_background_image_url": "http://a0.twimg.com/images/themes/theme5/bg.gif", - "statuses_count": 1048, - "friends_count": 63, - "following": null, - "show_all_inline_media": false, - "screen_name": "MonkiesFist" - }, - "in_reply_to_screen_name": null, - "source": "web", - "in_reply_to_status_id": null - }, - { - "coordinates": null, - "favorited": false, - "truncated": false, - "created_at": "Fri Sep 21 22:51:18 +0000 2012", - "id_str": "249279667666817024", - "entities": { - "urls": [ - - ], - "hashtags": [ - { - "text": "freebandnames", - "indices": [ - 20, - 34 - ] - } - ], - "user_mentions": [ - - ] - }, - "in_reply_to_user_id_str": null, - "contributors": null, - "text": "The Foolish Mortals #freebandnames", - "metadata": { - "iso_language_code": "en", - "result_type": "recent" - }, - "retweet_count": 0, - "in_reply_to_status_id_str": null, - "id": 249279667666817024, - "geo": null, - "retweeted": false, - "in_reply_to_user_id": null, - "place": null, - "user": { - "profile_sidebar_fill_color": "BFAC83", - "profile_sidebar_border_color": "615A44", - "profile_background_tile": true, - "name": "Marty Elmer", - "profile_image_url": "http://a0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png", - "created_at": "Mon May 04 00:05:00 +0000 2009", - "location": "Wisconsin, USA", - "follow_request_sent": null, - "profile_link_color": "3B2A26", - "is_translator": false, - "id_str": "37539828", - "entities": { - "url": { - "urls": [ - { - "expanded_url": null, - "url": "http://www.omnitarian.me", - "indices": [ - 0, - 24 - ] - } - ] - }, - "description": { - "urls": [ - - ] - } - }, - "default_profile": false, - "contributors_enabled": false, - "favourites_count": 647, - "url": "http://www.omnitarian.me", - "profile_image_url_https": "https://si0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png", - "utc_offset": -21600, - "id": 37539828, - "profile_use_background_image": true, - "listed_count": 52, - "profile_text_color": "000000", - "lang": "en", - "followers_count": 608, - "protected": false, - "notifications": null, - "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/106455659/rect6056-9.png", - "profile_background_color": "EEE3C4", - "verified": false, - "geo_enabled": false, - "time_zone": "Central Time (US & Canada)", - "description": "Cartoonist, Illustrator, and T-Shirt connoisseur", - "default_profile_image": false, - "profile_background_image_url": "http://a0.twimg.com/profile_background_images/106455659/rect6056-9.png", - "statuses_count": 3575, - "friends_count": 249, - "following": null, - "show_all_inline_media": true, - "screen_name": "Omnitarian" - }, - "in_reply_to_screen_name": null, - "source": "Twitter for iPhone", - "in_reply_to_status_id": null - } - ], - "search_metadata": { - "max_id": 250126199840518145, - "since_id": 24012619984051000, - "refresh_url": "?since_id=250126199840518145&q=%23freebandnames&result_type=mixed&include_entities=1", - "next_results": "?max_id=249279667666817023&q=%23freebandnames&count=4&include_entities=1&result_type=mixed", - "count": 4, - "completed_in": 0.035, - "since_id_str": "24012619984051000", - "query": "%23freebandnames", - "max_id_str": "250126199840518145" - } -}` - -type _TwitterStruct struct { - Statuses []struct { - Coordinates interface{} `json:"coordinates"` - Favorited bool `json:"favorited"` - Truncated bool `json:"truncated"` - CreatedAt string `json:"created_at"` - IDStr string `json:"id_str"` - Entities struct { - Urls []interface{} `json:"urls"` - Hashtags []struct { - Text string `json:"text"` - Indices []int `json:"indices"` - } `json:"hashtags"` - UserMentions []interface{} `json:"user_mentions"` - } `json:"entities"` - InReplyToUserIDStr interface{} `json:"in_reply_to_user_id_str"` - Contributors interface{} `json:"contributors"` - Text string `json:"text"` - Metadata struct { - IsoLanguageCode string `json:"iso_language_code"` - ResultType string `json:"result_type"` - } `json:"metadata"` - RetweetCount int `json:"retweet_count"` - InReplyToStatusIDStr interface{} `json:"in_reply_to_status_id_str"` - ID int64 `json:"id"` - Geo interface{} `json:"geo"` - Retweeted bool `json:"retweeted"` - InReplyToUserID interface{} `json:"in_reply_to_user_id"` - Place interface{} `json:"place"` - User struct { - ProfileSidebarFillColor string `json:"profile_sidebar_fill_color"` - ProfileSidebarBorderColor string `json:"profile_sidebar_border_color"` - ProfileBackgroundTile bool `json:"profile_background_tile"` - Name string `json:"name"` - ProfileImageURL string `json:"profile_image_url"` - CreatedAt string `json:"created_at"` - Location string `json:"location"` - FollowRequestSent interface{} `json:"follow_request_sent"` - ProfileLinkColor string `json:"profile_link_color"` - IsTranslator bool `json:"is_translator"` - IDStr string `json:"id_str"` - Entities struct { - URL struct { - Urls []struct { - ExpandedURL interface{} `json:"expanded_url"` - URL string `json:"url"` - Indices []int `json:"indices"` - } `json:"urls"` - } `json:"url"` - Description struct { - Urls []interface{} `json:"urls"` - } `json:"description"` - } `json:"entities"` - DefaultProfile bool `json:"default_profile"` - ContributorsEnabled bool `json:"contributors_enabled"` - FavouritesCount int `json:"favourites_count"` - URL interface{} `json:"url"` - ProfileImageURLHTTPS string `json:"profile_image_url_https"` - UtcOffset int `json:"utc_offset"` - ID int `json:"id"` - ProfileUseBackgroundImage bool `json:"profile_use_background_image"` - ListedCount int `json:"listed_count"` - ProfileTextColor string `json:"profile_text_color"` - Lang string `json:"lang"` - FollowersCount int `json:"followers_count"` - Protected bool `json:"protected"` - Notifications interface{} `json:"notifications"` - ProfileBackgroundImageURLHTTPS string `json:"profile_background_image_url_https"` - ProfileBackgroundColor string `json:"profile_background_color"` - Verified bool `json:"verified"` - GeoEnabled bool `json:"geo_enabled"` - TimeZone string `json:"time_zone"` - Description string `json:"description"` - DefaultProfileImage bool `json:"default_profile_image"` - ProfileBackgroundImageURL string `json:"profile_background_image_url"` - StatusesCount int `json:"statuses_count"` - FriendsCount int `json:"friends_count"` - Following interface{} `json:"following"` - ShowAllInlineMedia bool `json:"show_all_inline_media"` - ScreenName string `json:"screen_name"` - } `json:"user"` - InReplyToScreenName interface{} `json:"in_reply_to_screen_name"` - Source string `json:"source"` - InReplyToStatusID interface{} `json:"in_reply_to_status_id"` - } `json:"statuses"` - SearchMetadata struct { - MaxID int64 `json:"max_id"` - SinceID int64 `json:"since_id"` - RefreshURL string `json:"refresh_url"` - NextResults string `json:"next_results"` - Count int `json:"count"` - CompletedIn float64 `json:"completed_in"` - SinceIDStr string `json:"since_id_str"` - Query string `json:"query"` - MaxIDStr string `json:"max_id_str"` - } `json:"search_metadata"` -} diff --git a/internal/native/dispatch_amd64.go b/internal/native/dispatch_amd64.go index f7ccd1d6..494346e0 100644 --- a/internal/native/dispatch_amd64.go +++ b/internal/native/dispatch_amd64.go @@ -40,18 +40,6 @@ var ( //go:linkname Quote github.com/bytedance/sonic/internal/native.Quote func Quote(s unsafe.Pointer, nb int, dp unsafe.Pointer, dn *int, flags uint64) int -//go:nosplit -//go:linkname Unquote github.com/bytedance/sonic/internal/native.Unquote -func Unquote(s unsafe.Pointer, nb int, dp unsafe.Pointer, ep *int, flags uint64) int - -//go:nosplit -//go:linkname Value github.com/bytedance/sonic/internal/native.Value -func Value(s unsafe.Pointer, n int, p int, v *types.JsonState, allow_control int) int - -//go:nosplit -//go:linkname SkipOne github.com/bytedance/sonic/internal/native.SkipOne -func SkipOne(s *string, p *int, m *types.StateMachine) int - //go:nosplit //go:linkname I64toa github.com/bytedance/sonic/internal/native.I64toa func I64toa(out *byte, val int64) (ret int) diff --git a/internal/unquote/unquote.go b/internal/unquote/unquote.go deleted file mode 100644 index 1b0b95e7..00000000 --- a/internal/unquote/unquote.go +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2023 CloudWeGo Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package unquote - -import ( - "reflect" - "runtime" - "unsafe" - - "github.com/cloudwego/dynamicgo/internal/native" - "github.com/cloudwego/dynamicgo/internal/native/types" - "github.com/cloudwego/dynamicgo/internal/rt" -) - -func String(s string) (ret string, err types.ParsingError) { - mm := make([]byte, 0, len(s)) - err = intoBytesUnsafe(s, &mm) - ret = rt.Mem2Str(mm) - return -} - -func IntoBytes(s string, m *[]byte) types.ParsingError { - if cap(*m) < len(s) { - return types.ERR_EOF - } else { - return intoBytesUnsafe(s, m) - } -} - -func intoBytesUnsafe(s string, m *[]byte) types.ParsingError { - pos := -1 - slv := (*rt.GoSlice)(unsafe.Pointer(m)) - str := (*rt.GoString)(unsafe.Pointer(&s)) - ret := native.Unquote(str.Ptr, str.Len, slv.Ptr, &pos, 0) - - /* check for errors */ - if ret < 0 { - return types.ParsingError(-ret) - } - - /* update the length */ - slv.Len = ret - return 0 -} - -var typeByte = rt.UnpackType(reflect.TypeOf(byte(0))) - -func QuoteIntoBytes(val string, buf *[]byte) { - sp := (*rt.GoString)(unsafe.Pointer(&val)).Ptr - nb := len(val) - b := (*rt.GoSlice)(unsafe.Pointer(buf)) - // input buffer - for nb > 0 { - // output buffer - dp := unsafe.Pointer(uintptr(b.Ptr) + uintptr(b.Len)) - dn := b.Cap - b.Len - // call native.Quote, dn is byte count it outputs - ret := native.Quote(sp, nb, dp, &dn, 0) - // update *buf length - b.Len += dn - - // no need more output - if ret >= 0 { - break - } - - // double buf size - *b = rt.Growslice(typeByte, *b, b.Cap*2) - // ret is the complement of consumed input - ret = ^ret - // update input buffer - nb -= ret - sp = unsafe.Pointer(uintptr(sp) + uintptr(ret)) - } - runtime.KeepAlive(buf) - runtime.KeepAlive(sp) -} - -func QuoteString(val string) string { - buf := make([]byte, 0, len(val)+2) - QuoteIntoBytes(val, &buf) - return rt.Mem2Str(buf) -} diff --git a/testdata/baseline_j2t_test.go b/testdata/baseline_j2t_test.go index 676dfef2..6e7d3db9 100644 --- a/testdata/baseline_j2t_test.go +++ b/testdata/baseline_j2t_test.go @@ -33,7 +33,7 @@ import ( "github.com/cloudwego/dynamicgo/conv" "github.com/cloudwego/dynamicgo/conv/j2t" "github.com/cloudwego/dynamicgo/http" - "github.com/cloudwego/dynamicgo/internal/json" + json "github.com/bytedance/sonic/ast" "github.com/cloudwego/dynamicgo/testdata/kitex_gen/baseline" "github.com/cloudwego/kitex/pkg/generic" "github.com/cloudwego/kitex/pkg/generic/descriptor"