feat: fallback using kitex's http value encoding

conv/api.go

@@ -93,6 +93,9 @@ type Options struct {
 
 	// ConvertException indicates that it returns error for thrift exception fields when doing BinaryConv t2j
 	ConvertException bool
+
+	// UseKitexHttpEncoding indicating using kitex's text encoding to output complex http values
+	UseKitexHttpEncoding bool
 }
 
 var bufPool = sync.Pool{

conv/j2t/impl.go

@@ -65,8 +65,8 @@ func (self *BinaryConv) do(ctx context.Context, src []byte, desc *thrift.TypeDes
 			// since this case it always for top-level fields,
 			// we should only check opts.BackTraceRequireOrTopField to decide whether to traceback
 			err := reqs.HandleRequires(st, self.opts.ReadHttpValueFallback, self.opts.ReadHttpValueFallback, self.opts.ReadHttpValueFallback, func(f *thrift.FieldDescriptor) error {
-				val, _ := tryGetValueFromHttp(req, f.Alias())
-				if err := self.writeStringValue(ctx, buf, f, val, meta.EncodingJSON, req); err != nil {
+				val, _, enc := tryGetValueFromHttp(req, f.Alias())
+				if err := self.writeStringValue(ctx, buf, f, val, enc, req); err != nil {
 					return err
 				}
 				return nil
@@ -116,9 +116,14 @@ func isJsonString(val string) bool {
 	if len(val) < 2 {
 		return false
 	}
-	s := val[0]
-	e := val[len(val)-1]
-	return (s == '{' && e == '}') || (s == '[' && e == ']')
+
+	c := json.SkipBlank(val, 0) 
+	if c < 0 {
+		return false
+	}
+	s := val[c]
+	e := val[len(val)-1] //FIXME: may need exist blank
+	return (s == '{' && e == '}') || (s == '[' && e == ']') || (s == '"' && e == '"')
 }
 
 func (self *BinaryConv) writeStringValue(ctx context.Context, buf *[]byte, f *thrift.FieldDescriptor, val string, enc meta.Encoding, req http.RequestGetter) error {
@@ -148,16 +153,18 @@ func (self *BinaryConv) writeStringValue(ctx context.Context, buf *[]byte, f *th
 		if enc == meta.EncodingThriftBinary {
 			p.Buf = append(p.Buf, val...)
 			goto BACK
-		} else if enc != meta.EncodingJSON {
-			return newError(meta.ErrUnsupportedType, fmt.Sprintf("unsupported encoding type %d of field '%s'", enc, f.Name()), nil)
-		}
-		if ft := f.Type(); ft.Type().IsComplex() && isJsonString(val) {
+		} else if enc == meta.EncodingText || !f.Type().Type().IsComplex() || !isJsonString(val) {
+			if err := p.WriteStringWithDesc(val, f.Type(), self.opts.DisallowUnknownField, !self.opts.NoBase64Binary); err != nil {
+				return newError(meta.ErrConvert, fmt.Sprintf("failed to write field '%s' value", f.Name()), err)
+			}
+		} else if enc == meta.EncodingJSON {
 			// for nested type, we regard it as a json string and convert it directly
-			if err := self.doNative(ctx, rt.Str2Mem(val), ft, &p.Buf, req, false); err != nil {
+			if err := self.doNative(ctx, rt.Str2Mem(val), f.Type(), &p.Buf, req, false); err != nil {
 				return newError(meta.ErrConvert, fmt.Sprintf("failed to convert value of field '%s'", f.Name()), err)
 			}
-		} else if err := p.WriteStringWithDesc(val, ft, self.opts.DisallowUnknownField, !self.opts.NoBase64Binary); err != nil {
-			return newError(meta.ErrConvert, fmt.Sprintf("failed to write field '%s' value", f.Name()), err)
+		// try text encoding, see thrift.EncodeText
+		} else {
+			return newError(meta.ErrConvert, fmt.Sprintf("unsupported http-mapping encoding %v for '%s'", enc, f.Name()), nil)
 		}
 	}
 BACK:
@@ -248,11 +255,12 @@ func (self *BinaryConv) handleUnmatchedFields(ctx context.Context, fsm *types.J2
 			continue
 		}
 		var val string
+		var enc = meta.EncodingText
 		if self.opts.TracebackRequredOrRootFields && (top || f.Required() == thrift.RequiredRequireness) {
 			// try get value from http
-			val, _ = tryGetValueFromHttp(req, f.Alias())
+			val, _, enc = tryGetValueFromHttp(req, f.Alias())
 		}
-		if err := self.writeStringValue(ctx, buf, f, val, meta.EncodingJSON, req); err != nil {
+		if err := self.writeStringValue(ctx, buf, f, val, enc, req); err != nil {
 			return false, err
 		}
 	}
@@ -274,26 +282,26 @@ func (self *BinaryConv) handleUnmatchedFields(ctx context.Context, fsm *types.J2
 }
 
 // searching sequence: url -> [post] -> query -> header -> [body root]
-func tryGetValueFromHttp(req http.RequestGetter, key string) (string, bool) {
+func tryGetValueFromHttp(req http.RequestGetter, key string) (string, bool, meta.Encoding) {
 	if req == nil {
-		return "", false
+		return "", false, meta.EncodingText
 	}
 	if v := req.GetParam(key); v != "" {
-		return v, true
+		return v, true, meta.EncodingJSON
 	}
 	if v := req.GetQuery(key); v != "" {
-		return v, true
+		return v, true, meta.EncodingJSON
 	}
 	if v := req.GetHeader(key); v != "" {
-		return v, true
+		return v, true, meta.EncodingJSON
 	}
 	if v := req.GetCookie(key); v != "" {
-		return v, true
+		return v, true, meta.EncodingJSON
 	}
 	if v := req.GetMapBody(key); v != "" {
-		return v, true
+		return v, true, meta.EncodingJSON
 	}
-	return "", false
+	return "", false, meta.EncodingText
 }
 
 func (self *BinaryConv) handleValueMapping(ctx context.Context, fsm *types.J2TStateMachine, desc *thrift.StructDescriptor, buf *[]byte, pos int, src []byte) (bool, error) {

conv/t2j/conv_test.go

@@ -522,10 +522,10 @@ func TestJSONString(t *testing.T) {
 	exp := example3.NewExampleJSONString()
 	eobj := &example3.JSONObject{
 		A: "1",
-		B: 2,
+		B: -1,
 	}
 	exp.Header = eobj
-	exp.Header2 = map[int32]string{1: "1", 2: "2"}
+	exp.Header2 = map[int32]string{1: "1"}
 	exp.Cookie = &example3.JSONObject{}
 	exp.Cookie2 = []int32{1, 2}
 	in := make([]byte, exp.BLength())
@@ -653,7 +653,7 @@ func TestOptionalDefaultValue(t *testing.T) {
 		assert.Equal(t, exp, act)
 		require.Equal(t, "1.2", resp.Response.Header.Get("c"))
 		require.Equal(t, "const string", resp.Response.Cookies()[0].Value)
-		require.Equal(t, `""`, resp.Response.Header.Get("f"))
+		require.Equal(t, ``, resp.Response.Header.Get("f"))
 	})
 }
 
@@ -680,3 +680,41 @@ func TestSimpleArgs(t *testing.T) {
 		require.Equal(t, strconv.Itoa(math.MaxInt64), string(out))
 	})
 }
+
+func TestConvThrift2HTTP_KitexApiHeader(t *testing.T) {
+	// annotation.RegisterHttpMaping(annotation.APIHeader, annotation.HttpMapingHandler{Req:annotation.ApiHeaderRequest, Resp:annotation.ApiheaderKitexResponse, Enc:annotation.ApiHeaderKitexEncoding})
+
+	desc := thrift.FnResponse(thrift.GetFnDescFromFile("testdata/idl/example3.thrift", "JSONStringMethod", thrift.Options{}))
+	exp := example3.NewExampleJSONString()
+	eobj := &example3.JSONObject{
+		A: "1",
+		B: -1,
+	}
+	exp.Header = eobj
+	exp.Header2 = map[int32]string{1: "1"}
+	exp.Cookie = &example3.JSONObject{}
+	exp.Cookie2 = []int32{1, 2}
+	in := make([]byte, exp.BLength())
+	_ = exp.FastWriteNocopy(in, nil)
+
+	cv := NewBinaryConv(conv.Options{
+		EnableHttpMapping:      true,
+		WriteHttpValueFallback: true,
+		OmitHttpMappingErrors:  true,
+		UseKitexHttpEncoding: true,
+	})
+	ctx := context.Background()
+	resp := http.NewHTTPResponse()
+	ctx = context.WithValue(ctx, conv.CtxKeyHTTPResponse, resp)
+	out, err := cv.Do(ctx, desc, in)
+	require.NoError(t, err)
+
+	// act := example3.NewExampleJSONString()
+	require.Equal(t, "{\"Query\":{},\"Query2\":[]}", string(out))
+	require.Equal(t, "map[a:1 b:-1]", resp.Response.Header.Get("header"))
+	require.Equal(t, "map[1:1]", resp.Response.Header.Get("header2"))
+	require.Equal(t, "map[a: b:0]", resp.Cookies()[0].Value)
+	require.Equal(t, "1,2", resp.Cookies()[1].Value)
+
+	// annotation.RegisterHttpMaping(annotation.APIHeader, annotation.HttpMapingHandler{Req:annotation.ApiHeaderRequest, Resp: annotation.ApiHeaderResponse, Enc:annotation.ApiHeaderEncoding})
+}

conv/t2j/conv_timing_test.go

@@ -95,6 +95,7 @@ func BenchmarkThrift2HTTP_Parallel_DynamicGo(b *testing.B) {
 	conv := NewBinaryConv(conv.Options{
 		EnableValueMapping: true,
 		EnableHttpMapping:  true,
+		OmitHttpMappingErrors: true,
 	})
 	in := getExample3Data()
 	ctx := context.Background()

conv/t2j/impl.go

@@ -24,6 +24,7 @@ import (
 	"github.com/cloudwego/dynamicgo/conv"
 	"github.com/cloudwego/dynamicgo/http"
 	"github.com/cloudwego/dynamicgo/internal/json"
+	"github.com/cloudwego/dynamicgo/internal/primitive"
 	"github.com/cloudwego/dynamicgo/internal/rt"
 	"github.com/cloudwego/dynamicgo/meta"
 	"github.com/cloudwego/dynamicgo/thrift"
@@ -36,7 +37,7 @@ const (
 
 //go:noinline
 func wrapError(code meta.ErrCode, msg string, err error) error {
-	// panic(msg)
+	// panic(meta.NewError(meta.NewErrorCode(code, meta.THRIFT2JSON), msg, err))
 	return meta.NewError(meta.NewErrorCode(code, meta.THRIFT2JSON), msg, err)
 }
 
@@ -393,30 +394,16 @@ func (self *BinaryConv) handleUnsets(b *thrift.RequiresBitmap, desc *thrift.Stru
 		// check if field has http mapping
 		var ok = false
 		if hms := field.HTTPMappings(); self.opts.EnableHttpMapping && hms != nil {
-			tmp := make([]byte, 0, conv.DefaulHttpValueBufferSizeForJSON)
-			if err := writeDefaultOrEmpty(field, &tmp); err != nil {
-				return err
+			// make a default thrift value
+			p := thrift.BinaryProtocol{Buf: make([]byte, 0, conv.DefaulHttpValueBufferSizeForJSON)};
+			if err := p.WriteDefaultOrEmpty(field); err != nil {
+				return wrapError(meta.ErrWrite, fmt.Sprintf("encoding field '%s' default value failed", field.Name()), err)
 			}
-			val := rt.Mem2Str(tmp)
-			for _, hm := range hms {
-				if enc := hm.Encoding(); enc == meta.EncodingJSON {
-					if e := hm.Response(ctx, resp, field, val); e == nil {
-						ok = true
-						break
-					} else if !self.opts.OmitHttpMappingErrors {
-						return e
-					}
-				} else if enc == meta.EncodingThriftBinary {
-					// no thrift data, pass empty string to http mapping
-					if e := hm.Response(ctx, resp, field, ""); e == nil {
-						ok = true
-						break
-					} else if !self.opts.OmitHttpMappingErrors {
-						return e
-					}
-				} else {
-					return wrapError(meta.ErrUnsupportedType, fmt.Sprintf("unknown http mapping encoding %d", enc), nil)
-				}
+			// convert it into http
+			var err error
+			ok, err = self.writeHttpValue(ctx, resp, &p, field)
+			if err != nil {
+				return err
 			}
 		}
 		if ok {
@@ -527,59 +514,75 @@ func (self *BinaryConv) buildinTypeToKey(p *thrift.BinaryProtocol, dest *thrift.
 }
 
 func (self *BinaryConv) writeHttpValue(ctx context.Context, resp http.ResponseSetter, p *thrift.BinaryProtocol, field *thrift.FieldDescriptor) (ok bool, err error) {
-	var val string
-	var start = p.Read
-	if ft := field.Type(); ft.Type().IsComplex() {
-		// for nested type, convert it to a new JSON string
-		tmp := make([]byte, 0, conv.DefaulHttpValueBufferSizeForJSON)
-		err := self.doRecurse(ctx, ft, &tmp, resp, p)
-		if err != nil {
-			return false, unwrapError(fmt.Sprintf("mapping field %s failed, thrift pos:%d", field.Name(), p.Read), err)
-		}
-		val = rt.Mem2Str(tmp)
-	} else if ft.Type() == thrift.STRING && !ft.IsBinary() {
-		// special case for string, refer it directly from thrift
-		val, err = p.ReadString(!self.opts.NoCopyString)
-		if err != nil {
-			return false, wrapError(meta.ErrRead, "", err)
-		}
-	} else {
-		// scalar type, convert it to a generic string
-		tmp := make([]byte, 0, conv.DefaulHttpValueBufferSizeForScalar)
-		if err = p.ReadStringWithDesc(field.Type(), &tmp, self.opts.ByteAsUint8, self.opts.DisallowUnknownField, !self.opts.NoBase64Binary); err != nil {
-			return false, wrapError(meta.ErrRead, "", err)
-		}
-		val = rt.Mem2Str(tmp)
-	}
-
-	var rawVal string
+	var thriftVal []byte
+	var jsonVal []byte
+	var textVal []byte
+
 	for _, hm := range field.HTTPMappings() {
-		if enc := hm.Encoding(); enc == meta.EncodingJSON {
-			// NOTICE: ignore error if the value is not set
-			if e := hm.Response(ctx, resp, field, val); e == nil {
-				ok = true
-				break
-			} else if !self.opts.OmitHttpMappingErrors {
-				return false, e
-			}
-		} else if enc == meta.EncodingThriftBinary {
+		var val []byte
+		enc := hm.Encoding();
+
+		if enc == meta.EncodingThriftBinary {
 			// raw encoding, check if raw value is set
-			if rawVal == "" {
-				//  skip the value and save it if for later use
-				p.Read = start
+			if thriftVal == nil {
+				var start = p.Read
 				if err := p.Skip(field.Type().Type(), self.opts.UseNativeSkip); err != nil {
 					return false, wrapError(meta.ErrRead, "", err)
 				}
-				rawVal = rt.Mem2Str((p.Buf[start:p.Read]))
+				val = p.Buf[start:p.Read]
+				thriftVal = val
+			} else {
+				val = thriftVal
 			}
-			if e := hm.Response(ctx, resp, field, rawVal); e == nil {
-				ok = true
-				break
-			} else if !self.opts.OmitHttpMappingErrors {
-				return false, e
+
+		} else if enc == meta.EncodingText || !field.Type().Type().IsComplex() { // not complex value, must use text encoding
+			if textVal == nil {
+				tmp := make([]byte, 0, conv.DefaulHttpValueBufferSizeForJSON)
+				err = p.ReadStringWithDesc(field.Type(), &tmp, self.opts.ByteAsUint8, self.opts.DisallowUnknownField, !self.opts.NoBase64Binary)
+				if err != nil {
+					return false, unwrapError(fmt.Sprintf("reading thrift value of '%s' failed, thrift pos:%d", field.Name(), p.Read), err)
+				}
+				val = tmp
+				textVal = val 
+			} else {
+				val = textVal
+			} 
+		} else if self.opts.UseKitexHttpEncoding {
+			// kitex http encoding fallback
+			if textVal == nil {
+				obj, err := p.ReadAnyWithDesc(field.Type(), self.opts.ByteAsUint8, !self.opts.NoCopyString, self.opts.DisallowUnknownField, true)
+				if err != nil {
+					return false, unwrapError(fmt.Sprintf("reading thrift value of '%s' failed, thrift pos:%d", field.Name(), p.Read), err)
+				}
+				textVal = rt.Str2Mem(primitive.KitexToString(obj))
+				val = textVal
+			} else {
+				val = textVal
+			} 
+		} else if enc == meta.EncodingJSON {
+			// for nested type, convert it to a new JSON string
+			if jsonVal == nil {
+				tmp := make([]byte, 0, conv.DefaulHttpValueBufferSizeForJSON)
+				err := self.doRecurse(ctx, field.Type(), &tmp, resp, p)
+				if err != nil {
+					return false, unwrapError(fmt.Sprintf("mapping field %s failed, thrift pos:%d", field.Name(), p.Read), err)
+				}
+				val = tmp
+				jsonVal = val
+			} else {
+				val = jsonVal
 			}
+
 		} else {
-			return false, wrapError(meta.ErrUnsupportedType, fmt.Sprintf("unsupported http mapping encoding %d", enc), nil)
+			return false, wrapError(meta.ErrConvert, fmt.Sprintf("unsuported http-value encoding %v of field '%s'", enc, field.Name()), nil)
+		}
+
+		// NOTICE: ignore error if the value is not set
+		if e := hm.Response(ctx, resp, field, rt.Mem2Str(val)); e == nil {
+			ok = true
+			break
+		} else if !self.opts.OmitHttpMappingErrors {
+			return false, e
 		}
 	}
 	return