From 25d920adc737408e5dc80bfa1d52468e6c2e7b42 Mon Sep 17 00:00:00 2001 From: "liuqiang.06" Date: Thu, 27 Feb 2025 17:26:26 +0800 Subject: [PATCH] fix: support serde with unsupported types --- .../decoder/jitdec/assembler_regabi_amd64.go | 8 +- internal/decoder/jitdec/compiler.go | 15 +++- internal/decoder/optdec/compiler.go | 4 +- internal/decoder/optdec/errors.go | 6 ++ internal/decoder/optdec/functor.go | 13 +++ internal/encoder/compiler.go | 7 +- internal/encoder/ir/op.go | 6 ++ internal/encoder/pools_amd64.go | 21 +---- internal/encoder/vars/errors.go | 4 + internal/encoder/vm/vm.go | 9 +- .../encoder/x86/assembler_regabi_amd64.go | 10 +++ issue_test/common_test.go | 5 +- issue_test/issue491_test.go | 89 +++++++++++++++++++ issue_test/issue755_test.go | 3 + 14 files changed, 165 insertions(+), 35 deletions(-) create mode 100644 issue_test/issue491_test.go diff --git a/internal/decoder/jitdec/assembler_regabi_amd64.go b/internal/decoder/jitdec/assembler_regabi_amd64.go index 04e854df7..ed709ceb6 100644 --- a/internal/decoder/jitdec/assembler_regabi_amd64.go +++ b/internal/decoder/jitdec/assembler_regabi_amd64.go @@ -313,9 +313,10 @@ var _OpFuncTab = [256]func(*_Assembler, *_Instr) { _OP_check_char_0 : (*_Assembler)._asm_OP_check_char_0, _OP_dismatch_err : (*_Assembler)._asm_OP_dismatch_err, _OP_go_skip : (*_Assembler)._asm_OP_go_skip, - _OP_skip_emtpy : (*_Assembler)._asm_OP_skip_empty, + _OP_skip_emtpy : (*_Assembler)._asm_OP_skip_empty, _OP_add : (*_Assembler)._asm_OP_add, _OP_check_empty : (*_Assembler)._asm_OP_check_empty, + _OP_unsupported : (*_Assembler)._asm_OP_unsupported, _OP_debug : (*_Assembler)._asm_OP_debug, } @@ -1265,6 +1266,11 @@ func (self *_Assembler) _asm_OP_dyn(p *_Instr) { self.Link("_decode_end_{n}") // _decode_end_{n}: } +func (self *_Assembler) _asm_OP_unsupported(p *_Instr) { + self.Emit("MOVQ", jit.Type(p.vt()), _ET) // MOVQ ${p.vt()}, ET + self.Sjmp("JMP" , _LB_type_error) // JMP _LB_type_error +} + func (self *_Assembler) _asm_OP_str(_ *_Instr) { self.parse_string() // PARSE STRING self.unquote_once(jit.Ptr(_VP, 0), jit.Ptr(_VP, 8), false, true) // UNQUOTE once, (VP), 8(VP) diff --git a/internal/decoder/jitdec/compiler.go b/internal/decoder/jitdec/compiler.go index 33191262a..d73db94e2 100644 --- a/internal/decoder/jitdec/compiler.go +++ b/internal/decoder/jitdec/compiler.go @@ -99,6 +99,7 @@ const ( _OP_skip_emtpy _OP_add _OP_check_empty + _OP_unsupported _OP_debug ) @@ -176,6 +177,7 @@ var _OpNames = [256]string { _OP_add : "add", _OP_go_skip : "go_skip", _OP_check_empty : "check_empty", + _OP_unsupported : "unsupported type", _OP_debug : "debug", } @@ -630,10 +632,17 @@ func (self *_Compiler) compileOps(p *_Program, sp int, vt reflect.Type) { case reflect.Ptr : self.compilePtr (p, sp, vt) case reflect.Slice : self.compileSlice (p, sp, vt) case reflect.Struct : self.compileStruct (p, sp, vt) - default : panic (&json.UnmarshalTypeError{Type: vt}) + default : self.compileUnsupportedType (p, vt) } } +func (self *_Compiler) compileUnsupportedType(p *_Program, vt reflect.Type) { + i := p.pc() + p.add(_OP_is_null) + p.rtt(_OP_unsupported, vt) + p.pin(i) +} + func (self *_Compiler) compileMap(p *_Program, sp int, vt reflect.Type) { if reflect.PtrTo(vt.Key()).Implements(encodingTextUnmarshalerType) { self.compileMapOp(p, sp, vt, _OP_map_key_utext_p) @@ -1135,13 +1144,11 @@ func (self *_Compiler) compileInterface(p *_Program, vt reflect.Type) { p.pin(j) } -func (self *_Compiler) compilePrimitive(vt reflect.Type, p *_Program, op _Op) { +func (self *_Compiler) compilePrimitive(_ reflect.Type, p *_Program, op _Op) { i := p.pc() p.add(_OP_is_null) - // skip := self.checkPrimitive(p, vt) p.add(op) p.pin(i) - // p.pin(skip) } func (self *_Compiler) compileUnmarshalEnd(p *_Program, vt reflect.Type, i int) { diff --git a/internal/decoder/optdec/compiler.go b/internal/decoder/optdec/compiler.go index eeb252975..cc763f366 100644 --- a/internal/decoder/optdec/compiler.go +++ b/internal/decoder/optdec/compiler.go @@ -169,7 +169,9 @@ func (c *compiler) compileBasic(vt reflect.Type) decFunc { case reflect.Struct: return c.compileStruct(vt) default: - panic(&json.UnmarshalTypeError{Type: vt}) + return &unsupportedTypeDecoder{ + typ: rt.UnpackType(vt), + } } } diff --git a/internal/decoder/optdec/errors.go b/internal/decoder/optdec/errors.go index db0af547b..daa75c59c 100644 --- a/internal/decoder/optdec/errors.go +++ b/internal/decoder/optdec/errors.go @@ -70,4 +70,10 @@ Msg: msg, } } + + func error_unsuppoted(typ *rt.GoType) error { + return &json.UnsupportedTypeError{ + Type: typ.Pack(), + } +} \ No newline at end of file diff --git a/internal/decoder/optdec/functor.go b/internal/decoder/optdec/functor.go index 2a0523d5e..48a8953c1 100644 --- a/internal/decoder/optdec/functor.go +++ b/internal/decoder/optdec/functor.go @@ -279,3 +279,16 @@ func (d *recuriveDecoder) FromDom(vp unsafe.Pointer, node Node, ctx *context) er } return dec.FromDom(vp, node, ctx) } + +type unsupportedTypeDecoder struct { + typ *rt.GoType +} + + +func (d *unsupportedTypeDecoder) FromDom(vp unsafe.Pointer, node Node, ctx *context) error { + if node.IsNull() { + return nil + } + return error_unsuppoted(d.typ) +} + diff --git a/internal/encoder/compiler.go b/internal/encoder/compiler.go index 3975d90f4..575d362f8 100644 --- a/internal/encoder/compiler.go +++ b/internal/encoder/compiler.go @@ -219,7 +219,7 @@ func (self *Compiler) compileOps(p *ir.Program, sp int, vt reflect.Type) { case reflect.Struct: self.compileStruct(p, sp, vt) default: - panic(vars.Error_type(vt)) + self.compileUnsupportedType(p, vt) } } @@ -644,6 +644,11 @@ func (self *Compiler) compileInterface(p *ir.Program, vt reflect.Type) { p.Pin(e) } +func (self *Compiler) compileUnsupportedType(p *ir.Program, vt reflect.Type) { + p.Rtt(ir.OP_unsupported, vt) +} + + func (self *Compiler) compileMarshaler(p *ir.Program, op ir.Op, vt reflect.Type, mt reflect.Type) { pc := p.PC() vk := vt.Kind() diff --git a/internal/encoder/ir/op.go b/internal/encoder/ir/op.go index a0c693f00..32cc1ad01 100644 --- a/internal/encoder/ir/op.go +++ b/internal/encoder/ir/op.go @@ -80,6 +80,7 @@ const ( OP_marshal_text_p OP_cond_set OP_cond_testc + OP_unsupported ) const ( @@ -141,6 +142,7 @@ var OpNames = [256]string{ OP_marshal_text_p: "marshal_text_p", OP_cond_set: "cond_set", OP_cond_testc: "cond_testc", + OP_unsupported: "unsupported type", } func (self Op) String() string { @@ -273,6 +275,10 @@ func (self Instr) Vk() reflect.Kind { return (*rt.GoType)(self.p).Kind() } +func (self Instr) GoType() *rt.GoType { + return (*rt.GoType)(self.p) +} + func (self Instr) Vt() reflect.Type { return (*rt.GoType)(self.p).Pack() } diff --git a/internal/encoder/pools_amd64.go b/internal/encoder/pools_amd64.go index 43f026fbe..eda46025d 100644 --- a/internal/encoder/pools_amd64.go +++ b/internal/encoder/pools_amd64.go @@ -17,7 +17,6 @@ package encoder import ( - "errors" "reflect" "unsafe" @@ -52,29 +51,11 @@ var _KeepAlive struct { frame [x86.FP_offs]byte } -var errCallShadow = errors.New("DON'T CALL THIS!") - -// Faker func of _Encoder, used to export its stackmap as _Encoder's -func _Encoder_Shadow(rb *[]byte, vp unsafe.Pointer, sb *vars.Stack, fv uint64) (err error) { - // align to assembler_amd64.go: x86.FP_offs - var frame [x86.FP_offs]byte - - // must keep all args and frames noticeable to GC - _KeepAlive.rb = rb - _KeepAlive.vp = vp - _KeepAlive.sb = sb - _KeepAlive.fv = fv - _KeepAlive.err = err - _KeepAlive.frame = frame - - return errCallShadow -} - func makeEncoderX86(vt *rt.GoType, ex ...interface{}) (interface{}, error) { pp, err := NewCompiler().Compile(vt.Pack(), ex[0].(bool)) if err != nil { return nil, err - } + } as := x86.NewAssembler(pp) as.Name = vt.String() return as.Load(), nil diff --git a/internal/encoder/vars/errors.go b/internal/encoder/vars/errors.go index 77919c44a..ca3bbca1f 100644 --- a/internal/encoder/vars/errors.go +++ b/internal/encoder/vars/errors.go @@ -47,6 +47,10 @@ func Error_number(number json.Number) error { } } +func Error_unsuppoted(typ *rt.GoType) error { + return &json.UnsupportedTypeError{Type: typ.Pack() } +} + func Error_marshaler(ret []byte, pos int) error { return fmt.Errorf("invalid Marshaler output json syntax at %d: %q", pos, ret) } diff --git a/internal/encoder/vm/vm.go b/internal/encoder/vm/vm.go index b75ba807a..84a52cf97 100644 --- a/internal/encoder/vm/vm.go +++ b/internal/encoder/vm/vm.go @@ -338,6 +338,8 @@ func Execute(b *[]byte, p unsafe.Pointer, s *vars.Stack, flags uint64, prog *ir. if err := alg.EncodeJsonMarshaler(&buf, *(*json.Marshaler)(unsafe.Pointer(&it)), (flags)); err != nil { return err } + case ir.OP_unsupported: + return vars.Error_unsuppoted(ins.GoType()) default: panic(fmt.Sprintf("not implement %s at %d", ins.Op().String(), pc)) } @@ -347,13 +349,6 @@ func Execute(b *[]byte, p unsafe.Pointer, s *vars.Stack, flags uint64, prog *ir. return nil } -// func to_buf(w unsafe.Pointer, l int, c int) []byte { -// return rt.BytesFrom(unsafe.Pointer(uintptr(w)-uintptr(l)), l, c) -// } - -// func from_buf(buf []byte) (unsafe.Pointer, int, int) { -// return rt.IndexByte(buf, len(buf)), len(buf), cap(buf) -// } func has_opts(opts uint64, bit int) bool { return opts & (1< 0 && args[0].(bool) { + spew.Dump(string(jout), jerr) + } } diff --git a/issue_test/issue491_test.go b/issue_test/issue491_test.go new file mode 100644 index 000000000..3d1fb65ac --- /dev/null +++ b/issue_test/issue491_test.go @@ -0,0 +1,89 @@ +package issue_test + +import ( + "testing" + + "github.com/bytedance/sonic" +) + +type Function = func() + +type Unsupported struct { + Functions []Function +} + +type StructWithUnsupported struct { + Foo *Unsupported `json:"foo"` + Bar *Unsupported `json:"bar,omitempty"` +} + +type Foo2 struct { + A int + B *chan int +} + +type MockContext struct { + *Foo2 +} + +func TestIssue491_MarshalUnsupportedType(t *testing.T) { + // Wrapper a unbale serde type + tests := []interface{} { + map[string]*Function{}, + map[*Function]*Function{}, + map[string]Function{}, + []Function{}, + StructWithUnsupported{}, + struct { + Foo *int + }{}, + struct { + Foo Function + }{}, + chan int(nil), + new(MockContext), + } + for _, v := range(tests) { + assertMarshal(t, sonic.ConfigDefault, v) + } +} + + func TestIssue491_UnmarshalUnsupported(t *testing.T) { + type Test struct { + data string + value interface{} + } + + tests := []unmTestCase{ + { + name: "unsupported type slice", + data: []byte("null"), + newfn: func() interface{} { return new([]Function)}, + }, + { + name: "unsupported type", + data: []byte("[null, null]"), + newfn: func() interface{} { return new([]chan int) }, + }, + { + name: "unsupported type in struct", + data: []byte("{\"foo\": null}"), + newfn: func() interface{} { + return new(struct { + Foo Function + }) + }, + }, + { + name: "unsupported type in map key should be error", + data: []byte("null"), + newfn: func() interface{} { + return map[chan int]Function{} + }, + }, + } + for _, v := range(tests) { + assertUnmarshal(t, sonic.ConfigDefault, v) + } + } + \ No newline at end of file diff --git a/issue_test/issue755_test.go b/issue_test/issue755_test.go index e2364c80e..2934ae7a6 100644 --- a/issue_test/issue755_test.go +++ b/issue_test/issue755_test.go @@ -5,6 +5,8 @@ import ( "github.com/bytedance/sonic" ) +var _emptyFunc func() + func TestIssue755_NilEfaceWithDirectValue(t *testing.T) { tests := []interface{} { struct { @@ -14,6 +16,7 @@ func TestIssue755_NilEfaceWithDirectValue(t *testing.T) { Foo func() }{}, chan int(nil), + _emptyFunc, } for _, v := range(tests) { assertMarshal(t, sonic.ConfigDefault, v)