From 01ec106f0060815c7117cabd3f077723fdc77eb0 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Sat, 28 Sep 2024 19:44:34 +0200 Subject: [PATCH 1/3] Add native json.Number support Allow encoding and decoding of `json.Number` values, either as struct members or as interface members. Numbers will be encoded as integer, if possible, otherwise float64 is used. To cover the zero values, write an empty string, but invalid values will return errors when encoded It is possible to encode as string with `//msgp:replace json.Number with:string`. Fixes #292 --- _generated/def.go | 7 +++ _generated/def_test.go | 105 +++++++++++++++++++++++++++++++++++++ _generated/replace.go | 10 ++++ _generated/replace_test.go | 73 ++++++++++++++++++++++++++ gen/elem.go | 20 +++++-- gen/marshal.go | 2 +- msgp/read.go | 39 ++++++++++++++ msgp/read_bytes.go | 27 ++++++++++ msgp/size.go | 9 ++-- msgp/write.go | 20 +++++++ msgp/write_bytes.go | 21 ++++++++ 11 files changed, 323 insertions(+), 10 deletions(-) diff --git a/_generated/def.go b/_generated/def.go index a3d97b9e..4b06f071 100644 --- a/_generated/def.go +++ b/_generated/def.go @@ -1,6 +1,7 @@ package _generated import ( + "encoding/json" "os" "time" @@ -299,3 +300,9 @@ type StructByteSlice struct { AComplex128 []complex128 `msg:",allownil"` AStruct []Fixed `msg:",allownil"` } + +type NumberJSONSample struct { + Single json.Number + Array []json.Number + Map map[string]json.Number +} diff --git a/_generated/def_test.go b/_generated/def_test.go index 5d2e80ff..ad2b5fc9 100644 --- a/_generated/def_test.go +++ b/_generated/def_test.go @@ -2,6 +2,7 @@ package _generated import ( "bytes" + "encoding/json" "reflect" "testing" @@ -74,3 +75,107 @@ func TestRuneMarshalUnmarshal(t *testing.T) { t.Errorf("rune slice mismatch") } } + +func TestJSONNumber(t *testing.T) { + test := NumberJSONSample{ + Single: "-42", + Array: []json.Number{"0", "-0", "1", "-1", "0.1", "-0.1", "1234", "-1234", "12.34", "-12.34", "12E0", "12E1", "12e34", "12E-0", "12e+1", "12e-34", "-12E0", "-12E1", "-12e34", "-12E-0", "-12e+1", "-12e-34", "1.2E0", "1.2E1", "1.2e34", "1.2E-0", "1.2e+1", "1.2e-34", "-1.2E0", "-1.2E1", "-1.2e34", "-1.2E-0", "-1.2e+1", "-1.2e-34", "0E0", "0E1", "0e34", "0E-0", "0e+1", "0e-34", "-0E0", "-0E1", "-0e34", "-0E-0", "-0e+1", "-0e-34"}, + Map: map[string]json.Number{ + "a": json.Number("50.2"), + }, + } + + // This is not guaranteed to be symmetric + encoded, err := test.MarshalMsg(nil) + if err != nil { + t.Errorf("%v", err) + } + var v NumberJSONSample + _, err = v.UnmarshalMsg(encoded) + if err != nil { + t.Errorf("%v", err) + } + // Test two values + if v.Single != "-42" { + t.Errorf("want %v, got %v", "-42", v.Single) + } + if v.Map["a"] != "50.2" { + t.Errorf("want %v, got %v", "50.2", v.Map["a"]) + } + + var jsBuf bytes.Buffer + remain, err := msgp.UnmarshalAsJSON(&jsBuf, encoded) + if err != nil { + t.Errorf("%v", err) + } + if len(remain) != 0 { + t.Errorf("remain should be empty") + } + wantjs := `{"Single":-42,"Array":[0,0,1,-1,0.1,-0.1,1234,-1234,12.34,-12.34,12,120,120000000000000000000000000000000000,12,120,0.0000000000000000000000000000000012,-12,-120,-120000000000000000000000000000000000,-12,-120,-0.0000000000000000000000000000000012,1.2,12,12000000000000000000000000000000000,1.2,12,0.00000000000000000000000000000000012,-1.2,-12,-12000000000000000000000000000000000,-1.2,-12,-0.00000000000000000000000000000000012,0,0,0,0,0,0,-0,-0,-0,-0,-0,-0],"Map":{"a":50.2}}` + if jsBuf.String() != wantjs { + t.Errorf("jsBuf.String() = \n%s, want \n%s", jsBuf.String(), wantjs) + } + // Test encoding + var buf bytes.Buffer + en := msgp.NewWriter(&buf) + err = test.EncodeMsg(en) + if err != nil { + t.Errorf("%v", err) + } + en.Flush() + encoded = buf.Bytes() + + dc := msgp.NewReader(&buf) + err = v.DecodeMsg(dc) + if err != nil { + t.Errorf("%v", err) + } + // Test two values + if v.Single != "-42" { + t.Errorf("want %s, got %s", "-42", v.Single) + } + if v.Map["a"] != "50.2" { + t.Errorf("want %s, got %s", "50.2", v.Map["a"]) + } + + jsBuf.Reset() + remain, err = msgp.UnmarshalAsJSON(&jsBuf, encoded) + if err != nil { + t.Errorf("%v", err) + } + if len(remain) != 0 { + t.Errorf("remain should be empty") + } + if jsBuf.String() != wantjs { + t.Errorf("jsBuf.String() = \n%s, want \n%s", jsBuf.String(), wantjs) + } + + // Try interface encoder + jd := json.NewDecoder(&jsBuf) + jd.UseNumber() + var jsIntf map[string]any + err = jd.Decode(&jsIntf) + if err != nil { + t.Errorf("%v", err) + } + // Ensure we encode correctly + _ = (jsIntf["Single"]).(json.Number) + + fromInt, err := msgp.AppendIntf(nil, jsIntf) + if err != nil { + t.Errorf("%v", err) + } + + // Take the value from the JSON interface encoder and unmarshal back into our struct. + v = NumberJSONSample{} + _, err = v.UnmarshalMsg(fromInt) + if err != nil { + t.Errorf("%v", err) + } + if v.Single != "-42" { + t.Errorf("want %s, got %s", "-42", v.Single) + } + if v.Map["a"] != "50.2" { + t.Errorf("want %s, got %s", "50.2", v.Map["a"]) + } +} diff --git a/_generated/replace.go b/_generated/replace.go index c41b8610..121cf2cc 100644 --- a/_generated/replace.go +++ b/_generated/replace.go @@ -1,5 +1,7 @@ package _generated +import "encoding/json" + //go:generate msgp //msgp:replace Any with:any //msgp:replace MapString with:CompatibleMapString @@ -74,3 +76,11 @@ type ( String String } ) + +//msgp:replace json.Number with:string + +type NumberJSONSampleReplace struct { + Single json.Number + Array []json.Number + Map map[string]json.Number +} diff --git a/_generated/replace_test.go b/_generated/replace_test.go index 6764b388..31d26d71 100644 --- a/_generated/replace_test.go +++ b/_generated/replace_test.go @@ -1,8 +1,13 @@ package _generated import ( + "bytes" + "encoding/json" + "reflect" "testing" "time" + + "github.com/tinylib/msgp/msgp" ) func compareStructD(t *testing.T, a, b *CompatibleStructD) { @@ -288,3 +293,71 @@ func TestReplace_Dummy(t *testing.T) { t.Fatal("not same string") } } + +func TestJSONNumberReplace(t *testing.T) { + test := NumberJSONSampleReplace{ + Single: "-42", + Array: []json.Number{"0", "-0", "1", "-1", "0.1", "-0.1", "1234", "-1234", "12.34", "-12.34", "12E0", "12E1", "12e34", "12E-0", "12e+1", "12e-34", "-12E0", "-12E1", "-12e34", "-12E-0", "-12e+1", "-12e-34", "1.2E0", "1.2E1", "1.2e34", "1.2E-0", "1.2e+1", "1.2e-34", "-1.2E0", "-1.2E1", "-1.2e34", "-1.2E-0", "-1.2e+1", "-1.2e-34", "0E0", "0E1", "0e34", "0E-0", "0e+1", "0e-34", "-0E0", "-0E1", "-0e34", "-0E-0", "-0e+1", "-0e-34"}, + Map: map[string]json.Number{ + "a": json.Number("50"), + }, + } + + encoded, err := test.MarshalMsg(nil) + if err != nil { + t.Errorf("%v", err) + } + var v NumberJSONSampleReplace + _, err = v.UnmarshalMsg(encoded) + if err != nil { + t.Errorf("%v", err) + } + // Symmetric since we store strings. + if !reflect.DeepEqual(v, test) { + t.Fatalf("want %v, got %v", test, v) + } + + var jsBuf bytes.Buffer + remain, err := msgp.UnmarshalAsJSON(&jsBuf, encoded) + if err != nil { + t.Errorf("%v", err) + } + if len(remain) != 0 { + t.Errorf("remain should be empty") + } + // Retains number formatting. Map order is random, though. + wantjs := `{"Single":"-42","Array":["0","-0","1","-1","0.1","-0.1","1234","-1234","12.34","-12.34","12E0","12E1","12e34","12E-0","12e+1","12e-34","-12E0","-12E1","-12e34","-12E-0","-12e+1","-12e-34","1.2E0","1.2E1","1.2e34","1.2E-0","1.2e+1","1.2e-34","-1.2E0","-1.2E1","-1.2e34","-1.2E-0","-1.2e+1","-1.2e-34","0E0","0E1","0e34","0E-0","0e+1","0e-34","-0E0","-0E1","-0e34","-0E-0","-0e+1","-0e-34"],"Map":{"a":"50"}}` + if jsBuf.String() != wantjs { + t.Errorf("jsBuf.String() = \n%s, want \n%s", jsBuf.String(), wantjs) + } + // Test encoding + var buf bytes.Buffer + en := msgp.NewWriter(&buf) + err = test.EncodeMsg(en) + if err != nil { + t.Errorf("%v", err) + } + en.Flush() + encoded = buf.Bytes() + + dc := msgp.NewReader(&buf) + err = v.DecodeMsg(dc) + if err != nil { + t.Errorf("%v", err) + } + if !reflect.DeepEqual(v, test) { + t.Fatalf("want %v, got %v", test, v) + } + + jsBuf.Reset() + remain, err = msgp.UnmarshalAsJSON(&jsBuf, encoded) + if err != nil { + t.Errorf("%v", err) + } + if len(remain) != 0 { + t.Errorf("remain should be empty") + } + if jsBuf.String() != wantjs { + t.Errorf("jsBuf.String() = \n%s, want \n%s", jsBuf.String(), wantjs) + } +} diff --git a/gen/elem.go b/gen/elem.go index 3ace7764..50a5187a 100644 --- a/gen/elem.go +++ b/gen/elem.go @@ -88,10 +88,11 @@ const ( Int32 Int64 Bool - Intf // interface{} - Time // time.Time - Duration // time.Duration - Ext // extension + Intf // interface{} + Time // time.Time + Duration // time.Duration + Ext // extension + JsonNumber // json.Number IDENT // IDENT means an unrecognized identifier ) @@ -123,6 +124,7 @@ var primitives = map[string]Primitive{ "time.Time": Time, "time.Duration": Duration, "msgp.Extension": Ext, + "json.Number": JsonNumber, } // types built into the library @@ -634,6 +636,9 @@ func (s *BaseElem) BaseName() string { if s.Value == Duration { return "Duration" } + if s.Value == JsonNumber { + return "JSONNumber" + } return s.Value.String() } @@ -652,6 +657,8 @@ func (s *BaseElem) BaseType() string { return "time.Time" case Duration: return "time.Duration" + case JsonNumber: + return "json.Number" case Ext: return "msgp.Extension" @@ -719,9 +726,10 @@ func (s *BaseElem) ZeroExpr() string { return "0" case Bool: return "false" - case Time: return "(time.Time{})" + case JsonNumber: + return "(json.Number{})" } @@ -783,6 +791,8 @@ func (k Primitive) String() string { return "time.Duration" case Ext: return "Extension" + case JsonNumber: + return "json.Number" case IDENT: return "Ident" default: diff --git a/gen/marshal.go b/gen/marshal.go index 5b94ff39..370e5e7f 100644 --- a/gen/marshal.go +++ b/gen/marshal.go @@ -304,7 +304,7 @@ func (m *marshalGen) gBase(b *BaseElem) { case IDENT: echeck = true m.p.printf("\no, err = %s.MarshalMsg(o)", vname) - case Intf, Ext: + case Intf, Ext, JsonNumber: echeck = true m.p.printf("\no, err = msgp.Append%s(o, %s)", b.BaseName(), vname) default: diff --git a/msgp/read.go b/msgp/read.go index 31628782..b95fdef1 100644 --- a/msgp/read.go +++ b/msgp/read.go @@ -1,8 +1,10 @@ package msgp import ( + "encoding/json" "io" "math" + "strconv" "sync" "time" @@ -45,6 +47,7 @@ const ( Complex64Type Complex128Type TimeType + NumberType _maxtype ) @@ -74,6 +77,8 @@ func (t Type) String() string { return "ext" case NilType: return "nil" + case NumberType: + return "number" default: return "" } @@ -1276,6 +1281,40 @@ func (m *Reader) ReadTime() (t time.Time, err error) { return } +// ReadJSONNumber reads an integer or a float value and return as json.Number +func (m *Reader) ReadJSONNumber() (n json.Number, err error) { + t, err := m.NextType() + if err != nil { + return + } + switch t { + case IntType: + v, err := m.ReadInt64() + if err == nil { + return json.Number(strconv.FormatInt(v, 10)), nil + } + return "", err + case UintType: + v, err := m.ReadUint64() + if err == nil { + return json.Number(strconv.FormatUint(v, 10)), nil + } + return "", err + case Float32Type, Float64Type: + v, err := m.ReadFloat64() + if err == nil { + return json.Number(strconv.FormatFloat(v, 'f', -1, 64)), nil + } + return "", err + case StrType: + v, err := m.ReadString() + if err == nil { + return json.Number(v), nil + } + } + return "", TypeError{Method: NumberType, Encoded: t} +} + // ReadIntf reads out the next object as a raw interface{}/any. // Arrays are decoded as []interface{}, and maps are decoded // as map[string]interface{}. Integers are decoded as int64 diff --git a/msgp/read_bytes.go b/msgp/read_bytes.go index 05ca3bc7..bb8abdc0 100644 --- a/msgp/read_bytes.go +++ b/msgp/read_bytes.go @@ -3,7 +3,9 @@ package msgp import ( "bytes" "encoding/binary" + "encoding/json" "math" + "strconv" "time" ) @@ -1327,3 +1329,28 @@ func getSize(b []byte) (uintptr, uintptr, error) { return 0, 0, fatal } } + +// ReadJSONNumberBytes tries to read a number +// from 'b' and return the value and the remaining bytes. +// +// Possible errors: +// +// - [ErrShortBytes] (too few bytes) +// - TypeError (not a number (int/float)) +func ReadJSONNumberBytes(b []byte) (number json.Number, o []byte, err error) { + if len(b) < 1 { + return "", nil, ErrShortBytes + } + if i, o, err := ReadInt64Bytes(b); err == nil { + return json.Number(strconv.FormatInt(i, 10)), o, nil + } + f, o, err := ReadFloat64Bytes(b) + if err == nil { + return json.Number(strconv.FormatFloat(f, 'f', -1, 64)), o, nil + } + s, o, err := ReadStringBytes(b) + if err == nil { + return json.Number(s), o, nil + } + return "", nil, TypeError{Method: NumberType, Encoded: getType(b[0])} +} diff --git a/msgp/size.go b/msgp/size.go index e3a613b2..585a67fd 100644 --- a/msgp/size.go +++ b/msgp/size.go @@ -25,10 +25,11 @@ const ( Complex64Size = 10 Complex128Size = 18 - DurationSize = Int64Size - TimeSize = 15 - BoolSize = 1 - NilSize = 1 + DurationSize = Int64Size + TimeSize = 15 + BoolSize = 1 + NilSize = 1 + JSONNumberSize = Int64Size // Same as Float64Size MapHeaderSize = 5 ArrayHeaderSize = 5 diff --git a/msgp/write.go b/msgp/write.go index ec2f6f52..97671487 100644 --- a/msgp/write.go +++ b/msgp/write.go @@ -1,6 +1,7 @@ package msgp import ( + "encoding/json" "errors" "io" "math" @@ -624,6 +625,23 @@ func (mw *Writer) WriteTime(t time.Time) error { return nil } +// WriteJSONNumber writes the json.Number to the stream as either integer or float. +func (mw *Writer) WriteJSONNumber(n json.Number) error { + if n == "" { + // Allow writing the empty string to cover the zero value. + return mw.push(wfixstr(uint8(0))) + } + ii, err := n.Int64() + if err == nil { + return mw.WriteInt64(ii) + } + ff, err := n.Float64() + if err == nil { + return mw.WriteFloat64(ff) + } + return err +} + // WriteIntf writes the concrete type of 'v'. // WriteIntf will error if 'v' is not one of the following: // - A bool, float, string, []byte, int, uint, or complex @@ -689,6 +707,8 @@ func (mw *Writer) WriteIntf(v interface{}) error { return mw.WriteTime(v) case time.Duration: return mw.WriteDuration(v) + case json.Number: + return mw.WriteJSONNumber(v) } val := reflect.ValueOf(v) diff --git a/msgp/write_bytes.go b/msgp/write_bytes.go index 12606cc2..2b054a2d 100644 --- a/msgp/write_bytes.go +++ b/msgp/write_bytes.go @@ -1,6 +1,7 @@ package msgp import ( + "encoding/json" "errors" "math" "reflect" @@ -402,6 +403,8 @@ func AppendIntf(b []byte, i interface{}) ([]byte, error) { return AppendMapStrIntf(b, i) case map[string]string: return AppendMapStrStr(b, i), nil + case json.Number: + return AppendJSONNumber(b, i) case []interface{}: b = AppendArrayHeader(b, uint32(len(i))) var err error @@ -452,3 +455,21 @@ func AppendIntf(b []byte, i interface{}) ([]byte, error) { return b, &ErrUnsupportedType{T: v.Type()} } } + +// AppendJSONNumber appends a json.Number to the slice. +// An error will be returned if the json.Number returns error as both integer and float. +func AppendJSONNumber(b []byte, n json.Number) ([]byte, error) { + if n == "" { + // Allow writing the empty string to cover the zero value. + return append(b, mfixstr), nil + } + ii, err := n.Int64() + if err == nil { + return AppendInt64(b, ii), nil + } + ff, err := n.Float64() + if err == nil { + return AppendFloat64(b, ff), nil + } + return b, err +} From eaf6fc46519786e7e063836b63fd18f8c1ab2bc5 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Mon, 30 Sep 2024 20:57:26 +0200 Subject: [PATCH 2/3] Use `0` for zero value json.Number and check against empty string --- _generated/def.go | 1 + gen/elem.go | 2 +- msgp/read.go | 5 ----- msgp/read_bytes.go | 4 ---- msgp/write.go | 4 ++-- msgp/write_bytes.go | 4 ++-- 6 files changed, 6 insertions(+), 14 deletions(-) diff --git a/_generated/def.go b/_generated/def.go index 4b06f071..dc3ec3f4 100644 --- a/_generated/def.go +++ b/_generated/def.go @@ -305,4 +305,5 @@ type NumberJSONSample struct { Single json.Number Array []json.Number Map map[string]json.Number + OE json.Number `msg:",omitempty"` } diff --git a/gen/elem.go b/gen/elem.go index 50a5187a..1fd26ff7 100644 --- a/gen/elem.go +++ b/gen/elem.go @@ -729,7 +729,7 @@ func (s *BaseElem) ZeroExpr() string { case Time: return "(time.Time{})" case JsonNumber: - return "(json.Number{})" + return `""` } diff --git a/msgp/read.go b/msgp/read.go index b95fdef1..5eb0b107 100644 --- a/msgp/read.go +++ b/msgp/read.go @@ -1306,11 +1306,6 @@ func (m *Reader) ReadJSONNumber() (n json.Number, err error) { return json.Number(strconv.FormatFloat(v, 'f', -1, 64)), nil } return "", err - case StrType: - v, err := m.ReadString() - if err == nil { - return json.Number(v), nil - } } return "", TypeError{Method: NumberType, Encoded: t} } diff --git a/msgp/read_bytes.go b/msgp/read_bytes.go index bb8abdc0..cd20e97f 100644 --- a/msgp/read_bytes.go +++ b/msgp/read_bytes.go @@ -1348,9 +1348,5 @@ func ReadJSONNumberBytes(b []byte) (number json.Number, o []byte, err error) { if err == nil { return json.Number(strconv.FormatFloat(f, 'f', -1, 64)), o, nil } - s, o, err := ReadStringBytes(b) - if err == nil { - return json.Number(s), o, nil - } return "", nil, TypeError{Method: NumberType, Encoded: getType(b[0])} } diff --git a/msgp/write.go b/msgp/write.go index dd6aa680..d177dce5 100644 --- a/msgp/write.go +++ b/msgp/write.go @@ -638,8 +638,8 @@ func (mw *Writer) WriteTime(t time.Time) error { // WriteJSONNumber writes the json.Number to the stream as either integer or float. func (mw *Writer) WriteJSONNumber(n json.Number) error { if n == "" { - // Allow writing the empty string to cover the zero value. - return mw.push(wfixstr(uint8(0))) + // The zero value outputs the 0 integer. + return mw.push(0) } ii, err := n.Int64() if err == nil { diff --git a/msgp/write_bytes.go b/msgp/write_bytes.go index 29ee37e2..6e1683bd 100644 --- a/msgp/write_bytes.go +++ b/msgp/write_bytes.go @@ -470,8 +470,8 @@ func AppendIntf(b []byte, i interface{}) ([]byte, error) { // An error will be returned if the json.Number returns error as both integer and float. func AppendJSONNumber(b []byte, n json.Number) ([]byte, error) { if n == "" { - // Allow writing the empty string to cover the zero value. - return append(b, mfixstr), nil + // The zero value outputs the 0 integer. + return append(b, 0), nil } ii, err := n.Int64() if err == nil { From 276217712d464e679e3b3537170b438da38e3b96 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Mon, 30 Sep 2024 21:02:04 +0200 Subject: [PATCH 3/3] Use the brand new float writer to write floats32 if lossless. --- msgp/write.go | 2 +- msgp/write_bytes.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/msgp/write.go b/msgp/write.go index d177dce5..dfe0d3e8 100644 --- a/msgp/write.go +++ b/msgp/write.go @@ -647,7 +647,7 @@ func (mw *Writer) WriteJSONNumber(n json.Number) error { } ff, err := n.Float64() if err == nil { - return mw.WriteFloat64(ff) + return mw.WriteFloat(ff) } return err } diff --git a/msgp/write_bytes.go b/msgp/write_bytes.go index 6e1683bd..a95b1d0b 100644 --- a/msgp/write_bytes.go +++ b/msgp/write_bytes.go @@ -479,7 +479,7 @@ func AppendJSONNumber(b []byte, n json.Number) ([]byte, error) { } ff, err := n.Float64() if err == nil { - return AppendFloat64(b, ff), nil + return AppendFloat(b, ff), nil } return b, err }