From 19ae0ccfd889a027e0e5083cef4540d031c248b8 Mon Sep 17 00:00:00 2001 From: muXxer Date: Fri, 3 Nov 2023 20:23:46 +0100 Subject: [PATCH 01/17] Add proper map serialization support in serix --- serializer/serializer.go | 2 +- serializer/serix/decode.go | 37 +++- serializer/serix/encode.go | 60 +++++- serializer/serix/serix.go | 388 +++++++++++++++++++++++++++++++------ 4 files changed, 416 insertions(+), 71 deletions(-) diff --git a/serializer/serializer.go b/serializer/serializer.go index 14643d1c8..251de369f 100644 --- a/serializer/serializer.go +++ b/serializer/serializer.go @@ -268,7 +268,7 @@ func (s *Serializer) WriteVariableByteSlice(data []byte, lenType SeriLengthPrefi return s case minLen > 0 && sliceLen < minLen: - s.err = errProducer(ierrors.Wrapf(ErrSliceLengthTooShort, "slice (len %d) is less than min length of %d ", sliceLen, maxLen)) + s.err = errProducer(ierrors.Wrapf(ErrSliceLengthTooShort, "slice (len %d) is less than min length of %d ", sliceLen, minLen)) return s } diff --git a/serializer/serix/decode.go b/serializer/serix/decode.go index f4191a13b..4f8957c84 100644 --- a/serializer/serix/decode.go +++ b/serializer/serix/decode.go @@ -408,20 +408,40 @@ func (api *API) decodeMap(ctx context.Context, b []byte, value reflect.Value, if value.IsNil() { value.Set(reflect.MakeMap(valueType)) } + deserializeItem := func(b []byte) (bytesRead int, err error) { keyValue := reflect.New(valueType.Key()).Elem() elemValue := reflect.New(valueType.Elem()).Elem() - bytesRead, err = api.decodeMapKVPair(ctx, b, keyValue, elemValue, opts) + bytesRead, err = api.decodeMapKVPair(ctx, b, keyValue, elemValue, ts, opts) if err != nil { return 0, ierrors.WithStack(err) } + + if value.MapIndex(keyValue).IsValid() { + // map entry already exists + return 0, ierrors.Wrapf(ErrMapValidationViolatesUniqueness, "map entry with key %v already exists", keyValue.Interface()) + } + value.SetMapIndex(keyValue, elemValue) return bytesRead, nil } ts = ts.ensureOrdering() - return api.decodeSequence(b, deserializeItem, valueType, ts, opts) + consumedBytes, err := api.decodeSequence(b, deserializeItem, valueType, ts, opts) + if err != nil { + return consumedBytes, err + } + + if err := api.checkMapMinMaxBounds(value.Len(), ts); err != nil { + return consumedBytes, err + } + + if err := api.checkMapMaxByteSize(consumedBytes, ts); err != nil { + return consumedBytes, err + } + + return consumedBytes, nil } func (api *API) decodeSequence(b []byte, deserializeItem serializer.DeserializeFunc, valueType reflect.Type, ts TypeSettings, opts *options) (int, error) { @@ -448,13 +468,20 @@ func (api *API) decodeSequence(b []byte, deserializeItem serializer.DeserializeF return deseri.Done() } -func (api *API) decodeMapKVPair(ctx context.Context, b []byte, key, val reflect.Value, opts *options) (int, error) { - keyBytesRead, err := api.decode(ctx, b, key, TypeSettings{}, opts) +func (api *API) decodeMapKVPair(ctx context.Context, b []byte, key, val reflect.Value, ts TypeSettings, opts *options) (int, error) { + keyTypeSettings := TypeSettings{} + valueTypeSettings := TypeSettings{} + if ts.mapRules != nil { + keyTypeSettings = ts.mapRules.KeyRules.ToTypeSettings() + valueTypeSettings = ts.mapRules.ValueRules.ToTypeSettings() + } + + keyBytesRead, err := api.decode(ctx, b, key, keyTypeSettings, opts) if err != nil { return 0, ierrors.Wrapf(err, "failed to decode map key of type %s", key.Type()) } b = b[keyBytesRead:] - elemBytesRead, err := api.decode(ctx, b, val, TypeSettings{}, opts) + elemBytesRead, err := api.decode(ctx, b, val, valueTypeSettings, opts) if err != nil { return 0, ierrors.Wrapf(err, "failed to decode map element of type %s", val.Type()) } diff --git a/serializer/serix/encode.go b/serializer/serix/encode.go index 73ea92d34..1b47f873e 100644 --- a/serializer/serix/encode.go +++ b/serializer/serix/encode.go @@ -164,6 +164,30 @@ func (api *API) checkMinMaxBounds(v reflect.Value, ts TypeSettings) error { return nil } +// checkMapMinMaxBounds checks whether the given map is within its defined bounds in case it has defined map rules. +func (api *API) checkMapMinMaxBounds(length int, ts TypeSettings) error { + if ts.mapRules != nil { + switch { + case ts.mapRules.MaxEntries > 0 && uint(length) > ts.mapRules.MaxEntries: + return ierrors.Wrapf(ErrMapValidationMaxElementsExceeded, "map (len %d) exceeds max length of %d ", length, ts.mapRules.MaxEntries) + + case ts.mapRules.MinEntries > 0 && uint(length) < ts.mapRules.MinEntries: + return ierrors.Wrapf(ErrMapValidationMinElementsNotReached, "map (len %d) is less than min length of %d ", length, ts.mapRules.MinEntries) + } + } + + return nil +} + +// checkMapMaxByteSize checks whether the given map is within its defined max byte size in case it has defined map rules. +func (api *API) checkMapMaxByteSize(byteSize int, ts TypeSettings) error { + if ts.mapRules != nil && ts.mapRules.MaxByteSize > 0 && byteSize > int(ts.mapRules.MaxByteSize) { + return ierrors.Wrapf(ErrMapValidationMaxBytesExceeded, "map (len %d) exceeds max bytes of %d ", byteSize, ts.mapRules.MaxByteSize) + } + + return nil +} + // checks whether the given value has the concept of a length. func hasLength(v reflect.Value) bool { k := v.Kind() @@ -355,12 +379,17 @@ func (api *API) encodeSlice(ctx context.Context, value reflect.Value, valueType func (api *API) encodeMap(ctx context.Context, value reflect.Value, valueType reflect.Type, ts TypeSettings, opts *options) ([]byte, error) { size := value.Len() + + if err := api.checkMapMinMaxBounds(size, ts); err != nil { + return nil, err + } + data := make([][]byte, size) iter := value.MapRange() for i := 0; iter.Next(); i++ { key := iter.Key() elem := iter.Value() - b, err := api.encodeMapKVPair(ctx, key, elem, opts) + b, err := api.encodeMapKVPair(ctx, key, elem, ts, opts) if err != nil { return nil, ierrors.WithStack(err) } @@ -368,18 +397,36 @@ func (api *API) encodeMap(ctx context.Context, value reflect.Value, valueType re } ts = ts.ensureOrdering() - return encodeSliceOfBytes(data, valueType, ts, opts) + bytes, err := encodeSliceOfBytes(data, valueType, ts, opts) + if err != nil { + return nil, err + } + + if err := api.checkMapMaxByteSize(len(bytes), ts); err != nil { + return nil, err + } + + return bytes, nil } -func (api *API) encodeMapKVPair(ctx context.Context, key, val reflect.Value, opts *options) ([]byte, error) { - keyBytes, err := api.encode(ctx, key, TypeSettings{}, opts) +func (api *API) encodeMapKVPair(ctx context.Context, key, val reflect.Value, ts TypeSettings, opts *options) ([]byte, error) { + keyTypeSettings := TypeSettings{} + valueTypeSettings := TypeSettings{} + if ts.mapRules != nil { + keyTypeSettings = ts.mapRules.KeyRules.ToTypeSettings() + valueTypeSettings = ts.mapRules.ValueRules.ToTypeSettings() + } + + keyBytes, err := api.encode(ctx, key, keyTypeSettings, opts) if err != nil { return nil, ierrors.Wrapf(err, "failed to encode map key of type %s", key.Type()) } - elemBytes, err := api.encode(ctx, val, TypeSettings{}, opts) + + elemBytes, err := api.encode(ctx, val, valueTypeSettings, opts) if err != nil { return nil, ierrors.Wrapf(err, "failed to encode map element of type %s", val.Type()) } + buf := bytes.NewBuffer(keyBytes) buf.Write(elemBytes) @@ -391,13 +438,16 @@ func encodeSliceOfBytes(data [][]byte, valueType reflect.Type, ts TypeSettings, if !set { return nil, ierrors.Errorf("no LengthPrefixType was provided for type %s", valueType) } + arrayRules := ts.ArrayRules() if arrayRules == nil { arrayRules = new(ArrayRules) } + serializationMode := ts.toMode(opts) serializerArrayRules := serializer.ArrayRules(*arrayRules) serializerArrayRulesPtr := &serializerArrayRules + seri := serializer.NewSerializer() seri.WriteSliceOfByteSlices(data, serializationMode, diff --git a/serializer/serix/serix.go b/serializer/serix/serix.go index 55fc613d7..1746d4706 100644 --- a/serializer/serix/serix.go +++ b/serializer/serix/serix.go @@ -30,6 +30,17 @@ import ( "github.com/iotaledger/hive.go/serializer/v2" ) +var ( + // ErrMapValidationMinElementsNotReached gets returned if the count of elements is too small. + ErrMapValidationMinElementsNotReached = ierrors.New("min count of elements within the map not reached") + // ErrMapValidationMaxElementsExceeded gets returned if the count of elements is too big. + ErrMapValidationMaxElementsExceeded = ierrors.New("max count of elements within the map exceeded") + // ErrMapValidationMaxBytesExceeded gets returned if the serialized byte size of the map is too big. + ErrMapValidationMaxBytesExceeded = ierrors.New("max bytes size of the map exceeded") + // ErrMapValidationViolatesUniqueness gets returned if the map elements are not unique. + ErrMapValidationViolatesUniqueness = ierrors.New("map elements must be unique") +) + // Serializable is a type that can serialize itself. // Serix will call its .Encode() method instead of trying to serialize it in the default way. // The behavior is totally the same as in the standard "encoding/json" package and json.Marshaler interface. @@ -149,10 +160,6 @@ func (o *options) toMode() serializer.DeSerializationMode { return mode } -// ArrayRules defines rules around a to be deserialized array. -// Min and Max at 0 define an unbounded array. -type ArrayRules serializer.ArrayRules - // LengthPrefixType defines the type of the value denoting the length of a collection. type LengthPrefixType serializer.SeriLengthPrefixType @@ -165,6 +172,43 @@ const ( LengthPrefixTypeAsUint32 = LengthPrefixType(serializer.SeriLengthPrefixTypeAsUint32) ) +// ArrayRules defines rules around a to be deserialized array. +// Min and Max at 0 define an unbounded array. +type ArrayRules serializer.ArrayRules + +// MapElementRules defines rules around to be deserialized map elements (key or value). +// MinLength and MaxLength at 0 define an unbounded map element. +type MapElementRules struct { + LengthPrefixType *LengthPrefixType + MinLength uint + MaxLength uint +} + +func (m *MapElementRules) ToTypeSettings() TypeSettings { + return TypeSettings{ + lengthPrefixType: m.LengthPrefixType, + arrayRules: &ArrayRules{ + Min: m.MinLength, + Max: m.MaxLength, + }, + } +} + +// MapRules defines rules around a to be deserialized map. +type MapRules struct { + // MinEntries defines the min entries for the map. + MinEntries uint + // MaxEntries defines the max entries for the map. 0 means unbounded. + MaxEntries uint + // MaxByteSize defines the max serialized byte size for the map. 0 means unbounded. + MaxByteSize uint + + // KeyRules define the rules applied to the keys of the map. + KeyRules *MapElementRules + // ValueRules define the rules applied to the values of the map. + ValueRules *MapElementRules +} + // TypeSettings holds various settings for a particular type. // Those settings determine how the object should be serialized/deserialized. // There are three ways to provide TypeSettings @@ -180,6 +224,7 @@ type TypeSettings struct { lexicalOrdering *bool mapKey *string arrayRules *ArrayRules + mapRules *MapRules } // WithLengthPrefixType specifies LengthPrefixType. @@ -198,6 +243,37 @@ func (ts TypeSettings) LengthPrefixType() (LengthPrefixType, bool) { return *ts.lengthPrefixType, true } +// WithObjectType specifies the object type. It can be either uint8 or uint32 number. +// The object type holds two meanings: the actual code (number) and the serializer.TypeDenotationType like uint8 or uint32. +// serix uses object type to actually encode the number +// and to know its serializer.TypeDenotationType to be able to decode it. +func (ts TypeSettings) WithObjectType(t interface{}) TypeSettings { + ts.objectType = t + + return ts +} + +// ObjectType returns the object type as an uint8 or uint32 number. +func (ts TypeSettings) ObjectType() interface{} { + return ts.objectType +} + +// WithLexicalOrdering specifies whether the type must be lexically ordered during serialization. +func (ts TypeSettings) WithLexicalOrdering(val bool) TypeSettings { + ts.lexicalOrdering = &val + + return ts +} + +// LexicalOrdering returns lexical ordering flag. +func (ts TypeSettings) LexicalOrdering() (val bool, set bool) { + if ts.lexicalOrdering == nil { + return false, false + } + + return *ts.lexicalOrdering, true +} + // WithMapKey specifies the name for the map key. func (ts TypeSettings) WithMapKey(name string) TypeSettings { ts.mapKey = &name @@ -223,6 +299,18 @@ func (ts TypeSettings) MustMapKey() string { return *ts.mapKey } +// WithArrayRules specifies serializer.ArrayRules. +func (ts TypeSettings) WithArrayRules(rules *ArrayRules) TypeSettings { + ts.arrayRules = rules + + return ts +} + +// ArrayRules returns serializer.ArrayRules. +func (ts TypeSettings) ArrayRules() *ArrayRules { + return ts.arrayRules +} + // WithMinLen specifies the min length for the object. func (ts TypeSettings) WithMinLen(l uint) TypeSettings { if ts.arrayRules == nil { @@ -275,47 +363,124 @@ func (ts TypeSettings) MinMaxLen() (int, int) { return min, max } -// WithObjectType specifies the object type. It can be either uint8 or uint32 number. -// The object type holds two meanings: the actual code (number) and the serializer.TypeDenotationType like uint8 or uint32. -// serix uses object type to actually encode the number -// and to know its serializer.TypeDenotationType to be able to decode it. -func (ts TypeSettings) WithObjectType(t interface{}) TypeSettings { - ts.objectType = t +// WithMapRules specifies the map rules. +func (ts TypeSettings) WithMapRules(rules *MapRules) TypeSettings { + ts.mapRules = rules return ts } -// ObjectType returns the object type as an uint8 or uint32 number. -func (ts TypeSettings) ObjectType() interface{} { - return ts.objectType +// MapRules returns the map rules. +func (ts TypeSettings) MapRules() *MapRules { + return ts.mapRules } -// WithLexicalOrdering specifies whether the type must be lexically ordered during serialization. -func (ts TypeSettings) WithLexicalOrdering(val bool) TypeSettings { - ts.lexicalOrdering = &val +// WithMapMinEntries specifies the min entries for the map. +func (ts TypeSettings) WithMapMinEntries(l uint) TypeSettings { + if ts.mapRules == nil { + ts.mapRules = new(MapRules) + } + ts.mapRules.MinEntries = l return ts } -// LexicalOrdering returns lexical ordering flag. -func (ts TypeSettings) LexicalOrdering() (val bool, set bool) { - if ts.lexicalOrdering == nil { - return false, false +// WithMapMaxEntries specifies the max entries for the map. +func (ts TypeSettings) WithMapMaxEntries(l uint) TypeSettings { + if ts.mapRules == nil { + ts.mapRules = new(MapRules) } + ts.mapRules.MaxEntries = l - return *ts.lexicalOrdering, true + return ts } -// WithArrayRules specifies serializer.ArrayRules. -func (ts TypeSettings) WithArrayRules(rules *ArrayRules) TypeSettings { - ts.arrayRules = rules +// WithMapMaxByteSize specifies max serialized byte size for the map. 0 means unbounded. +func (ts TypeSettings) WithMapMaxByteSize(l uint) TypeSettings { + if ts.mapRules == nil { + ts.mapRules = new(MapRules) + } + ts.mapRules.MaxByteSize = l return ts } -// ArrayRules returns serializer.ArrayRules. -func (ts TypeSettings) ArrayRules() *ArrayRules { - return ts.arrayRules +// WithMapKeyLengthPrefixType specifies MapKeyLengthPrefixType. +func (ts TypeSettings) WithMapKeyLengthPrefixType(lpt LengthPrefixType) TypeSettings { + if ts.mapRules == nil { + ts.mapRules = new(MapRules) + } + if ts.mapRules.KeyRules == nil { + ts.mapRules.KeyRules = new(MapElementRules) + } + ts.mapRules.KeyRules.LengthPrefixType = &lpt + + return ts +} + +// WithMapKeyMinLen specifies the min length for the object in the map key. +func (ts TypeSettings) WithMapKeyMinLen(l uint) TypeSettings { + if ts.mapRules == nil { + ts.mapRules = new(MapRules) + } + if ts.mapRules.KeyRules == nil { + ts.mapRules.KeyRules = new(MapElementRules) + } + ts.mapRules.KeyRules.MinLength = l + + return ts +} + +// WithMapKeyMaxLen specifies the max length for the object in the map key. +func (ts TypeSettings) WithMapKeyMaxLen(l uint) TypeSettings { + if ts.mapRules == nil { + ts.mapRules = new(MapRules) + } + if ts.mapRules.KeyRules == nil { + ts.mapRules.KeyRules = new(MapElementRules) + } + ts.mapRules.KeyRules.MaxLength = l + + return ts +} + +// MapValueLengthPrefixType specifies MapValueLengthPrefixType. +func (ts TypeSettings) WithMapValueLengthPrefixType(lpt LengthPrefixType) TypeSettings { + if ts.mapRules == nil { + ts.mapRules = new(MapRules) + } + if ts.mapRules.ValueRules == nil { + ts.mapRules.ValueRules = new(MapElementRules) + } + ts.mapRules.ValueRules.LengthPrefixType = &lpt + + return ts +} + +// WithMapValueMinLen specifies the min length for the object in the map value. +func (ts TypeSettings) WithMapValueMinLen(l uint) TypeSettings { + if ts.mapRules == nil { + ts.mapRules = new(MapRules) + } + if ts.mapRules.ValueRules == nil { + ts.mapRules.ValueRules = new(MapElementRules) + } + ts.mapRules.ValueRules.MinLength = l + + return ts +} + +// WithMapValueMaxLen specifies the max length for the object in the map value. +func (ts TypeSettings) WithMapValueMaxLen(l uint) TypeSettings { + if ts.mapRules == nil { + ts.mapRules = new(MapRules) + } + if ts.mapRules.ValueRules == nil { + ts.mapRules.ValueRules = new(MapElementRules) + } + ts.mapRules.ValueRules.MaxLength = l + + return ts } func (ts TypeSettings) ensureOrdering() TypeSettings { @@ -346,6 +511,9 @@ func (ts TypeSettings) merge(other TypeSettings) TypeSettings { if ts.mapKey == nil { ts.mapKey = other.mapKey } + if ts.mapRules == nil { + ts.mapRules = other.mapRules + } return ts } @@ -886,6 +1054,55 @@ func (api *API) parseStructType(structType reflect.Type) ([]structField, error) return structFields, nil } +func parseStructTagValue(name string, keyValue []string, currentPart string) (string, error) { + if len(keyValue) != 2 { + return "", ierrors.Errorf("incorrect %s tag format: %s", name, currentPart) + } + + return keyValue[1], nil +} + +func parseStructTagValueUint(name string, keyValue []string, currentPart string) (uint, error) { + value, err := parseStructTagValue(name, keyValue, currentPart) + if err != nil { + return 0, err + } + + result, err := strconv.ParseUint(value, 10, 64) + if err != nil { + return 0, ierrors.Wrapf(err, "failed to parse %s %s", name, currentPart) + } + + return uint(result), nil +} + +func parseLengthPrefixType(prefixTypeRaw string) (LengthPrefixType, error) { + switch prefixTypeRaw { + case "byte", "uint8": + return LengthPrefixTypeAsByte, nil + case "uint16": + return LengthPrefixTypeAsUint16, nil + case "uint32": + return LengthPrefixTypeAsUint32, nil + default: + return LengthPrefixTypeAsByte, ierrors.Errorf("unknown length prefix type: %s", prefixTypeRaw) + } +} + +func parseStructTagValuePrefixType(name string, keyValue []string, currentPart string) (LengthPrefixType, error) { + value, err := parseStructTagValue(name, keyValue, currentPart) + if err != nil { + return 0, err + } + + lengthPrefixType, err := parseLengthPrefixType(value) + if err != nil { + return 0, ierrors.Wrapf(err, "failed to parse %s %s", name, currentPart) + } + + return lengthPrefixType, nil +} + func parseStructTag(tag string) (tagSettings, error) { if tag == "" { return tagSettings{}, ierrors.New("struct tag is empty") @@ -900,51 +1117,115 @@ func parseStructTag(tag string) (tagSettings, error) { settings.position = position parts = parts[1:] seenParts := map[string]struct{}{} + for _, currentPart := range parts { if _, ok := seenParts[currentPart]; ok { return tagSettings{}, ierrors.Errorf("duplicated tag part: %s", currentPart) } keyValue := strings.Split(currentPart, "=") partName := keyValue[0] + switch partName { case "optional": settings.isOptional = true + case "nest": settings.nest = true + case "omitempty": settings.omitEmpty = true + case "mapKey": - if len(keyValue) != 2 { - return tagSettings{}, ierrors.Errorf("incorrect mapKey tag format: %s", currentPart) + value, err := parseStructTagValue("mapKey", keyValue, currentPart) + if err != nil { + return tagSettings{}, err } - settings.ts = settings.ts.WithMapKey(keyValue[1]) + settings.ts = settings.ts.WithMapKey(value) + case "minLen": - if len(keyValue) != 2 { - return tagSettings{}, ierrors.Errorf("incorrect minLen tag format: %s", currentPart) - } - minLen, err := strconv.ParseUint(keyValue[1], 10, 64) + value, err := parseStructTagValueUint("minLen", keyValue, currentPart) if err != nil { - return tagSettings{}, ierrors.Wrapf(err, "failed to parse minLen %s", currentPart) + return tagSettings{}, err } - settings.ts = settings.ts.WithMinLen(uint(minLen)) + settings.ts = settings.ts.WithMinLen(value) + case "maxLen": - if len(keyValue) != 2 { - return tagSettings{}, ierrors.Errorf("incorrect maxLen tag format: %s", currentPart) - } - maxLen, err := strconv.ParseUint(keyValue[1], 10, 64) + value, err := parseStructTagValueUint("maxLen", keyValue, currentPart) if err != nil { - return tagSettings{}, ierrors.Wrapf(err, "failed to parse maxLen %s", currentPart) + return tagSettings{}, err } - settings.ts = settings.ts.WithMaxLen(uint(maxLen)) + settings.ts = settings.ts.WithMaxLen(value) + case "lengthPrefixType": - if len(keyValue) != 2 { - return tagSettings{}, ierrors.Errorf("incorrect lengthPrefixType tag format: %s", currentPart) + value, err := parseStructTagValuePrefixType("lengthPrefixType", keyValue, currentPart) + if err != nil { + return tagSettings{}, err + } + settings.ts = settings.ts.WithLengthPrefixType(value) + + case "mapMinEntries": + value, err := parseStructTagValueUint("mapMinEntries", keyValue, currentPart) + if err != nil { + return tagSettings{}, err + } + settings.ts = settings.ts.WithMapMinEntries(value) + + case "mapMaxEntries": + value, err := parseStructTagValueUint("mapMaxEntries", keyValue, currentPart) + if err != nil { + return tagSettings{}, err + } + settings.ts = settings.ts.WithMapMaxEntries(value) + + case "mapMaxByteSize": + value, err := parseStructTagValueUint("mapMaxByteSize", keyValue, currentPart) + if err != nil { + return tagSettings{}, err + } + settings.ts = settings.ts.WithMapMaxByteSize(value) + + case "mapKeyLengthPrefixType": + value, err := parseStructTagValuePrefixType("mapKeyLengthPrefixType", keyValue, currentPart) + if err != nil { + return tagSettings{}, err + } + settings.ts = settings.ts.WithMapKeyLengthPrefixType(value) + + case "mapKeyMinLen": + value, err := parseStructTagValueUint("mapKeyMinLen", keyValue, currentPart) + if err != nil { + return tagSettings{}, err } - lengthPrefixType, err := parseLengthPrefixType(keyValue[1]) + settings.ts = settings.ts.WithMapKeyMinLen(value) + + case "mapKeyMaxLen": + value, err := parseStructTagValueUint("mapKeyMaxLen", keyValue, currentPart) if err != nil { - return tagSettings{}, ierrors.Wrapf(err, "failed to parse lengthPrefixType %s", currentPart) + return tagSettings{}, err } - settings.ts = settings.ts.WithLengthPrefixType(lengthPrefixType) + settings.ts = settings.ts.WithMapKeyMaxLen(value) + + case "mapValueLengthPrefixType": + value, err := parseStructTagValuePrefixType("mapValueLengthPrefixType", keyValue, currentPart) + if err != nil { + return tagSettings{}, err + } + settings.ts = settings.ts.WithMapValueLengthPrefixType(value) + + case "mapValueMinLen": + value, err := parseStructTagValueUint("mapValueMinLen", keyValue, currentPart) + if err != nil { + return tagSettings{}, err + } + settings.ts = settings.ts.WithMapValueMinLen(value) + + case "mapValueMaxLen": + value, err := parseStructTagValueUint("mapValueMaxLen", keyValue, currentPart) + if err != nil { + return tagSettings{}, err + } + settings.ts = settings.ts.WithMapValueMaxLen(value) + default: return tagSettings{}, ierrors.Errorf("unknown tag part: %s", currentPart) } @@ -954,19 +1235,6 @@ func parseStructTag(tag string) (tagSettings, error) { return settings, nil } -func parseLengthPrefixType(prefixTypeRaw string) (LengthPrefixType, error) { - switch prefixTypeRaw { - case "byte", "uint8": - return LengthPrefixTypeAsByte, nil - case "uint16": - return LengthPrefixTypeAsUint16, nil - case "uint32": - return LengthPrefixTypeAsUint32, nil - default: - return LengthPrefixTypeAsByte, ierrors.Errorf("unknown length prefix type: %s", prefixTypeRaw) - } -} - func sliceFromArray(arrValue reflect.Value) reflect.Value { arrType := arrValue.Type() sliceType := reflect.SliceOf(arrType.Elem()) From 81c90139c63988379cb76c71c20065f2732ae5f1 Mon Sep 17 00:00:00 2001 From: muXxer Date: Mon, 6 Nov 2023 14:36:48 +0100 Subject: [PATCH 02/17] Move TypeSettings to own file --- serializer/serix/serix.go | 434 +++--------------------------- serializer/serix/type_settings.go | 373 +++++++++++++++++++++++++ 2 files changed, 406 insertions(+), 401 deletions(-) create mode 100644 serializer/serix/type_settings.go diff --git a/serializer/serix/serix.go b/serializer/serix/serix.go index 1746d4706..60b71c521 100644 --- a/serializer/serix/serix.go +++ b/serializer/serix/serix.go @@ -41,6 +41,17 @@ var ( ErrMapValidationViolatesUniqueness = ierrors.New("map elements must be unique") ) +var ( + bytesType = reflect.TypeOf([]byte(nil)) + bigIntPtrType = reflect.TypeOf((*big.Int)(nil)) + timeType = reflect.TypeOf(time.Time{}) + errorType = reflect.TypeOf((*error)(nil)).Elem() + ctxType = reflect.TypeOf((*context.Context)(nil)).Elem() +) + +// DefaultAPI is the default instance of the API type. +var DefaultAPI = NewAPI() + // Serializable is a type that can serialize itself. // Serix will call its .Encode() method instead of trying to serialize it in the default way. // The behavior is totally the same as in the standard "encoding/json" package and json.Marshaler interface. @@ -74,25 +85,6 @@ type DeserializableJSON interface { DecodeJSON(b any) error } -// API is the main object of the package that provides the methods for client to use. -// It holds all the settings and configuration. It also stores the cache. -// Most often you will need a single object of API for the whole program. -// You register all type settings and interfaces on the program start or in init() function. -// Instead of creating a new API object you can also use the default singleton API object: DefaultAPI. -type API struct { - interfacesRegistryMutex sync.RWMutex - interfacesRegistry map[reflect.Type]*interfaceObjects - - typeSettingsRegistryMutex sync.RWMutex - typeSettingsRegistry map[reflect.Type]TypeSettings - - validatorsRegistryMutex sync.RWMutex - validatorsRegistry map[reflect.Type]validators - - typeCacheMutex sync.RWMutex - typeCache map[reflect.Type][]structField -} - type validators struct { bytesValidator reflect.Value syntacticValidator reflect.Value @@ -104,29 +96,6 @@ type interfaceObjects struct { typeDenotation serializer.TypeDenotationType } -// DefaultAPI is the default instance of the API type. -var DefaultAPI = NewAPI() - -// NewAPI creates a new instance of the API type. -func NewAPI() *API { - api := &API{ - interfacesRegistry: map[reflect.Type]*interfaceObjects{}, - typeSettingsRegistry: map[reflect.Type]TypeSettings{}, - validatorsRegistry: map[reflect.Type]validators{}, - typeCache: map[reflect.Type][]structField{}, - } - - return api -} - -var ( - bytesType = reflect.TypeOf([]byte(nil)) - bigIntPtrType = reflect.TypeOf((*big.Int)(nil)) - timeType = reflect.TypeOf(time.Time{}) - errorType = reflect.TypeOf((*error)(nil)).Elem() - ctxType = reflect.TypeOf((*context.Context)(nil)).Elem() -) - // Option is an option for Encode/Decode methods. type Option func(o *options) @@ -160,372 +129,35 @@ func (o *options) toMode() serializer.DeSerializationMode { return mode } -// LengthPrefixType defines the type of the value denoting the length of a collection. -type LengthPrefixType serializer.SeriLengthPrefixType - -const ( - // LengthPrefixTypeAsByte defines a collection length to be denoted by a byte. - LengthPrefixTypeAsByte = LengthPrefixType(serializer.SeriLengthPrefixTypeAsByte) - // LengthPrefixTypeAsUint16 defines a collection length to be denoted by a uint16. - LengthPrefixTypeAsUint16 = LengthPrefixType(serializer.SeriLengthPrefixTypeAsUint16) - // LengthPrefixTypeAsUint32 defines a collection length to be denoted by a uint32. - LengthPrefixTypeAsUint32 = LengthPrefixType(serializer.SeriLengthPrefixTypeAsUint32) -) - -// ArrayRules defines rules around a to be deserialized array. -// Min and Max at 0 define an unbounded array. -type ArrayRules serializer.ArrayRules - -// MapElementRules defines rules around to be deserialized map elements (key or value). -// MinLength and MaxLength at 0 define an unbounded map element. -type MapElementRules struct { - LengthPrefixType *LengthPrefixType - MinLength uint - MaxLength uint -} - -func (m *MapElementRules) ToTypeSettings() TypeSettings { - return TypeSettings{ - lengthPrefixType: m.LengthPrefixType, - arrayRules: &ArrayRules{ - Min: m.MinLength, - Max: m.MaxLength, - }, - } -} - -// MapRules defines rules around a to be deserialized map. -type MapRules struct { - // MinEntries defines the min entries for the map. - MinEntries uint - // MaxEntries defines the max entries for the map. 0 means unbounded. - MaxEntries uint - // MaxByteSize defines the max serialized byte size for the map. 0 means unbounded. - MaxByteSize uint - - // KeyRules define the rules applied to the keys of the map. - KeyRules *MapElementRules - // ValueRules define the rules applied to the values of the map. - ValueRules *MapElementRules -} - -// TypeSettings holds various settings for a particular type. -// Those settings determine how the object should be serialized/deserialized. -// There are three ways to provide TypeSettings -// 1. Via global registry: API.RegisterTypeSettings(). -// 2. Parse from struct tags. -// 3. Pass as an option to API.Encode/API.Decode methods. -// The type settings provided via struct tags or an option override the type settings from the registry. -// So the precedence is the following 1<2<3. -// See API.RegisterTypeSettings() and WithTypeSettings() for more detail. -type TypeSettings struct { - lengthPrefixType *LengthPrefixType - objectType interface{} - lexicalOrdering *bool - mapKey *string - arrayRules *ArrayRules - mapRules *MapRules -} - -// WithLengthPrefixType specifies LengthPrefixType. -func (ts TypeSettings) WithLengthPrefixType(lpt LengthPrefixType) TypeSettings { - ts.lengthPrefixType = &lpt - - return ts -} - -// LengthPrefixType returns LengthPrefixType. -func (ts TypeSettings) LengthPrefixType() (LengthPrefixType, bool) { - if ts.lengthPrefixType == nil { - return 0, false - } - - return *ts.lengthPrefixType, true -} - -// WithObjectType specifies the object type. It can be either uint8 or uint32 number. -// The object type holds two meanings: the actual code (number) and the serializer.TypeDenotationType like uint8 or uint32. -// serix uses object type to actually encode the number -// and to know its serializer.TypeDenotationType to be able to decode it. -func (ts TypeSettings) WithObjectType(t interface{}) TypeSettings { - ts.objectType = t - - return ts -} - -// ObjectType returns the object type as an uint8 or uint32 number. -func (ts TypeSettings) ObjectType() interface{} { - return ts.objectType -} - -// WithLexicalOrdering specifies whether the type must be lexically ordered during serialization. -func (ts TypeSettings) WithLexicalOrdering(val bool) TypeSettings { - ts.lexicalOrdering = &val - - return ts -} - -// LexicalOrdering returns lexical ordering flag. -func (ts TypeSettings) LexicalOrdering() (val bool, set bool) { - if ts.lexicalOrdering == nil { - return false, false - } - - return *ts.lexicalOrdering, true -} - -// WithMapKey specifies the name for the map key. -func (ts TypeSettings) WithMapKey(name string) TypeSettings { - ts.mapKey = &name - - return ts -} - -// MapKey returns the map key name. -func (ts TypeSettings) MapKey() (string, bool) { - if ts.mapKey == nil { - return "", false - } - - return *ts.mapKey, true -} - -// MustMapKey must return a map key name. -func (ts TypeSettings) MustMapKey() string { - if ts.mapKey == nil { - panic("no map key set") - } - - return *ts.mapKey -} - -// WithArrayRules specifies serializer.ArrayRules. -func (ts TypeSettings) WithArrayRules(rules *ArrayRules) TypeSettings { - ts.arrayRules = rules - - return ts -} - -// ArrayRules returns serializer.ArrayRules. -func (ts TypeSettings) ArrayRules() *ArrayRules { - return ts.arrayRules -} - -// WithMinLen specifies the min length for the object. -func (ts TypeSettings) WithMinLen(l uint) TypeSettings { - if ts.arrayRules == nil { - ts.arrayRules = new(ArrayRules) - } - ts.arrayRules.Min = l - - return ts -} - -// MinLen returns min length for the object. -func (ts TypeSettings) MinLen() (uint, bool) { - if ts.arrayRules == nil || ts.arrayRules.Min == 0 { - return 0, false - } - - return ts.arrayRules.Min, true -} - -// WithMaxLen specifies the max length for the object. -func (ts TypeSettings) WithMaxLen(l uint) TypeSettings { - if ts.arrayRules == nil { - ts.arrayRules = new(ArrayRules) - } - ts.arrayRules.Max = l - - return ts -} - -// MaxLen returns max length for the object. -func (ts TypeSettings) MaxLen() (uint, bool) { - if ts.arrayRules == nil || ts.arrayRules.Max == 0 { - return 0, false - } - - return ts.arrayRules.Max, true -} - -// MinMaxLen returns min/max lengths for the object. -// Returns 0 for either value if they are not set. -func (ts TypeSettings) MinMaxLen() (int, int) { - var min, max int - if ts.arrayRules != nil { - min = int(ts.arrayRules.Min) - } - if ts.arrayRules != nil { - max = int(ts.arrayRules.Max) - } - - return min, max -} - -// WithMapRules specifies the map rules. -func (ts TypeSettings) WithMapRules(rules *MapRules) TypeSettings { - ts.mapRules = rules - - return ts -} - -// MapRules returns the map rules. -func (ts TypeSettings) MapRules() *MapRules { - return ts.mapRules -} - -// WithMapMinEntries specifies the min entries for the map. -func (ts TypeSettings) WithMapMinEntries(l uint) TypeSettings { - if ts.mapRules == nil { - ts.mapRules = new(MapRules) - } - ts.mapRules.MinEntries = l - - return ts -} - -// WithMapMaxEntries specifies the max entries for the map. -func (ts TypeSettings) WithMapMaxEntries(l uint) TypeSettings { - if ts.mapRules == nil { - ts.mapRules = new(MapRules) - } - ts.mapRules.MaxEntries = l - - return ts -} - -// WithMapMaxByteSize specifies max serialized byte size for the map. 0 means unbounded. -func (ts TypeSettings) WithMapMaxByteSize(l uint) TypeSettings { - if ts.mapRules == nil { - ts.mapRules = new(MapRules) - } - ts.mapRules.MaxByteSize = l - - return ts -} - -// WithMapKeyLengthPrefixType specifies MapKeyLengthPrefixType. -func (ts TypeSettings) WithMapKeyLengthPrefixType(lpt LengthPrefixType) TypeSettings { - if ts.mapRules == nil { - ts.mapRules = new(MapRules) - } - if ts.mapRules.KeyRules == nil { - ts.mapRules.KeyRules = new(MapElementRules) - } - ts.mapRules.KeyRules.LengthPrefixType = &lpt - - return ts -} - -// WithMapKeyMinLen specifies the min length for the object in the map key. -func (ts TypeSettings) WithMapKeyMinLen(l uint) TypeSettings { - if ts.mapRules == nil { - ts.mapRules = new(MapRules) - } - if ts.mapRules.KeyRules == nil { - ts.mapRules.KeyRules = new(MapElementRules) - } - ts.mapRules.KeyRules.MinLength = l - - return ts -} - -// WithMapKeyMaxLen specifies the max length for the object in the map key. -func (ts TypeSettings) WithMapKeyMaxLen(l uint) TypeSettings { - if ts.mapRules == nil { - ts.mapRules = new(MapRules) - } - if ts.mapRules.KeyRules == nil { - ts.mapRules.KeyRules = new(MapElementRules) - } - ts.mapRules.KeyRules.MaxLength = l - - return ts -} - -// MapValueLengthPrefixType specifies MapValueLengthPrefixType. -func (ts TypeSettings) WithMapValueLengthPrefixType(lpt LengthPrefixType) TypeSettings { - if ts.mapRules == nil { - ts.mapRules = new(MapRules) - } - if ts.mapRules.ValueRules == nil { - ts.mapRules.ValueRules = new(MapElementRules) - } - ts.mapRules.ValueRules.LengthPrefixType = &lpt - - return ts -} - -// WithMapValueMinLen specifies the min length for the object in the map value. -func (ts TypeSettings) WithMapValueMinLen(l uint) TypeSettings { - if ts.mapRules == nil { - ts.mapRules = new(MapRules) - } - if ts.mapRules.ValueRules == nil { - ts.mapRules.ValueRules = new(MapElementRules) - } - ts.mapRules.ValueRules.MinLength = l - - return ts -} - -// WithMapValueMaxLen specifies the max length for the object in the map value. -func (ts TypeSettings) WithMapValueMaxLen(l uint) TypeSettings { - if ts.mapRules == nil { - ts.mapRules = new(MapRules) - } - if ts.mapRules.ValueRules == nil { - ts.mapRules.ValueRules = new(MapElementRules) - } - ts.mapRules.ValueRules.MaxLength = l - - return ts -} - -func (ts TypeSettings) ensureOrdering() TypeSettings { - newTS := ts.WithLexicalOrdering(true) - arrayRules := newTS.ArrayRules() - newArrayRules := new(ArrayRules) - if arrayRules != nil { - *newArrayRules = *arrayRules - } - newArrayRules.ValidationMode |= serializer.ArrayValidationModeLexicalOrdering +// API is the main object of the package that provides the methods for client to use. +// It holds all the settings and configuration. It also stores the cache. +// Most often you will need a single object of API for the whole program. +// You register all type settings and interfaces on the program start or in init() function. +// Instead of creating a new API object you can also use the default singleton API object: DefaultAPI. +type API struct { + interfacesRegistryMutex sync.RWMutex + interfacesRegistry map[reflect.Type]*interfaceObjects - return newTS.WithArrayRules(newArrayRules) -} + typeSettingsRegistryMutex sync.RWMutex + typeSettingsRegistry map[reflect.Type]TypeSettings -func (ts TypeSettings) merge(other TypeSettings) TypeSettings { - if ts.lengthPrefixType == nil { - ts.lengthPrefixType = other.lengthPrefixType - } - if ts.objectType == nil { - ts.objectType = other.objectType - } - if ts.lexicalOrdering == nil { - ts.lexicalOrdering = other.lexicalOrdering - } - if ts.arrayRules == nil { - ts.arrayRules = other.arrayRules - } - if ts.mapKey == nil { - ts.mapKey = other.mapKey - } - if ts.mapRules == nil { - ts.mapRules = other.mapRules - } + validatorsRegistryMutex sync.RWMutex + validatorsRegistry map[reflect.Type]validators - return ts + typeCacheMutex sync.RWMutex + typeCache map[reflect.Type][]structField } -func (ts TypeSettings) toMode(opts *options) serializer.DeSerializationMode { - mode := opts.toMode() - lexicalOrdering, set := ts.LexicalOrdering() - if set && lexicalOrdering { - mode |= serializer.DeSeriModePerformLexicalOrdering +// NewAPI creates a new instance of the API type. +func NewAPI() *API { + api := &API{ + interfacesRegistry: map[reflect.Type]*interfaceObjects{}, + typeSettingsRegistry: map[reflect.Type]TypeSettings{}, + validatorsRegistry: map[reflect.Type]validators{}, + typeCache: map[reflect.Type][]structField{}, } - return mode + return api } // Encode serializes the provided object obj into bytes. diff --git a/serializer/serix/type_settings.go b/serializer/serix/type_settings.go new file mode 100644 index 000000000..ee0cdf683 --- /dev/null +++ b/serializer/serix/type_settings.go @@ -0,0 +1,373 @@ +package serix + +import ( + "github.com/iotaledger/hive.go/serializer/v2" +) + +// LengthPrefixType defines the type of the value denoting the length of a collection. +type LengthPrefixType serializer.SeriLengthPrefixType + +const ( + // LengthPrefixTypeAsByte defines a collection length to be denoted by a byte. + LengthPrefixTypeAsByte = LengthPrefixType(serializer.SeriLengthPrefixTypeAsByte) + // LengthPrefixTypeAsUint16 defines a collection length to be denoted by a uint16. + LengthPrefixTypeAsUint16 = LengthPrefixType(serializer.SeriLengthPrefixTypeAsUint16) + // LengthPrefixTypeAsUint32 defines a collection length to be denoted by a uint32. + LengthPrefixTypeAsUint32 = LengthPrefixType(serializer.SeriLengthPrefixTypeAsUint32) +) + +// ArrayRules defines rules around a to be deserialized array. +// Min and Max at 0 define an unbounded array. +type ArrayRules serializer.ArrayRules + +// MapElementRules defines rules around to be deserialized map elements (key or value). +// MinLength and MaxLength at 0 define an unbounded map element. +type MapElementRules struct { + LengthPrefixType *LengthPrefixType + MinLength uint + MaxLength uint +} + +func (m *MapElementRules) ToTypeSettings() TypeSettings { + return TypeSettings{ + lengthPrefixType: m.LengthPrefixType, + arrayRules: &ArrayRules{ + Min: m.MinLength, + Max: m.MaxLength, + }, + } +} + +// MapRules defines rules around a to be deserialized map. +type MapRules struct { + // MinEntries defines the min entries for the map. + MinEntries uint + // MaxEntries defines the max entries for the map. 0 means unbounded. + MaxEntries uint + // MaxByteSize defines the max serialized byte size for the map. 0 means unbounded. + MaxByteSize uint + + // KeyRules define the rules applied to the keys of the map. + KeyRules *MapElementRules + // ValueRules define the rules applied to the values of the map. + ValueRules *MapElementRules +} + +// TypeSettings holds various settings for a particular type. +// Those settings determine how the object should be serialized/deserialized. +// There are three ways to provide TypeSettings +// 1. Via global registry: API.RegisterTypeSettings(). +// 2. Parse from struct tags. +// 3. Pass as an option to API.Encode/API.Decode methods. +// The type settings provided via struct tags or an option override the type settings from the registry. +// So the precedence is the following 1<2<3. +// See API.RegisterTypeSettings() and WithTypeSettings() for more detail. +type TypeSettings struct { + lengthPrefixType *LengthPrefixType + objectType interface{} + lexicalOrdering *bool + mapKey *string + arrayRules *ArrayRules + mapRules *MapRules +} + +// WithLengthPrefixType specifies LengthPrefixType. +func (ts TypeSettings) WithLengthPrefixType(lpt LengthPrefixType) TypeSettings { + ts.lengthPrefixType = &lpt + + return ts +} + +// LengthPrefixType returns LengthPrefixType. +func (ts TypeSettings) LengthPrefixType() (LengthPrefixType, bool) { + if ts.lengthPrefixType == nil { + return 0, false + } + + return *ts.lengthPrefixType, true +} + +// WithObjectType specifies the object type. It can be either uint8 or uint32 number. +// The object type holds two meanings: the actual code (number) and the serializer.TypeDenotationType like uint8 or uint32. +// serix uses object type to actually encode the number +// and to know its serializer.TypeDenotationType to be able to decode it. +func (ts TypeSettings) WithObjectType(t interface{}) TypeSettings { + ts.objectType = t + + return ts +} + +// ObjectType returns the object type as an uint8 or uint32 number. +func (ts TypeSettings) ObjectType() interface{} { + return ts.objectType +} + +// WithLexicalOrdering specifies whether the type must be lexically ordered during serialization. +func (ts TypeSettings) WithLexicalOrdering(val bool) TypeSettings { + ts.lexicalOrdering = &val + + return ts +} + +// LexicalOrdering returns lexical ordering flag. +func (ts TypeSettings) LexicalOrdering() (val bool, set bool) { + if ts.lexicalOrdering == nil { + return false, false + } + + return *ts.lexicalOrdering, true +} + +// WithMapKey specifies the name for the map key. +func (ts TypeSettings) WithMapKey(name string) TypeSettings { + ts.mapKey = &name + + return ts +} + +// MapKey returns the map key name. +func (ts TypeSettings) MapKey() (string, bool) { + if ts.mapKey == nil { + return "", false + } + + return *ts.mapKey, true +} + +// MustMapKey must return a map key name. +func (ts TypeSettings) MustMapKey() string { + if ts.mapKey == nil { + panic("no map key set") + } + + return *ts.mapKey +} + +// WithArrayRules specifies serializer.ArrayRules. +func (ts TypeSettings) WithArrayRules(rules *ArrayRules) TypeSettings { + ts.arrayRules = rules + + return ts +} + +// ArrayRules returns serializer.ArrayRules. +func (ts TypeSettings) ArrayRules() *ArrayRules { + return ts.arrayRules +} + +// WithMinLen specifies the min length for the object. +func (ts TypeSettings) WithMinLen(l uint) TypeSettings { + if ts.arrayRules == nil { + ts.arrayRules = new(ArrayRules) + } + ts.arrayRules.Min = l + + return ts +} + +// MinLen returns min length for the object. +func (ts TypeSettings) MinLen() (uint, bool) { + if ts.arrayRules == nil || ts.arrayRules.Min == 0 { + return 0, false + } + + return ts.arrayRules.Min, true +} + +// WithMaxLen specifies the max length for the object. +func (ts TypeSettings) WithMaxLen(l uint) TypeSettings { + if ts.arrayRules == nil { + ts.arrayRules = new(ArrayRules) + } + ts.arrayRules.Max = l + + return ts +} + +// MaxLen returns max length for the object. +func (ts TypeSettings) MaxLen() (uint, bool) { + if ts.arrayRules == nil || ts.arrayRules.Max == 0 { + return 0, false + } + + return ts.arrayRules.Max, true +} + +// MinMaxLen returns min/max lengths for the object. +// Returns 0 for either value if they are not set. +func (ts TypeSettings) MinMaxLen() (int, int) { + var min, max int + if ts.arrayRules != nil { + min = int(ts.arrayRules.Min) + } + if ts.arrayRules != nil { + max = int(ts.arrayRules.Max) + } + + return min, max +} + +// WithMapRules specifies the map rules. +func (ts TypeSettings) WithMapRules(rules *MapRules) TypeSettings { + ts.mapRules = rules + + return ts +} + +// MapRules returns the map rules. +func (ts TypeSettings) MapRules() *MapRules { + return ts.mapRules +} + +// WithMapMinEntries specifies the min entries for the map. +func (ts TypeSettings) WithMapMinEntries(l uint) TypeSettings { + if ts.mapRules == nil { + ts.mapRules = new(MapRules) + } + ts.mapRules.MinEntries = l + + return ts +} + +// WithMapMaxEntries specifies the max entries for the map. +func (ts TypeSettings) WithMapMaxEntries(l uint) TypeSettings { + if ts.mapRules == nil { + ts.mapRules = new(MapRules) + } + ts.mapRules.MaxEntries = l + + return ts +} + +// WithMapMaxByteSize specifies max serialized byte size for the map. 0 means unbounded. +func (ts TypeSettings) WithMapMaxByteSize(l uint) TypeSettings { + if ts.mapRules == nil { + ts.mapRules = new(MapRules) + } + ts.mapRules.MaxByteSize = l + + return ts +} + +// WithMapKeyLengthPrefixType specifies MapKeyLengthPrefixType. +func (ts TypeSettings) WithMapKeyLengthPrefixType(lpt LengthPrefixType) TypeSettings { + if ts.mapRules == nil { + ts.mapRules = new(MapRules) + } + if ts.mapRules.KeyRules == nil { + ts.mapRules.KeyRules = new(MapElementRules) + } + ts.mapRules.KeyRules.LengthPrefixType = &lpt + + return ts +} + +// WithMapKeyMinLen specifies the min length for the object in the map key. +func (ts TypeSettings) WithMapKeyMinLen(l uint) TypeSettings { + if ts.mapRules == nil { + ts.mapRules = new(MapRules) + } + if ts.mapRules.KeyRules == nil { + ts.mapRules.KeyRules = new(MapElementRules) + } + ts.mapRules.KeyRules.MinLength = l + + return ts +} + +// WithMapKeyMaxLen specifies the max length for the object in the map key. +func (ts TypeSettings) WithMapKeyMaxLen(l uint) TypeSettings { + if ts.mapRules == nil { + ts.mapRules = new(MapRules) + } + if ts.mapRules.KeyRules == nil { + ts.mapRules.KeyRules = new(MapElementRules) + } + ts.mapRules.KeyRules.MaxLength = l + + return ts +} + +// MapValueLengthPrefixType specifies MapValueLengthPrefixType. +func (ts TypeSettings) WithMapValueLengthPrefixType(lpt LengthPrefixType) TypeSettings { + if ts.mapRules == nil { + ts.mapRules = new(MapRules) + } + if ts.mapRules.ValueRules == nil { + ts.mapRules.ValueRules = new(MapElementRules) + } + ts.mapRules.ValueRules.LengthPrefixType = &lpt + + return ts +} + +// WithMapValueMinLen specifies the min length for the object in the map value. +func (ts TypeSettings) WithMapValueMinLen(l uint) TypeSettings { + if ts.mapRules == nil { + ts.mapRules = new(MapRules) + } + if ts.mapRules.ValueRules == nil { + ts.mapRules.ValueRules = new(MapElementRules) + } + ts.mapRules.ValueRules.MinLength = l + + return ts +} + +// WithMapValueMaxLen specifies the max length for the object in the map value. +func (ts TypeSettings) WithMapValueMaxLen(l uint) TypeSettings { + if ts.mapRules == nil { + ts.mapRules = new(MapRules) + } + if ts.mapRules.ValueRules == nil { + ts.mapRules.ValueRules = new(MapElementRules) + } + ts.mapRules.ValueRules.MaxLength = l + + return ts +} + +func (ts TypeSettings) ensureOrdering() TypeSettings { + newTS := ts.WithLexicalOrdering(true) + arrayRules := newTS.ArrayRules() + newArrayRules := new(ArrayRules) + if arrayRules != nil { + *newArrayRules = *arrayRules + } + newArrayRules.ValidationMode |= serializer.ArrayValidationModeLexicalOrdering + + return newTS.WithArrayRules(newArrayRules) +} + +func (ts TypeSettings) merge(other TypeSettings) TypeSettings { + if ts.lengthPrefixType == nil { + ts.lengthPrefixType = other.lengthPrefixType + } + if ts.objectType == nil { + ts.objectType = other.objectType + } + if ts.lexicalOrdering == nil { + ts.lexicalOrdering = other.lexicalOrdering + } + if ts.arrayRules == nil { + ts.arrayRules = other.arrayRules + } + if ts.mapKey == nil { + ts.mapKey = other.mapKey + } + if ts.mapRules == nil { + ts.mapRules = other.mapRules + } + + return ts +} + +func (ts TypeSettings) toMode(opts *options) serializer.DeSerializationMode { + mode := opts.toMode() + lexicalOrdering, set := ts.LexicalOrdering() + if set && lexicalOrdering { + mode |= serializer.DeSeriModePerformLexicalOrdering + } + + return mode +} From de6357f55db7d03725d675a55ef196640e528ee3 Mon Sep 17 00:00:00 2001 From: muXxer Date: Mon, 6 Nov 2023 14:37:42 +0100 Subject: [PATCH 03/17] Move common funcs from encode.go to serix.go --- serializer/serix/encode.go | 60 -------------------------------------- serializer/serix/serix.go | 60 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 60 deletions(-) diff --git a/serializer/serix/encode.go b/serializer/serix/encode.go index 1b47f873e..5bbdd7b49 100644 --- a/serializer/serix/encode.go +++ b/serializer/serix/encode.go @@ -143,66 +143,6 @@ func (api *API) encodeBasedOnType( return nil, ierrors.Errorf("can't encode: unsupported type %T", valueI) } -// checks whether the given value is within its defined bounds in case it has a length. -func (api *API) checkMinMaxBounds(v reflect.Value, ts TypeSettings) error { - if has := hasLength(v); !has { - return nil - } - - l := uint(v.Len()) - if minLen, ok := ts.MinLen(); ok { - if l < minLen { - return ierrors.Wrapf(serializer.ErrArrayValidationMinElementsNotReached, "can't serialize '%s' type: min length %d not reached (len %d)", v.Kind(), minLen, l) - } - } - if maxLen, ok := ts.MaxLen(); ok { - if l > maxLen { - return ierrors.Wrapf(serializer.ErrArrayValidationMaxElementsExceeded, "can't serialize '%s' type: max length %d exceeded (len %d)", v.Kind(), maxLen, l) - } - } - - return nil -} - -// checkMapMinMaxBounds checks whether the given map is within its defined bounds in case it has defined map rules. -func (api *API) checkMapMinMaxBounds(length int, ts TypeSettings) error { - if ts.mapRules != nil { - switch { - case ts.mapRules.MaxEntries > 0 && uint(length) > ts.mapRules.MaxEntries: - return ierrors.Wrapf(ErrMapValidationMaxElementsExceeded, "map (len %d) exceeds max length of %d ", length, ts.mapRules.MaxEntries) - - case ts.mapRules.MinEntries > 0 && uint(length) < ts.mapRules.MinEntries: - return ierrors.Wrapf(ErrMapValidationMinElementsNotReached, "map (len %d) is less than min length of %d ", length, ts.mapRules.MinEntries) - } - } - - return nil -} - -// checkMapMaxByteSize checks whether the given map is within its defined max byte size in case it has defined map rules. -func (api *API) checkMapMaxByteSize(byteSize int, ts TypeSettings) error { - if ts.mapRules != nil && ts.mapRules.MaxByteSize > 0 && byteSize > int(ts.mapRules.MaxByteSize) { - return ierrors.Wrapf(ErrMapValidationMaxBytesExceeded, "map (len %d) exceeds max bytes of %d ", byteSize, ts.mapRules.MaxByteSize) - } - - return nil -} - -// checks whether the given value has the concept of a length. -func hasLength(v reflect.Value) bool { - k := v.Kind() - switch k { - case reflect.Array: - case reflect.Map: - case reflect.Slice: - case reflect.String: - default: - return false - } - - return true -} - func (api *API) encodeInterface( ctx context.Context, value reflect.Value, valueType reflect.Type, ts TypeSettings, opts *options, ) ([]byte, error) { diff --git a/serializer/serix/serix.go b/serializer/serix/serix.go index 60b71c521..b10383114 100644 --- a/serializer/serix/serix.go +++ b/serializer/serix/serix.go @@ -160,6 +160,66 @@ func NewAPI() *API { return api } +// checks whether the given value has the concept of a length. +func hasLength(v reflect.Value) bool { + k := v.Kind() + switch k { + case reflect.Array: + case reflect.Map: + case reflect.Slice: + case reflect.String: + default: + return false + } + + return true +} + +// checks whether the given value is within its defined bounds in case it has a length. +func (api *API) checkMinMaxBounds(v reflect.Value, ts TypeSettings) error { + if has := hasLength(v); !has { + return nil + } + + l := uint(v.Len()) + if minLen, ok := ts.MinLen(); ok { + if l < minLen { + return ierrors.Wrapf(serializer.ErrArrayValidationMinElementsNotReached, "can't serialize '%s' type: min length %d not reached (len %d)", v.Kind(), minLen, l) + } + } + if maxLen, ok := ts.MaxLen(); ok { + if l > maxLen { + return ierrors.Wrapf(serializer.ErrArrayValidationMaxElementsExceeded, "can't serialize '%s' type: max length %d exceeded (len %d)", v.Kind(), maxLen, l) + } + } + + return nil +} + +// checkMapMinMaxBounds checks whether the given map is within its defined bounds in case it has defined map rules. +func (api *API) checkMapMinMaxBounds(length int, ts TypeSettings) error { + if ts.mapRules != nil { + switch { + case ts.mapRules.MaxEntries > 0 && uint(length) > ts.mapRules.MaxEntries: + return ierrors.Wrapf(ErrMapValidationMaxElementsExceeded, "map (len %d) exceeds max length of %d ", length, ts.mapRules.MaxEntries) + + case ts.mapRules.MinEntries > 0 && uint(length) < ts.mapRules.MinEntries: + return ierrors.Wrapf(ErrMapValidationMinElementsNotReached, "map (len %d) is less than min length of %d ", length, ts.mapRules.MinEntries) + } + } + + return nil +} + +// checkMapMaxByteSize checks whether the given map is within its defined max byte size in case it has defined map rules. +func (api *API) checkMapMaxByteSize(byteSize int, ts TypeSettings) error { + if ts.mapRules != nil && ts.mapRules.MaxByteSize > 0 && byteSize > int(ts.mapRules.MaxByteSize) { + return ierrors.Wrapf(ErrMapValidationMaxBytesExceeded, "map (len %d) exceeds max bytes of %d ", byteSize, ts.mapRules.MaxByteSize) + } + + return nil +} + // Encode serializes the provided object obj into bytes. // serix traverses the object recursively and serializes everything based on the type. // If a type implements the custom Serializable interface serix delegates the serialization to that type. From 111fd385daeb49928f5f9285bdeda0841ebf9c05 Mon Sep 17 00:00:00 2001 From: muXxer Date: Mon, 6 Nov 2023 15:01:58 +0100 Subject: [PATCH 04/17] Add map rules check to mapEncodeMap and mapDecodeMap --- serializer/serix/decode.go | 42 ++++++++++++++--------------- serializer/serix/encode.go | 48 +++++++++++++++++----------------- serializer/serix/map_decode.go | 34 +++++++++++++++++------- serializer/serix/map_encode.go | 48 ++++++++++++++++++++++------------ 4 files changed, 101 insertions(+), 71 deletions(-) diff --git a/serializer/serix/decode.go b/serializer/serix/decode.go index 4f8957c84..5e6c3dc29 100644 --- a/serializer/serix/decode.go +++ b/serializer/serix/decode.go @@ -403,6 +403,27 @@ func (api *API) decodeSlice(ctx context.Context, b []byte, value reflect.Value, return bytesRead, nil } +func (api *API) decodeMapKVPair(ctx context.Context, b []byte, key, val reflect.Value, ts TypeSettings, opts *options) (int, error) { + keyTypeSettings := TypeSettings{} + valueTypeSettings := TypeSettings{} + if ts.mapRules != nil { + keyTypeSettings = ts.mapRules.KeyRules.ToTypeSettings() + valueTypeSettings = ts.mapRules.ValueRules.ToTypeSettings() + } + + keyBytesRead, err := api.decode(ctx, b, key, keyTypeSettings, opts) + if err != nil { + return 0, ierrors.Wrapf(err, "failed to decode map key of type %s", key.Type()) + } + b = b[keyBytesRead:] + elemBytesRead, err := api.decode(ctx, b, val, valueTypeSettings, opts) + if err != nil { + return 0, ierrors.Wrapf(err, "failed to decode map element of type %s", val.Type()) + } + + return keyBytesRead + elemBytesRead, nil +} + func (api *API) decodeMap(ctx context.Context, b []byte, value reflect.Value, valueType reflect.Type, ts TypeSettings, opts *options) (int, error) { if value.IsNil() { @@ -467,24 +488,3 @@ func (api *API) decodeSequence(b []byte, deserializeItem serializer.DeserializeF return deseri.Done() } - -func (api *API) decodeMapKVPair(ctx context.Context, b []byte, key, val reflect.Value, ts TypeSettings, opts *options) (int, error) { - keyTypeSettings := TypeSettings{} - valueTypeSettings := TypeSettings{} - if ts.mapRules != nil { - keyTypeSettings = ts.mapRules.KeyRules.ToTypeSettings() - valueTypeSettings = ts.mapRules.ValueRules.ToTypeSettings() - } - - keyBytesRead, err := api.decode(ctx, b, key, keyTypeSettings, opts) - if err != nil { - return 0, ierrors.Wrapf(err, "failed to decode map key of type %s", key.Type()) - } - b = b[keyBytesRead:] - elemBytesRead, err := api.decode(ctx, b, val, valueTypeSettings, opts) - if err != nil { - return 0, ierrors.Wrapf(err, "failed to decode map element of type %s", val.Type()) - } - - return keyBytesRead + elemBytesRead, nil -} diff --git a/serializer/serix/encode.go b/serializer/serix/encode.go index 5bbdd7b49..51e873013 100644 --- a/serializer/serix/encode.go +++ b/serializer/serix/encode.go @@ -316,6 +316,30 @@ func (api *API) encodeSlice(ctx context.Context, value reflect.Value, valueType return encodeSliceOfBytes(data, valueType, ts, opts) } +func (api *API) encodeMapKVPair(ctx context.Context, key, val reflect.Value, ts TypeSettings, opts *options) ([]byte, error) { + keyTypeSettings := TypeSettings{} + valueTypeSettings := TypeSettings{} + if ts.mapRules != nil { + keyTypeSettings = ts.mapRules.KeyRules.ToTypeSettings() + valueTypeSettings = ts.mapRules.ValueRules.ToTypeSettings() + } + + keyBytes, err := api.encode(ctx, key, keyTypeSettings, opts) + if err != nil { + return nil, ierrors.Wrapf(err, "failed to encode map key of type %s", key.Type()) + } + + elemBytes, err := api.encode(ctx, val, valueTypeSettings, opts) + if err != nil { + return nil, ierrors.Wrapf(err, "failed to encode map element of type %s", val.Type()) + } + + buf := bytes.NewBuffer(keyBytes) + buf.Write(elemBytes) + + return buf.Bytes(), nil +} + func (api *API) encodeMap(ctx context.Context, value reflect.Value, valueType reflect.Type, ts TypeSettings, opts *options) ([]byte, error) { size := value.Len() @@ -349,30 +373,6 @@ func (api *API) encodeMap(ctx context.Context, value reflect.Value, valueType re return bytes, nil } -func (api *API) encodeMapKVPair(ctx context.Context, key, val reflect.Value, ts TypeSettings, opts *options) ([]byte, error) { - keyTypeSettings := TypeSettings{} - valueTypeSettings := TypeSettings{} - if ts.mapRules != nil { - keyTypeSettings = ts.mapRules.KeyRules.ToTypeSettings() - valueTypeSettings = ts.mapRules.ValueRules.ToTypeSettings() - } - - keyBytes, err := api.encode(ctx, key, keyTypeSettings, opts) - if err != nil { - return nil, ierrors.Wrapf(err, "failed to encode map key of type %s", key.Type()) - } - - elemBytes, err := api.encode(ctx, val, valueTypeSettings, opts) - if err != nil { - return nil, ierrors.Wrapf(err, "failed to encode map element of type %s", val.Type()) - } - - buf := bytes.NewBuffer(keyBytes) - buf.Write(elemBytes) - - return buf.Bytes(), nil -} - func encodeSliceOfBytes(data [][]byte, valueType reflect.Type, ts TypeSettings, opts *options) ([]byte, error) { lengthPrefixType, set := ts.LengthPrefixType() if !set { diff --git a/serializer/serix/map_decode.go b/serializer/serix/map_decode.go index 94358081b..620cbb017 100644 --- a/serializer/serix/map_decode.go +++ b/serializer/serix/map_decode.go @@ -127,7 +127,7 @@ func (api *API) mapDecodeBasedOnType(ctx context.Context, mapVal any, value refl case reflect.Slice: return api.mapDecodeSlice(ctx, mapVal, value, valueType, opts) case reflect.Map: - return api.mapDecodeMap(ctx, mapVal, value, valueType, opts) + return api.mapDecodeMap(ctx, mapVal, value, valueType, ts, opts) case reflect.Array: sliceValue := sliceFromArray(value) sliceValueType := sliceValue.Type() @@ -426,7 +426,7 @@ func (api *API) mapDecodeSlice(ctx context.Context, mapVal any, value reflect.Va } func (api *API) mapDecodeMap(ctx context.Context, mapVal any, value reflect.Value, - valueType reflect.Type, opts *options) error { + valueType reflect.Type, ts TypeSettings, opts *options) error { m, ok := mapVal.(map[string]any) if !ok { return ierrors.Errorf("non map[string]any in struct map decode, got %T instead", mapVal) @@ -436,19 +436,35 @@ func (api *API) mapDecodeMap(ctx context.Context, mapVal any, value reflect.Valu value.Set(reflect.MakeMap(valueType)) } + keyTypeSettings := TypeSettings{} + valueTypeSettings := TypeSettings{} + if ts.mapRules != nil { + keyTypeSettings = ts.mapRules.KeyRules.ToTypeSettings() + valueTypeSettings = ts.mapRules.ValueRules.ToTypeSettings() + } + for k, v := range m { - key := reflect.New(valueType.Key()).Elem() - val := reflect.New(valueType.Elem()).Elem() + keyValue := reflect.New(valueType.Key()).Elem() + elemValue := reflect.New(valueType.Elem()).Elem() - if err := api.mapDecode(ctx, k, key, TypeSettings{}, opts); err != nil { - return ierrors.Wrapf(err, "failed to map decode map key of type %s", key.Type()) + if err := api.mapDecode(ctx, k, keyValue, keyTypeSettings, opts); err != nil { + return ierrors.Wrapf(err, "failed to map decode map key of type %s", keyValue.Type()) } - if err := api.mapDecode(ctx, v, val, TypeSettings{}, opts); err != nil { - return ierrors.Wrapf(err, "failed to map decode map element of type %s", val.Type()) + if value.MapIndex(keyValue).IsValid() { + // map entry already exists + return ierrors.Wrapf(ErrMapValidationViolatesUniqueness, "map entry with key %v already exists", keyValue.Interface()) } - value.SetMapIndex(key, val) + if err := api.mapDecode(ctx, v, elemValue, valueTypeSettings, opts); err != nil { + return ierrors.Wrapf(err, "failed to map decode map element of type %s", elemValue.Type()) + } + + value.SetMapIndex(keyValue, elemValue) + } + + if err := api.checkMapMinMaxBounds(value.Len(), ts); err != nil { + return err } return nil diff --git a/serializer/serix/map_encode.go b/serializer/serix/map_encode.go index 9e06048a8..12296bc6e 100644 --- a/serializer/serix/map_encode.go +++ b/serializer/serix/map_encode.go @@ -83,7 +83,7 @@ func (api *API) mapEncodeBasedOnType( case reflect.Slice: return api.mapEncodeSlice(ctx, value, valueType, ts, opts) case reflect.Map: - return api.mapEncodeMap(ctx, value, opts) + return api.mapEncodeMap(ctx, value, ts, opts) case reflect.Array: sliceValue := sliceFromArray(value) sliceValueType := sliceValue.Type() @@ -259,13 +259,41 @@ func (api *API) mapEncodeSlice(ctx context.Context, value reflect.Value, valueTy return data, nil } -func (api *API) mapEncodeMap(ctx context.Context, value reflect.Value, opts *options) (*orderedmap.OrderedMap, error) { +func (api *API) mapEncodeMapKVPair(ctx context.Context, key, val reflect.Value, ts TypeSettings, opts *options) (string, any, error) { + keyTypeSettings := TypeSettings{} + valueTypeSettings := TypeSettings{} + if ts.mapRules != nil { + keyTypeSettings = ts.mapRules.KeyRules.ToTypeSettings() + valueTypeSettings = ts.mapRules.ValueRules.ToTypeSettings() + } + + k, err := api.mapEncode(ctx, key, keyTypeSettings, opts) + if err != nil { + return "", nil, ierrors.Wrapf(err, "failed to encode map key of type %s", key.Type()) + } + + v, err := api.mapEncode(ctx, val, valueTypeSettings, opts) + if err != nil { + return "", nil, ierrors.Wrapf(err, "failed to encode map element of type %s", val.Type()) + } + + //nolint:forcetypeassert // map keys are always strings + return k.(string), v, nil +} + +func (api *API) mapEncodeMap(ctx context.Context, value reflect.Value, ts TypeSettings, opts *options) (*orderedmap.OrderedMap, error) { + size := value.Len() + + if err := api.checkMapMinMaxBounds(size, ts); err != nil { + return nil, err + } + m := orderedmap.New() iter := value.MapRange() for i := 0; iter.Next(); i++ { key := iter.Key() elem := iter.Value() - k, v, err := api.mapEncodeMapKVPair(ctx, key, elem, opts) + k, v, err := api.mapEncodeMapKVPair(ctx, key, elem, ts, opts) if err != nil { return nil, ierrors.WithStack(err) } @@ -274,17 +302,3 @@ func (api *API) mapEncodeMap(ctx context.Context, value reflect.Value, opts *opt return m, nil } - -func (api *API) mapEncodeMapKVPair(ctx context.Context, key, val reflect.Value, opts *options) (string, any, error) { - k, err := api.mapEncode(ctx, key, TypeSettings{}, opts) - if err != nil { - return "", nil, ierrors.Wrapf(err, "failed to encode map key of type %s", key.Type()) - } - v, err := api.mapEncode(ctx, val, TypeSettings{}, opts) - if err != nil { - return "", nil, ierrors.Wrapf(err, "failed to encode map element of type %s", val.Type()) - } - - //nolint:forcetypeassert // map keys are always strings - return k.(string), v, nil -} From b57209b45ceec96c466c543d7740395e8941690c Mon Sep 17 00:00:00 2001 From: muXxer Date: Tue, 7 Nov 2023 08:39:00 +0100 Subject: [PATCH 05/17] Add check for map max size --- serializer/serix/map_decode.go | 4 ++++ serializer/serix/map_encode.go | 4 ++++ serializer/serix/map_encode_test.go | 4 ++-- serializer/serix/serix.go | 19 ++++++++++++++++++- serializer/serix/type_settings.go | 19 +++++++++++++++++++ 5 files changed, 47 insertions(+), 3 deletions(-) diff --git a/serializer/serix/map_decode.go b/serializer/serix/map_decode.go index 620cbb017..bd360beb3 100644 --- a/serializer/serix/map_decode.go +++ b/serializer/serix/map_decode.go @@ -40,6 +40,10 @@ func (api *API) mapDecode(ctx context.Context, mapVal any, value reflect.Value, } } + if err := api.checkMapSerializedSize(ctx, value, ts, opts); err != nil { + return err + } + return nil } diff --git a/serializer/serix/map_encode.go b/serializer/serix/map_encode.go index 12296bc6e..89679bac9 100644 --- a/serializer/serix/map_encode.go +++ b/serializer/serix/map_encode.go @@ -36,6 +36,10 @@ func (api *API) mapEncode(ctx context.Context, value reflect.Value, ts TypeSetti } } + if err := api.checkMapSerializedSize(ctx, value, ts, opts); err != nil { + return nil, err + } + if serializable, ok := valueI.(SerializableJSON); ok { ele, err = serializable.EncodeJSON() if err != nil { diff --git a/serializer/serix/map_encode_test.go b/serializer/serix/map_encode_test.go index bdc873703..12326faa9 100644 --- a/serializer/serix/map_encode_test.go +++ b/serializer/serix/map_encode_test.go @@ -406,10 +406,10 @@ func TestMapEncodeDecode(t *testing.T) { api: api, in: &example{ Entries: map[serializableStruct]struct{}{ - serializableStruct{ + { bytes: blake2b.Sum256([]byte("test")), index: 1, - }: struct{}{}, + }: {}, }, }, } diff --git a/serializer/serix/serix.go b/serializer/serix/serix.go index b10383114..1cdee130a 100644 --- a/serializer/serix/serix.go +++ b/serializer/serix/serix.go @@ -220,6 +220,23 @@ func (api *API) checkMapMaxByteSize(byteSize int, ts TypeSettings) error { return nil } +func (api *API) checkMapSerializedSize(ctx context.Context, value reflect.Value, ts TypeSettings, opts *options) error { + if ts.mapRules == nil || ts.mapRules.MaxByteSize == 0 { + return nil + } + + if value.Kind() != reflect.Map { + return ierrors.Errorf("can't get map serialized size: value is not a map, got %s", value.Kind()) + } + + bytes, err := api.encode(ctx, value, ts, opts) + if err != nil { + return ierrors.Wrapf(err, "can't get map serialized size: failed to encode map") + } + + return api.checkMapMaxByteSize(len(bytes), ts) +} + // Encode serializes the provided object obj into bytes. // serix traverses the object recursively and serializes everything based on the type. // If a type implements the custom Serializable interface serix delegates the serialization to that type. @@ -777,7 +794,7 @@ func parseLengthPrefixType(prefixTypeRaw string) (LengthPrefixType, error) { case "uint32": return LengthPrefixTypeAsUint32, nil default: - return LengthPrefixTypeAsByte, ierrors.Errorf("unknown length prefix type: %s", prefixTypeRaw) + return LengthPrefixTypeAsByte, ierrors.Wrapf(ErrUnknownLengthPrefixType, "%s", prefixTypeRaw) } } diff --git a/serializer/serix/type_settings.go b/serializer/serix/type_settings.go index ee0cdf683..7009967bb 100644 --- a/serializer/serix/type_settings.go +++ b/serializer/serix/type_settings.go @@ -1,9 +1,15 @@ package serix import ( + "github.com/iotaledger/hive.go/ierrors" "github.com/iotaledger/hive.go/serializer/v2" ) +var ( + // ErrUnknownLengthPrefixType gets returned when an unknown LengthPrefixType is used. + ErrUnknownLengthPrefixType = ierrors.New("unknown length prefix type") +) + // LengthPrefixType defines the type of the value denoting the length of a collection. type LengthPrefixType serializer.SeriLengthPrefixType @@ -16,6 +22,19 @@ const ( LengthPrefixTypeAsUint32 = LengthPrefixType(serializer.SeriLengthPrefixTypeAsUint32) ) +func LengthPrefixTypeSize(t LengthPrefixType) (int, error) { + switch t { + case LengthPrefixTypeAsByte: + return 1, nil + case LengthPrefixTypeAsUint16: + return 2, nil + case LengthPrefixTypeAsUint32: + return 4, nil + default: + return 0, ErrUnknownLengthPrefixType + } +} + // ArrayRules defines rules around a to be deserialized array. // Min and Max at 0 define an unbounded array. type ArrayRules serializer.ArrayRules From 2f9908342f13353a101b2c9d8427125bb11ddf36 Mon Sep 17 00:00:00 2001 From: muXxer Date: Tue, 7 Nov 2023 08:40:13 +0100 Subject: [PATCH 06/17] Move serializable tests to own file --- serializer/serix/serix_serializable_test.go | 440 ++++++++++++++++++++ serializer/serix/serix_test.go | 430 ------------------- 2 files changed, 440 insertions(+), 430 deletions(-) create mode 100644 serializer/serix/serix_serializable_test.go diff --git a/serializer/serix/serix_serializable_test.go b/serializer/serix/serix_serializable_test.go new file mode 100644 index 000000000..95a3c0651 --- /dev/null +++ b/serializer/serix/serix_serializable_test.go @@ -0,0 +1,440 @@ +//nolint:scopelint // we don't care about these linters in test cases +package serix_test + +import ( + "context" + "fmt" + "log" + "math/big" + "os" + "testing" + "time" + + "github.com/iotaledger/hive.go/ierrors" + "github.com/iotaledger/hive.go/serializer/v2" + "github.com/iotaledger/hive.go/serializer/v2/serix" +) + +type Bool bool + +func (b Bool) MarshalJSON() ([]byte, error) { + // ToDo: implement me + panic("implement me") +} + +func (b Bool) UnmarshalJSON(bytes []byte) error { + // ToDo: implement me + panic("implement me") +} + +func (b Bool) Deserialize(data []byte, deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) (int, error) { + // ToDo: implement me + panic("implement me") +} + +func (b Bool) Serialize(deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) ([]byte, error) { + ser := serializer.NewSerializer() + ser.WriteBool(bool(b), defaultErrProducer) + + return ser.Serialize() +} + +type Bools []Bool + +var boolsLenType = serix.LengthPrefixTypeAsUint16 + +func (bs Bools) MarshalJSON() ([]byte, error) { + // ToDo: implement me + panic("implement me") +} + +func (bs Bools) UnmarshalJSON(bytes []byte) error { + // ToDo: implement me + panic("implement me") +} + +func (bs Bools) Deserialize(data []byte, deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) (int, error) { + // ToDo: implement me + panic("implement me") +} + +func (bs Bools) Serialize(deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) ([]byte, error) { + seri := serializer.NewSerializer() + seri.WriteSliceOfObjects(bs, deSeriMode, deSeriCtx, serializer.SeriLengthPrefixType(boolsLenType), defaultArrayRules, defaultErrProducer) + + return seri.Serialize() +} + +func (bs Bools) ToSerializables() serializer.Serializables { + serializables := make(serializer.Serializables, len(bs)) + for i, b := range bs { + serializables[i] = b + } + + return serializables +} + +func (bs Bools) FromSerializables(seris serializer.Serializables) { + // ToDo: implement me + panic("implement me") +} + +type SimpleStruct struct { + Bool bool `serix:"0"` + Uint uint64 `serix:"1"` + String string `serix:"2,lengthPrefixType=uint16"` + Bytes []byte `serix:"3,lengthPrefixType=uint32"` + BytesArray [16]byte `serix:"4"` + BigInt *big.Int `serix:"5"` + Time time.Time `serix:"6"` + Int uint64 `serix:"7"` + Float float64 `serix:"8"` +} + +func NewSimpleStruct() SimpleStruct { + return SimpleStruct{ + Bool: true, + Uint: 10, + String: "foo", + Bytes: []byte{1, 2, 3}, + BytesArray: [16]byte{3, 2, 1}, + BigInt: big.NewInt(8), + Time: time.Unix(1000, 1000).UTC(), + Int: 23, + Float: 4.44, + } +} + +var simpleStructObjectCode = uint32(0) + +func (ss SimpleStruct) MarshalJSON() ([]byte, error) { + // ToDo: implement me + panic("implement me") +} + +func (ss SimpleStruct) UnmarshalJSON(bytes []byte) error { + // ToDo: implement me + panic("implement me") +} + +func (ss SimpleStruct) Deserialize(data []byte, deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) (int, error) { + // ToDo: implement me + panic("implement me") +} + +func (ss SimpleStruct) Serialize(deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) ([]byte, error) { + seri := serializer.NewSerializer() + seri.WriteNum(simpleStructObjectCode, defaultErrProducer) + seri.WriteBool(ss.Bool, defaultErrProducer) + seri.WriteNum(ss.Uint, defaultErrProducer) + seri.WriteString(ss.String, serializer.SeriLengthPrefixTypeAsUint16, defaultErrProducer, 0, 0) + seri.WriteVariableByteSlice(ss.Bytes, serializer.SeriLengthPrefixTypeAsUint32, defaultErrProducer, 0, 0) + seri.WriteBytes(ss.BytesArray[:], defaultErrProducer) + seri.WriteUint256(ss.BigInt, defaultErrProducer) + seri.WriteTime(ss.Time, defaultErrProducer) + seri.WriteNum(ss.Int, defaultErrProducer) + seri.WriteNum(ss.Float, defaultErrProducer) + + return seri.Serialize() +} + +type Interface interface { + Method() + + serix.Serializable + serix.Deserializable +} + +type InterfaceImpl struct { + interfaceImpl `serix:"0"` +} + +func (ii *InterfaceImpl) SetDeserializationContext(ctx context.Context) { + ctx.Value("contextValue").(func())() +} + +type interfaceImpl struct { + A uint8 `serix:"0"` + B uint8 `serix:"1"` +} + +func (ii *InterfaceImpl) Encode() ([]byte, error) { + return testAPI.Encode(context.Background(), ii.interfaceImpl, serix.WithValidation()) +} + +func (ii *InterfaceImpl) Decode(b []byte) (consumedBytes int, err error) { + return testAPI.Decode(context.Background(), b, &ii.interfaceImpl, serix.WithValidation()) +} + +var interfaceImplObjectCode = uint32(1) + +func (ii *InterfaceImpl) Method() {} + +func (ii *InterfaceImpl) MarshalJSON() ([]byte, error) { + // ToDo: implement me + panic("implement me") +} + +func (ii *InterfaceImpl) UnmarshalJSON(bytes []byte) error { + // ToDo: implement me + panic("implement me") +} + +func (ii *InterfaceImpl) Deserialize(data []byte, deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) (int, error) { + // ToDo: implement me + panic("implement me") +} + +func (ii *InterfaceImpl) Serialize(deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) ([]byte, error) { + seri := serializer.NewSerializer() + seri.WriteNum(interfaceImplObjectCode, defaultErrProducer) + seri.WriteNum(ii.A, defaultErrProducer) + seri.WriteNum(ii.B, defaultErrProducer) + + return seri.Serialize() +} + +type StructWithInterface struct { + Interface Interface `serix:"0"` +} + +func (si StructWithInterface) MarshalJSON() ([]byte, error) { + // ToDo: implement me + panic("implement me") +} + +func (si StructWithInterface) UnmarshalJSON(bytes []byte) error { + // ToDo: implement me + panic("implement me") +} + +func (si StructWithInterface) Deserialize(data []byte, deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) (int, error) { + // ToDo: implement me + panic("implement me") +} + +func (si StructWithInterface) Serialize(deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) ([]byte, error) { + seri := serializer.NewSerializer() + seri.WriteObject(si.Interface.(serializer.Serializable), defaultSeriMode, deSeriCtx, defaultWriteGuard, defaultErrProducer) + + return seri.Serialize() +} + +type StructWithOptionalField struct { + Optional *ExportedStruct `serix:"0,optional"` +} + +func (so StructWithOptionalField) MarshalJSON() ([]byte, error) { + // ToDo: implement me + panic("implement me") +} + +func (so StructWithOptionalField) UnmarshalJSON(bytes []byte) error { + // ToDo: implement me + panic("implement me") +} + +func (so StructWithOptionalField) Deserialize(data []byte, deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) (int, error) { + // ToDo: implement me + panic("implement me") +} + +func (so StructWithOptionalField) Serialize(deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) ([]byte, error) { + seri := serializer.NewSerializer() + seri.WritePayloadLength(0, defaultErrProducer) + + return seri.Serialize() +} + +type StructWithEmbeddedStructs struct { + unexportedStruct `serix:"0"` + ExportedStruct `serix:"1,nest"` +} + +func (se StructWithEmbeddedStructs) MarshalJSON() ([]byte, error) { + // ToDo: implement me + panic("implement me") +} + +func (se StructWithEmbeddedStructs) UnmarshalJSON(bytes []byte) error { + // ToDo: implement me + panic("implement me") +} + +func (se StructWithEmbeddedStructs) Deserialize(data []byte, deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) (int, error) { + // ToDo: implement me + panic("implement me") +} + +func (se StructWithEmbeddedStructs) Serialize(deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) ([]byte, error) { + seri := serializer.NewSerializer() + seri.WriteNum(se.unexportedStruct.Foo, defaultErrProducer) + seri.WriteNum(exportedStructObjectCode, defaultErrProducer) + seri.WriteNum(se.ExportedStruct.Bar, defaultErrProducer) + + return seri.Serialize() +} + +type unexportedStruct struct { + Foo uint64 `serix:"0"` +} + +type ExportedStruct struct { + Bar uint64 `serix:"0"` +} + +func (e ExportedStruct) SetDeserializationContext(ctx context.Context) { + ctx.Value("contextValue").(func())() +} + +var exportedStructObjectCode = uint32(3) + +type Map map[uint64]uint64 + +var mapLenType = serix.LengthPrefixTypeAsUint32 + +func (m Map) MarshalJSON() ([]byte, error) { + // ToDo: implement me + panic("implement me") +} + +func (m Map) UnmarshalJSON(bytes []byte) error { + // ToDo: implement me + panic("implement me") +} + +func (m Map) Deserialize(data []byte, deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) (int, error) { + // ToDo: implement me + panic("implement me") +} + +func (m Map) Serialize(deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) ([]byte, error) { + bytes := make([][]byte, len(m)) + var i int + for k, v := range m { + seri := serializer.NewSerializer() + seri.WriteNum(k, defaultErrProducer) + seri.WriteNum(v, defaultErrProducer) + b, err := seri.Serialize() + if err != nil { + return nil, err + } + bytes[i] = b + i++ + } + seri := serializer.NewSerializer() + mode := defaultSeriMode | serializer.DeSeriModePerformLexicalOrdering + arrayRules := &serializer.ArrayRules{ValidationMode: serializer.ArrayValidationModeLexicalOrdering} + seri.WriteSliceOfByteSlices(bytes, mode, serializer.SeriLengthPrefixType(mapLenType), arrayRules, defaultErrProducer) + + return seri.Serialize() +} + +type CustomSerializable int + +func (cs CustomSerializable) SetDeserializationContext(ctx context.Context) { + ctx.Value("contextValue").(func())() +} + +func (cs CustomSerializable) MarshalJSON() ([]byte, error) { + // ToDo: implement me + panic("implement me") +} + +func (cs CustomSerializable) UnmarshalJSON(bytes []byte) error { + // ToDo: implement me + panic("implement me") +} + +func (cs CustomSerializable) Deserialize(data []byte, deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) (int, error) { + // ToDo: implement me + panic("implement me") +} + +func (cs CustomSerializable) Serialize(deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) ([]byte, error) { + return cs.Encode() +} + +func (cs CustomSerializable) Encode() ([]byte, error) { + b := []byte(fmt.Sprintf("int: %d", cs)) + + return b, nil +} + +func (cs *CustomSerializable) Decode(b []byte) (int, error) { + _, err := fmt.Sscanf(string(b), "int: %d", cs) + if err != nil { + return 0, err + } + + return len(b), nil +} + +type ObjectForSyntacticValidation struct{} + +var errSyntacticValidation = ierrors.New("syntactic validation failed") + +func SyntacticValidation(ctx context.Context, obj ObjectForSyntacticValidation) error { + return errSyntacticValidation +} + +type ObjectForBytesValidation struct{} + +var errBytesValidation = ierrors.New("bytes validation failed") + +func BytesValidation(ctx context.Context, b []byte) error { + return errBytesValidation +} + +func TestMain(m *testing.M) { + exitCode := func() int { + if err := testAPI.RegisterTypeSettings( + SimpleStruct{}, + serix.TypeSettings{}.WithObjectType(simpleStructObjectCode), + ); err != nil { + log.Panic(err) + } + if err := testAPI.RegisterTypeSettings( + InterfaceImpl{}, + serix.TypeSettings{}.WithObjectType(interfaceImplObjectCode), + ); err != nil { + log.Panic(err) + } + if err := testAPI.RegisterTypeSettings( + ExportedStruct{}, + serix.TypeSettings{}.WithObjectType(exportedStructObjectCode), + ); err != nil { + log.Panic(err) + } + if err := testAPI.RegisterInterfaceObjects((*Interface)(nil), (*InterfaceImpl)(nil)); err != nil { + log.Panic(err) + } + if err := testAPI.RegisterValidators(ObjectForSyntacticValidation{}, nil, SyntacticValidation); err != nil { + log.Panic(err) + } + if err := testAPI.RegisterValidators(ObjectForBytesValidation{}, BytesValidation, nil); err != nil { + log.Panic(err) + } + + return m.Run() + }() + os.Exit(exitCode) +} + +func BenchmarkEncode(b *testing.B) { + simpleStruct := NewSimpleStruct() + for i := 0; i < b.N; i++ { + testAPI.Encode(context.Background(), simpleStruct) + } +} + +func BenchmarkDecode(b *testing.B) { + simpleStruct := NewSimpleStruct() + encoded, err := testAPI.Encode(context.Background(), simpleStruct) + if err != nil { + b.Fatal(err) + } + for i := 0; i < b.N; i++ { + testAPI.Decode(context.Background(), encoded, new(SimpleStruct)) + } +} diff --git a/serializer/serix/serix_test.go b/serializer/serix/serix_test.go index 19711d58b..f3206ab61 100644 --- a/serializer/serix/serix_test.go +++ b/serializer/serix/serix_test.go @@ -3,17 +3,11 @@ package serix_test import ( "context" - "fmt" - "log" - "math/big" - "os" "reflect" "testing" - "time" "github.com/stretchr/testify/require" - "github.com/iotaledger/hive.go/ierrors" "github.com/iotaledger/hive.go/serializer/v2" "github.com/iotaledger/hive.go/serializer/v2/serix" ) @@ -28,412 +22,6 @@ var ( defaultWriteGuard = func(seri serializer.Serializable) error { return nil } ) -type Bool bool - -func (b Bool) MarshalJSON() ([]byte, error) { - // ToDo: implement me - panic("implement me") -} - -func (b Bool) UnmarshalJSON(bytes []byte) error { - // ToDo: implement me - panic("implement me") -} - -func (b Bool) Deserialize(data []byte, deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) (int, error) { - // ToDo: implement me - panic("implement me") -} - -func (b Bool) Serialize(deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) ([]byte, error) { - ser := serializer.NewSerializer() - ser.WriteBool(bool(b), defaultErrProducer) - - return ser.Serialize() -} - -type Bools []Bool - -var boolsLenType = serix.LengthPrefixTypeAsUint16 - -func (bs Bools) MarshalJSON() ([]byte, error) { - // ToDo: implement me - panic("implement me") -} - -func (bs Bools) UnmarshalJSON(bytes []byte) error { - // ToDo: implement me - panic("implement me") -} - -func (bs Bools) Deserialize(data []byte, deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) (int, error) { - // ToDo: implement me - panic("implement me") -} - -func (bs Bools) Serialize(deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) ([]byte, error) { - seri := serializer.NewSerializer() - seri.WriteSliceOfObjects(bs, deSeriMode, deSeriCtx, serializer.SeriLengthPrefixType(boolsLenType), defaultArrayRules, defaultErrProducer) - - return seri.Serialize() -} - -func (bs Bools) ToSerializables() serializer.Serializables { - serializables := make(serializer.Serializables, len(bs)) - for i, b := range bs { - serializables[i] = b - } - - return serializables -} - -func (bs Bools) FromSerializables(seris serializer.Serializables) { - // ToDo: implement me - panic("implement me") -} - -type SimpleStruct struct { - Bool bool `serix:"0"` - Uint uint64 `serix:"1"` - String string `serix:"2,lengthPrefixType=uint16"` - Bytes []byte `serix:"3,lengthPrefixType=uint32"` - BytesArray [16]byte `serix:"4"` - BigInt *big.Int `serix:"5"` - Time time.Time `serix:"6"` - Int uint64 `serix:"7"` - Float float64 `serix:"8"` -} - -func NewSimpleStruct() SimpleStruct { - return SimpleStruct{ - Bool: true, - Uint: 10, - String: "foo", - Bytes: []byte{1, 2, 3}, - BytesArray: [16]byte{3, 2, 1}, - BigInt: big.NewInt(8), - Time: time.Unix(1000, 1000).UTC(), - Int: 23, - Float: 4.44, - } -} - -var simpleStructObjectCode = uint32(0) - -func (ss SimpleStruct) MarshalJSON() ([]byte, error) { - // ToDo: implement me - panic("implement me") -} - -func (ss SimpleStruct) UnmarshalJSON(bytes []byte) error { - // ToDo: implement me - panic("implement me") -} - -func (ss SimpleStruct) Deserialize(data []byte, deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) (int, error) { - // ToDo: implement me - panic("implement me") -} - -func (ss SimpleStruct) Serialize(deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) ([]byte, error) { - seri := serializer.NewSerializer() - seri.WriteNum(simpleStructObjectCode, defaultErrProducer) - seri.WriteBool(ss.Bool, defaultErrProducer) - seri.WriteNum(ss.Uint, defaultErrProducer) - seri.WriteString(ss.String, serializer.SeriLengthPrefixTypeAsUint16, defaultErrProducer, 0, 0) - seri.WriteVariableByteSlice(ss.Bytes, serializer.SeriLengthPrefixTypeAsUint32, defaultErrProducer, 0, 0) - seri.WriteBytes(ss.BytesArray[:], defaultErrProducer) - seri.WriteUint256(ss.BigInt, defaultErrProducer) - seri.WriteTime(ss.Time, defaultErrProducer) - seri.WriteNum(ss.Int, defaultErrProducer) - seri.WriteNum(ss.Float, defaultErrProducer) - - return seri.Serialize() -} - -type Interface interface { - Method() - - serix.Serializable - serix.Deserializable -} - -type InterfaceImpl struct { - interfaceImpl `serix:"0"` -} - -func (ii *InterfaceImpl) SetDeserializationContext(ctx context.Context) { - ctx.Value("contextValue").(func())() -} - -type interfaceImpl struct { - A uint8 `serix:"0"` - B uint8 `serix:"1"` -} - -func (ii *InterfaceImpl) Encode() ([]byte, error) { - return testAPI.Encode(context.Background(), ii.interfaceImpl, serix.WithValidation()) -} - -func (ii *InterfaceImpl) Decode(b []byte) (consumedBytes int, err error) { - return testAPI.Decode(context.Background(), b, &ii.interfaceImpl, serix.WithValidation()) -} - -var interfaceImplObjectCode = uint32(1) - -func (ii *InterfaceImpl) Method() {} - -func (ii *InterfaceImpl) MarshalJSON() ([]byte, error) { - // ToDo: implement me - panic("implement me") -} - -func (ii *InterfaceImpl) UnmarshalJSON(bytes []byte) error { - // ToDo: implement me - panic("implement me") -} - -func (ii *InterfaceImpl) Deserialize(data []byte, deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) (int, error) { - // ToDo: implement me - panic("implement me") -} - -func (ii *InterfaceImpl) Serialize(deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) ([]byte, error) { - seri := serializer.NewSerializer() - seri.WriteNum(interfaceImplObjectCode, defaultErrProducer) - seri.WriteNum(ii.A, defaultErrProducer) - seri.WriteNum(ii.B, defaultErrProducer) - - return seri.Serialize() -} - -type StructWithInterface struct { - Interface Interface `serix:"0"` -} - -func (si StructWithInterface) MarshalJSON() ([]byte, error) { - // ToDo: implement me - panic("implement me") -} - -func (si StructWithInterface) UnmarshalJSON(bytes []byte) error { - // ToDo: implement me - panic("implement me") -} - -func (si StructWithInterface) Deserialize(data []byte, deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) (int, error) { - // ToDo: implement me - panic("implement me") -} - -func (si StructWithInterface) Serialize(deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) ([]byte, error) { - seri := serializer.NewSerializer() - seri.WriteObject(si.Interface.(serializer.Serializable), defaultSeriMode, deSeriCtx, defaultWriteGuard, defaultErrProducer) - - return seri.Serialize() -} - -type StructWithOptionalField struct { - Optional *ExportedStruct `serix:"0,optional"` -} - -func (so StructWithOptionalField) MarshalJSON() ([]byte, error) { - // ToDo: implement me - panic("implement me") -} - -func (so StructWithOptionalField) UnmarshalJSON(bytes []byte) error { - // ToDo: implement me - panic("implement me") -} - -func (so StructWithOptionalField) Deserialize(data []byte, deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) (int, error) { - // ToDo: implement me - panic("implement me") -} - -func (so StructWithOptionalField) Serialize(deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) ([]byte, error) { - seri := serializer.NewSerializer() - seri.WritePayloadLength(0, defaultErrProducer) - - return seri.Serialize() -} - -type StructWithEmbeddedStructs struct { - unexportedStruct `serix:"0"` - ExportedStruct `serix:"1,nest"` -} - -func (se StructWithEmbeddedStructs) MarshalJSON() ([]byte, error) { - // ToDo: implement me - panic("implement me") -} - -func (se StructWithEmbeddedStructs) UnmarshalJSON(bytes []byte) error { - // ToDo: implement me - panic("implement me") -} - -func (se StructWithEmbeddedStructs) Deserialize(data []byte, deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) (int, error) { - // ToDo: implement me - panic("implement me") -} - -func (se StructWithEmbeddedStructs) Serialize(deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) ([]byte, error) { - seri := serializer.NewSerializer() - seri.WriteNum(se.unexportedStruct.Foo, defaultErrProducer) - seri.WriteNum(exportedStructObjectCode, defaultErrProducer) - seri.WriteNum(se.ExportedStruct.Bar, defaultErrProducer) - - return seri.Serialize() -} - -type unexportedStruct struct { - Foo uint64 `serix:"0"` -} - -type ExportedStruct struct { - Bar uint64 `serix:"0"` -} - -func (e ExportedStruct) SetDeserializationContext(ctx context.Context) { - ctx.Value("contextValue").(func())() -} - -var exportedStructObjectCode = uint32(3) - -type Map map[uint64]uint64 - -var mapLenType = serix.LengthPrefixTypeAsUint32 - -func (m Map) MarshalJSON() ([]byte, error) { - // ToDo: implement me - panic("implement me") -} - -func (m Map) UnmarshalJSON(bytes []byte) error { - // ToDo: implement me - panic("implement me") -} - -func (m Map) Deserialize(data []byte, deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) (int, error) { - // ToDo: implement me - panic("implement me") -} - -func (m Map) Serialize(deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) ([]byte, error) { - bytes := make([][]byte, len(m)) - var i int - for k, v := range m { - seri := serializer.NewSerializer() - seri.WriteNum(k, defaultErrProducer) - seri.WriteNum(v, defaultErrProducer) - b, err := seri.Serialize() - if err != nil { - return nil, err - } - bytes[i] = b - i++ - } - seri := serializer.NewSerializer() - mode := defaultSeriMode | serializer.DeSeriModePerformLexicalOrdering - arrayRules := &serializer.ArrayRules{ValidationMode: serializer.ArrayValidationModeLexicalOrdering} - seri.WriteSliceOfByteSlices(bytes, mode, serializer.SeriLengthPrefixType(mapLenType), arrayRules, defaultErrProducer) - - return seri.Serialize() -} - -type CustomSerializable int - -func (cs CustomSerializable) SetDeserializationContext(ctx context.Context) { - ctx.Value("contextValue").(func())() -} - -func (cs CustomSerializable) MarshalJSON() ([]byte, error) { - // ToDo: implement me - panic("implement me") -} - -func (cs CustomSerializable) UnmarshalJSON(bytes []byte) error { - // ToDo: implement me - panic("implement me") -} - -func (cs CustomSerializable) Deserialize(data []byte, deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) (int, error) { - // ToDo: implement me - panic("implement me") -} - -func (cs CustomSerializable) Serialize(deSeriMode serializer.DeSerializationMode, deSeriCtx interface{}) ([]byte, error) { - return cs.Encode() -} - -func (cs CustomSerializable) Encode() ([]byte, error) { - b := []byte(fmt.Sprintf("int: %d", cs)) - - return b, nil -} - -func (cs *CustomSerializable) Decode(b []byte) (int, error) { - _, err := fmt.Sscanf(string(b), "int: %d", cs) - if err != nil { - return 0, err - } - - return len(b), nil -} - -type ObjectForSyntacticValidation struct{} - -var errSyntacticValidation = ierrors.New("syntactic validation failed") - -func SyntacticValidation(ctx context.Context, obj ObjectForSyntacticValidation) error { - return errSyntacticValidation -} - -type ObjectForBytesValidation struct{} - -var errBytesValidation = ierrors.New("bytes validation failed") - -func BytesValidation(ctx context.Context, b []byte) error { - return errBytesValidation -} - -func TestMain(m *testing.M) { - exitCode := func() int { - if err := testAPI.RegisterTypeSettings( - SimpleStruct{}, - serix.TypeSettings{}.WithObjectType(simpleStructObjectCode), - ); err != nil { - log.Panic(err) - } - if err := testAPI.RegisterTypeSettings( - InterfaceImpl{}, - serix.TypeSettings{}.WithObjectType(interfaceImplObjectCode), - ); err != nil { - log.Panic(err) - } - if err := testAPI.RegisterTypeSettings( - ExportedStruct{}, - serix.TypeSettings{}.WithObjectType(exportedStructObjectCode), - ); err != nil { - log.Panic(err) - } - if err := testAPI.RegisterInterfaceObjects((*Interface)(nil), (*InterfaceImpl)(nil)); err != nil { - log.Panic(err) - } - if err := testAPI.RegisterValidators(ObjectForSyntacticValidation{}, nil, SyntacticValidation); err != nil { - log.Panic(err) - } - if err := testAPI.RegisterValidators(ObjectForBytesValidation{}, BytesValidation, nil); err != nil { - log.Panic(err) - } - - return m.Run() - }() - os.Exit(exitCode) -} - func TestMinMax(t *testing.T) { type paras struct { api *serix.API @@ -544,21 +132,3 @@ func TestMinMax(t *testing.T) { }) } } - -func BenchmarkEncode(b *testing.B) { - simpleStruct := NewSimpleStruct() - for i := 0; i < b.N; i++ { - testAPI.Encode(context.Background(), simpleStruct) - } -} - -func BenchmarkDecode(b *testing.B) { - simpleStruct := NewSimpleStruct() - encoded, err := testAPI.Encode(context.Background(), simpleStruct) - if err != nil { - b.Fatal(err) - } - for i := 0; i < b.N; i++ { - testAPI.Decode(context.Background(), encoded, new(SimpleStruct)) - } -} From 54bd520012e6626c00e80615bc74ef8d3e583353 Mon Sep 17 00:00:00 2001 From: muXxer Date: Tue, 7 Nov 2023 08:40:46 +0100 Subject: [PATCH 07/17] Add serix map tests --- serializer/serix/serix_test.go | 166 +++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) diff --git a/serializer/serix/serix_test.go b/serializer/serix/serix_test.go index f3206ab61..178f08cb5 100644 --- a/serializer/serix/serix_test.go +++ b/serializer/serix/serix_test.go @@ -132,3 +132,169 @@ func TestMinMax(t *testing.T) { }) } } + +type deSerializeTest struct { + name string + source any + target any + size int + seriErr error + deSeriErr error +} + +func (test *deSerializeTest) deSerialize(t *testing.T) { + serixData, err := testAPI.Encode(context.Background(), test.source, serix.WithValidation()) + if test.seriErr != nil { + require.ErrorIs(t, err, test.seriErr) + + return + } + require.NoError(t, err) + + require.Equal(t, test.size, len(serixData)) + + serixTarget := reflect.New(reflect.TypeOf(test.target).Elem()).Interface() + bytesRead, err := testAPI.Decode(context.Background(), serixData, serixTarget) + if test.deSeriErr != nil { + require.ErrorIs(t, err, test.deSeriErr) + + return + } + require.NoError(t, err) + require.Len(t, serixData, bytesRead) + require.EqualValues(t, test.source, serixTarget) + + sourceJSON, err := testAPI.JSONEncode(context.Background(), test.source) + require.NoError(t, err) + + jsonDest := reflect.New(reflect.TypeOf(test.target).Elem()).Interface() + require.NoError(t, testAPI.JSONDecode(context.Background(), sourceJSON, jsonDest)) + + require.EqualValues(t, test.source, jsonDest) +} + +func TestSerixMap(t *testing.T) { + + type MyMapType map[string]string + + type MapStruct struct { + MyMap MyMapType `serix:"0,lengthPrefixType=uint8,mapMinEntries=2,mapMaxEntries=4,mapMaxByteSize=50,mapKeyLengthPrefixType=uint16,mapKeyMinLen=2,mapKeyMaxLen=5,mapValueLengthPrefixType=uint32,mapValueMinLen=1,mapValueMaxLen=6"` + } + + testAPI.RegisterTypeSettings(MapStruct{}, serix.TypeSettings{}.WithObjectType(uint8(0))) + + tests := []deSerializeTest{ + { + name: "ok", + source: &MapStruct{ + MyMap: map[string]string{ + "k1": "v1", + "k2": "v2", + }, + }, + target: &MapStruct{}, + size: 22, + seriErr: nil, + deSeriErr: nil, + }, + { + name: "fail - not enough entries", + source: &MapStruct{ + MyMap: map[string]string{ + "k1": "v1", + }, + }, + target: &MapStruct{}, + size: 0, + seriErr: serix.ErrMapValidationMinElementsNotReached, + deSeriErr: nil, + }, + { + name: "fail - too many entries", + source: &MapStruct{ + MyMap: map[string]string{ + "k1": "v1", + "k2": "v2", + "k3": "v3", + "k4": "v4", + "k5": "v5", + }, + }, + target: &MapStruct{}, + size: 0, + seriErr: serix.ErrMapValidationMaxElementsExceeded, + deSeriErr: nil, + }, + { + name: "fail - too big", + source: &MapStruct{ + MyMap: map[string]string{ + "k1": "v1000", + "k2": "v2000", + "k3": "v3000", + "k4": "v4000", + }, + }, + target: &MapStruct{}, + size: 0, + seriErr: serix.ErrMapValidationMaxBytesExceeded, + deSeriErr: nil, + }, + { + name: "fail - key too short", + source: &MapStruct{ + MyMap: map[string]string{ + "k1": "v1", + "k": "v2", + }, + }, + target: &MapStruct{}, + size: 0, + seriErr: serializer.ErrArrayValidationMinElementsNotReached, + deSeriErr: nil, + }, + { + name: "fail - key too long", + source: &MapStruct{ + MyMap: map[string]string{ + "k1": "v1", + "k20000": "v2", + }, + }, + target: &MapStruct{}, + size: 0, + seriErr: serializer.ErrArrayValidationMaxElementsExceeded, + deSeriErr: nil, + }, + { + name: "fail - value too short", + source: &MapStruct{ + MyMap: map[string]string{ + "k1": "v1", + "k2": "", + }, + }, + target: &MapStruct{}, + size: 0, + seriErr: serializer.ErrArrayValidationMinElementsNotReached, + deSeriErr: nil, + }, + { + name: "fail - value too long", + source: &MapStruct{ + MyMap: map[string]string{ + "k1": "v1", + "k2": "v200000", + }, + }, + target: &MapStruct{}, + size: 0, + seriErr: serializer.ErrArrayValidationMaxElementsExceeded, + deSeriErr: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, tt.deSerialize) + } +} From 9be19c083a606536e354324b96492f1343afb59e Mon Sep 17 00:00:00 2001 From: muXxer Date: Tue, 7 Nov 2023 13:05:06 +0100 Subject: [PATCH 08/17] Add missing bounds checks --- serializer/error.go | 4 + serializer/serializer.go | 8 +- serializer/serix/decode.go | 22 ++- serializer/serix/decode_test.go | 2 +- serializer/serix/encode_test.go | 2 +- serializer/serix/map_decode.go | 25 +++- serializer/serix/map_encode.go | 13 ++ serializer/serix/serix.go | 30 ++-- serializer/serix/serix_test.go | 241 ++++++++++++++++++++++++++------ 9 files changed, 283 insertions(+), 64 deletions(-) diff --git a/serializer/error.go b/serializer/error.go index 3e6bcd3f2..b4eae64a0 100644 --- a/serializer/error.go +++ b/serializer/error.go @@ -32,6 +32,10 @@ var ( ErrDeserializationInvalidBoolValue = ierrors.New("invalid bool value") // ErrDeserializationLengthInvalid gets returned if a length denotation exceeds a specified limit. ErrDeserializationLengthInvalid = ierrors.New("length denotation invalid") + // ErrDeserializationLengthMinNotReached gets returned if a length denotation is less than a specified limit. + ErrDeserializationLengthMinNotReached = ierrors.New("min length not reached") + // ErrDeserializationLengthMaxExceeded gets returned if a length denotation is more than a specified limit. + ErrDeserializationLengthMaxExceeded = ierrors.New("max length exceeded") // ErrDeserializationNotAllConsumed gets returned if not all bytes were consumed during deserialization of a given type. ErrDeserializationNotAllConsumed = ierrors.New("not all data has been consumed but should have been") // ErrUint256NumNegative gets returned if a supposed uint256 has a sign bit. diff --git a/serializer/serializer.go b/serializer/serializer.go index 251de369f..96623b8d9 100644 --- a/serializer/serializer.go +++ b/serializer/serializer.go @@ -776,9 +776,9 @@ func (d *Deserializer) ReadVariableByteSlice(slice *[]byte, lenType SeriLengthPr switch { case maxLen > 0 && sliceLength > maxLen: - d.err = errProducer(ierrors.Wrapf(ErrDeserializationLengthInvalid, "denoted %d bytes, max allowed %d ", sliceLength, maxLen)) + d.err = errProducer(ierrors.Join(ErrDeserializationLengthInvalid, ierrors.Wrapf(ErrDeserializationLengthMaxExceeded, "denoted %d bytes, max allowed %d ", sliceLength, maxLen))) case minLen > 0 && sliceLength < minLen: - d.err = errProducer(ierrors.Wrapf(ErrDeserializationLengthInvalid, "denoted %d bytes, min required %d ", sliceLength, minLen)) + d.err = errProducer(ierrors.Join(ErrDeserializationLengthInvalid, ierrors.Wrapf(ErrDeserializationLengthMinNotReached, "denoted %d bytes, min required %d ", sliceLength, minLen))) } dest := make([]byte, sliceLength) @@ -1158,9 +1158,9 @@ func (d *Deserializer) ReadString(s *string, lenType SeriLengthPrefixType, errPr switch { case maxLen > 0 && strLen > maxLen: - d.err = errProducer(ierrors.Wrapf(ErrDeserializationLengthInvalid, "string defined to be of %d bytes length but max %d is allowed", strLen, maxLen)) + d.err = errProducer(ierrors.Join(ErrDeserializationLengthInvalid, ierrors.Wrapf(ErrDeserializationLengthMaxExceeded, "string defined to be of %d bytes length but max %d is allowed", strLen, maxLen))) case minLen > 0 && strLen < minLen: - d.err = errProducer(ierrors.Wrapf(ErrDeserializationLengthInvalid, "string defined to be of %d bytes length but min %d is required", strLen, minLen)) + d.err = errProducer(ierrors.Join(ErrDeserializationLengthInvalid, ierrors.Wrapf(ErrDeserializationLengthMinNotReached, "string defined to be of %d bytes length but min %d is required", strLen, minLen))) } if len(d.src[d.offset:]) < strLen { diff --git a/serializer/serix/decode.go b/serializer/serix/decode.go index 5e6c3dc29..cd785ba53 100644 --- a/serializer/serix/decode.go +++ b/serializer/serix/decode.go @@ -168,7 +168,16 @@ func (api *API) decodeBasedOnType(ctx context.Context, b []byte, value reflect.V addrValue.Interface().(*string), serializer.SeriLengthPrefixType(lengthPrefixType), func(err error) error { - return ierrors.Wrap(err, "failed to read string value from the deserializer") + err = ierrors.Wrap(err, "failed to read string value from the deserializer") + + switch { + case ierrors.Is(err, serializer.ErrDeserializationLengthMinNotReached): + return ierrors.Join(err, serializer.ErrArrayValidationMinElementsNotReached) + case ierrors.Is(err, serializer.ErrDeserializationLengthMaxExceeded): + return ierrors.Join(err, serializer.ErrArrayValidationMaxElementsExceeded) + default: + return err + } }, minLen, maxLen) return deseri.Done() @@ -373,7 +382,16 @@ func (api *API) decodeSlice(ctx context.Context, b []byte, value reflect.Value, addrValue.Interface().(*[]byte), serializer.SeriLengthPrefixType(lengthPrefixType), func(err error) error { - return ierrors.Wrap(err, "failed to read bytes from the deserializer") + err = ierrors.Wrap(err, "failed to read bytes from the deserializer") + + switch { + case ierrors.Is(err, serializer.ErrDeserializationLengthMinNotReached): + return ierrors.Join(err, serializer.ErrArrayValidationMinElementsNotReached) + case ierrors.Is(err, serializer.ErrDeserializationLengthMaxExceeded): + return ierrors.Join(err, serializer.ErrArrayValidationMaxElementsExceeded) + default: + return err + } }, minLen, maxLen) return deseri.Done() diff --git a/serializer/serix/decode_test.go b/serializer/serix/decode_test.go index 47e58e6f9..7be0a00a6 100644 --- a/serializer/serix/decode_test.go +++ b/serializer/serix/decode_test.go @@ -143,7 +143,7 @@ func TestDecode_ArrayRules(t *testing.T) { ts := serix.TypeSettings{}.WithLengthPrefixType(boolsLenType).WithArrayRules(rules) bytesRead, err := testAPI.Decode(ctx, bytes, testObj, serix.WithValidation(), serix.WithTypeSettings(ts)) require.Zero(t, bytesRead) - assert.Contains(t, err.Error(), "min count of elements within the array not reached") + assert.Contains(t, err.Error(), serializer.ErrArrayValidationMinElementsNotReached.Error()) } func testDecode(t testing.TB, ctx context.Context, expected serializer.Serializable, opts ...serix.Option) { diff --git a/serializer/serix/encode_test.go b/serializer/serix/encode_test.go index 30be0d6e0..c06948327 100644 --- a/serializer/serix/encode_test.go +++ b/serializer/serix/encode_test.go @@ -96,7 +96,7 @@ func TestEncode_ArrayRules(t *testing.T) { ts := serix.TypeSettings{}.WithLengthPrefixType(serix.LengthPrefixTypeAsUint32).WithArrayRules(rules) got, err := testAPI.Encode(ctx, testObj, serix.WithValidation(), serix.WithTypeSettings(ts)) require.Nil(t, got) - assert.Contains(t, err.Error(), "min count of elements within the array not reached") + assert.Contains(t, err.Error(), serializer.ErrArrayValidationMinElementsNotReached.Error()) } func testEncode(t testing.TB, testObj serializer.Serializable, opts ...serix.Option) { diff --git a/serializer/serix/map_decode.go b/serializer/serix/map_decode.go index bd360beb3..c47a55697 100644 --- a/serializer/serix/map_decode.go +++ b/serializer/serix/map_decode.go @@ -113,13 +113,18 @@ func (api *API) mapDecodeBasedOnType(ctx context.Context, mapVal any, value refl if err != nil { return ierrors.Wrap(err, "failed to read byte slice from map") } + + if err := api.checkMinMaxBoundsLength(len(byteSlice), ts); err != nil { + return ierrors.Wrapf(err, "can't deserialize '%s' type", value.Kind()) + } + copy(sliceValue.Bytes(), byteSlice) fillArrayFromSlice(value.Elem(), sliceValue) return nil } - return api.mapDecodeSlice(ctx, mapVal, sliceValue, sliceValueType, opts) + return api.mapDecodeSlice(ctx, mapVal, sliceValue, sliceValueType, ts, opts) } case reflect.Struct: @@ -129,7 +134,7 @@ func (api *API) mapDecodeBasedOnType(ctx context.Context, mapVal any, value refl return api.mapDecodeStruct(ctx, mapVal, value, valueType, ts, opts) case reflect.Slice: - return api.mapDecodeSlice(ctx, mapVal, value, valueType, opts) + return api.mapDecodeSlice(ctx, mapVal, value, valueType, ts, opts) case reflect.Map: return api.mapDecodeMap(ctx, mapVal, value, valueType, ts, opts) case reflect.Array: @@ -146,7 +151,7 @@ func (api *API) mapDecodeBasedOnType(ctx context.Context, mapVal any, value refl return nil } - return api.mapDecodeSlice(ctx, mapVal, sliceValue, sliceValueType, opts) + return api.mapDecodeSlice(ctx, mapVal, sliceValue, sliceValueType, ts, opts) case reflect.Interface: return api.mapDecodeInterface(ctx, mapVal, value, valueType, ts, opts) case reflect.String: @@ -160,6 +165,10 @@ func (api *API) mapDecodeBasedOnType(ctx context.Context, mapVal any, value refl addrValue := value.Addr().Convert(reflect.TypeOf((*string)(nil))) addrValue.Elem().Set(reflect.ValueOf(mapVal)) + if err := api.checkMinMaxBoundsLength(len(str), ts); err != nil { + return ierrors.Wrapf(err, "can't deserialize '%s' type", value.Kind()) + } + return nil case reflect.Bool: addrValue := value.Addr().Convert(reflect.TypeOf((*bool)(nil))) @@ -396,7 +405,7 @@ func (api *API) mapDecodeStructFields( } func (api *API) mapDecodeSlice(ctx context.Context, mapVal any, value reflect.Value, - valueType reflect.Type, opts *options) error { + valueType reflect.Type, ts TypeSettings, opts *options) error { if valueType.AssignableTo(bytesType) { //nolint:forcetypeassert // false positive, we already checked the type via reflect fieldValStr := mapVal.(string) @@ -405,6 +414,10 @@ func (api *API) mapDecodeSlice(ctx context.Context, mapVal any, value reflect.Va return ierrors.Wrap(err, "failed to read byte slice from map") } + if err := api.checkMinMaxBoundsLength(len(byteSlice), ts); err != nil { + return ierrors.Wrapf(err, "can't deserialize '%s' type", value.Kind()) + } + addrValue := value.Addr().Convert(reflect.TypeOf((*[]byte)(nil))) addrValue.Elem().SetBytes(byteSlice) @@ -420,6 +433,10 @@ func (api *API) mapDecodeSlice(ctx context.Context, mapVal any, value reflect.Va value.Set(reflect.Append(value, elemValue)) } + if err := api.checkMinMaxBounds(value, ts); err != nil { + return ierrors.Wrapf(err, "can't serialize '%s' type", value.Kind()) + } + // check if the slice is a nil pointer to the slice type (in case the sliceLength is zero and the slice was not initialized before) if value.IsNil() { // initialize a new empty slice diff --git a/serializer/serix/map_encode.go b/serializer/serix/map_encode.go index 89679bac9..23e5cf780 100644 --- a/serializer/serix/map_encode.go +++ b/serializer/serix/map_encode.go @@ -101,6 +101,10 @@ func (api *API) mapEncodeBasedOnType( return nil, ErrNonUTF8String } + if err := api.checkMinMaxBoundsLength(len(str), ts); err != nil { + return nil, ierrors.Wrapf(err, "can't serialize '%s' type", value.Kind()) + } + return value.String(), nil case reflect.Bool: return value.Bool(), nil @@ -246,10 +250,19 @@ func (api *API) mapEncodeSlice(ctx context.Context, value reflect.Value, valueTy } if valueType.AssignableTo(bytesType) { + if err := api.checkMinMaxBoundsLength(len(value.Bytes()), ts); err != nil { + return nil, ierrors.Wrapf(err, "can't serialize '%s' type", value.Kind()) + } + return EncodeHex(value.Bytes()), nil } sliceLen := value.Len() + + if err := api.checkMinMaxBoundsLength(sliceLen, ts); err != nil { + return nil, ierrors.Wrapf(err, "can't serialize '%s' type", value.Kind()) + } + data := make([]any, sliceLen) for i := 0; i < sliceLen; i++ { elemValue := value.Index(i) diff --git a/serializer/serix/serix.go b/serializer/serix/serix.go index 1cdee130a..b270a267e 100644 --- a/serializer/serix/serix.go +++ b/serializer/serix/serix.go @@ -175,27 +175,35 @@ func hasLength(v reflect.Value) bool { return true } -// checks whether the given value is within its defined bounds in case it has a length. -func (api *API) checkMinMaxBounds(v reflect.Value, ts TypeSettings) error { - if has := hasLength(v); !has { - return nil - } - - l := uint(v.Len()) +// checkMinMaxBoundsLength checks whether the given length is within its defined bounds. +func (api *API) checkMinMaxBoundsLength(length int, ts TypeSettings) error { if minLen, ok := ts.MinLen(); ok { - if l < minLen { - return ierrors.Wrapf(serializer.ErrArrayValidationMinElementsNotReached, "can't serialize '%s' type: min length %d not reached (len %d)", v.Kind(), minLen, l) + if uint(length) < minLen { + return ierrors.Wrapf(serializer.ErrArrayValidationMinElementsNotReached, "min length %d not reached (len %d)", minLen, length) } } if maxLen, ok := ts.MaxLen(); ok { - if l > maxLen { - return ierrors.Wrapf(serializer.ErrArrayValidationMaxElementsExceeded, "can't serialize '%s' type: max length %d exceeded (len %d)", v.Kind(), maxLen, l) + if uint(length) > maxLen { + return ierrors.Wrapf(serializer.ErrArrayValidationMaxElementsExceeded, "max length %d exceeded (len %d)", maxLen, length) } } return nil } +// checkMinMaxBounds checks whether the given value is within its defined bounds in case it has a length. +func (api *API) checkMinMaxBounds(v reflect.Value, ts TypeSettings) error { + if has := hasLength(v); !has { + return nil + } + + if err := api.checkMinMaxBoundsLength(v.Len(), ts); err != nil { + return ierrors.Wrapf(err, "can't serialize '%s' type", v.Kind()) + } + + return nil +} + // checkMapMinMaxBounds checks whether the given map is within its defined bounds in case it has defined map rules. func (api *API) checkMapMinMaxBounds(length int, ts TypeSettings) error { if ts.mapRules != nil { diff --git a/serializer/serix/serix_test.go b/serializer/serix/serix_test.go index 178f08cb5..d12b37a28 100644 --- a/serializer/serix/serix_test.go +++ b/serializer/serix/serix_test.go @@ -133,85 +133,250 @@ func TestMinMax(t *testing.T) { } } -type deSerializeTest struct { - name string - source any - target any - size int - seriErr error - deSeriErr error +type serializeTest struct { + name string + source any + target any + size int + seriErr error } -func (test *deSerializeTest) deSerialize(t *testing.T) { +func (test *serializeTest) run(t *testing.T) { + // binary serialize serixData, err := testAPI.Encode(context.Background(), test.source, serix.WithValidation()) if test.seriErr != nil { require.ErrorIs(t, err, test.seriErr) + // we also need to check the json serialization + _, err := testAPI.JSONEncode(context.Background(), test.source, serix.WithValidation()) + require.ErrorIs(t, err, test.seriErr) + return } require.NoError(t, err) require.Equal(t, test.size, len(serixData)) + // binary deserialize serixTarget := reflect.New(reflect.TypeOf(test.target).Elem()).Interface() bytesRead, err := testAPI.Decode(context.Background(), serixData, serixTarget) + require.NoError(t, err) + + require.Len(t, serixData, bytesRead) + require.EqualValues(t, test.source, serixTarget) + + // json serialize + sourceJSON, err := testAPI.JSONEncode(context.Background(), test.source, serix.WithValidation()) + require.NoError(t, err) + + // json deserialize + jsonDest := reflect.New(reflect.TypeOf(test.target).Elem()).Interface() + require.NoError(t, testAPI.JSONDecode(context.Background(), sourceJSON, jsonDest, serix.WithValidation())) + + require.EqualValues(t, test.source, jsonDest) +} + +func TestSerixMapSerialize(t *testing.T) { + + type MyMapType map[string]string + + type MapStruct struct { + MyMap MyMapType `serix:"0,lengthPrefixType=uint8,mapMinEntries=2,mapMaxEntries=4,mapMaxByteSize=50,mapKeyLengthPrefixType=uint16,mapKeyMinLen=2,mapKeyMaxLen=5,mapValueLengthPrefixType=uint32,mapValueMinLen=1,mapValueMaxLen=6"` + } + testAPI.RegisterTypeSettings(MapStruct{}, serix.TypeSettings{}) + + tests := []serializeTest{ + { + name: "ok", + source: &MapStruct{ + MyMap: map[string]string{ + "k1": "v1", + "k2": "v2", + }, + }, + target: &MapStruct{}, + size: 21, + seriErr: nil, + }, + { + name: "fail - not enough entries", + source: &MapStruct{ + MyMap: map[string]string{ + "k1": "v1", + }, + }, + target: &MapStruct{}, + size: 0, + seriErr: serix.ErrMapValidationMinElementsNotReached, + }, + { + name: "fail - too many entries", + source: &MapStruct{ + MyMap: map[string]string{ + "k1": "v1", + "k2": "v2", + "k3": "v3", + "k4": "v4", + "k5": "v5", + }, + }, + target: &MapStruct{}, + size: 0, + seriErr: serix.ErrMapValidationMaxElementsExceeded, + }, + { + name: "fail - too big", + source: &MapStruct{ + MyMap: map[string]string{ + "k1": "v1000", + "k2": "v2000", + "k3": "v3000", + "k4": "v4000", + }, + }, + target: &MapStruct{}, + size: 0, + seriErr: serix.ErrMapValidationMaxBytesExceeded, + }, + { + name: "fail - key too short", + source: &MapStruct{ + MyMap: map[string]string{ + "k1": "v1", + "k": "v2", + }, + }, + target: &MapStruct{}, + size: 0, + seriErr: serializer.ErrArrayValidationMinElementsNotReached, + }, + { + name: "fail - key too long", + source: &MapStruct{ + MyMap: map[string]string{ + "k1": "v1", + "k20000": "v2", + }, + }, + target: &MapStruct{}, + size: 0, + seriErr: serializer.ErrArrayValidationMaxElementsExceeded, + }, + { + name: "fail - value too short", + source: &MapStruct{ + MyMap: map[string]string{ + "k1": "v1", + "k2": "", + }, + }, + target: &MapStruct{}, + size: 0, + seriErr: serializer.ErrArrayValidationMinElementsNotReached, + }, + { + name: "fail - value too long", + source: &MapStruct{ + MyMap: map[string]string{ + "k1": "v1", + "k2": "v200000", + }, + }, + target: &MapStruct{}, + size: 0, + seriErr: serializer.ErrArrayValidationMaxElementsExceeded, + }, + } + + for _, tt := range tests { + t.Run(tt.name, tt.run) + } +} + +type deSerializeTest struct { + name string + source any + target any + size int + deSeriErr error +} + +func (test *deSerializeTest) run(t *testing.T) { + // binary serialize test data + serixData, err := testAPI.Encode(context.Background(), test.source, serix.WithValidation()) + require.NoError(t, err) + + // json serialize test data + sourceJSON, err := testAPI.JSONEncode(context.Background(), test.source, serix.WithValidation()) + require.NoError(t, err) + + // binary deserialize + serixTarget := reflect.New(reflect.TypeOf(test.target).Elem()).Interface() + bytesRead, err := testAPI.Decode(context.Background(), serixData, serixTarget, serix.WithValidation()) if test.deSeriErr != nil { require.ErrorIs(t, err, test.deSeriErr) + // we also need to check the json deserialization + jsonDest := reflect.New(reflect.TypeOf(test.target).Elem()).Interface() + err := testAPI.JSONDecode(context.Background(), sourceJSON, jsonDest, serix.WithValidation()) + require.ErrorIs(t, err, test.deSeriErr) + return } require.NoError(t, err) - require.Len(t, serixData, bytesRead) - require.EqualValues(t, test.source, serixTarget) - sourceJSON, err := testAPI.JSONEncode(context.Background(), test.source) - require.NoError(t, err) + require.Equal(t, test.size, bytesRead) + require.EqualValues(t, test.source, serixTarget) + // json deserialize jsonDest := reflect.New(reflect.TypeOf(test.target).Elem()).Interface() - require.NoError(t, testAPI.JSONDecode(context.Background(), sourceJSON, jsonDest)) + require.NoError(t, testAPI.JSONDecode(context.Background(), sourceJSON, jsonDest, serix.WithValidation())) require.EqualValues(t, test.source, jsonDest) } -func TestSerixMap(t *testing.T) { +func TestSerixMapDeserialize(t *testing.T) { type MyMapType map[string]string + // used to create test data + type TestVectorMapStruct struct { + MyMap MyMapType `serix:"0,lengthPrefixType=uint8,mapMinEntries=1,mapMaxEntries=5,mapMaxByteSize=100,mapKeyLengthPrefixType=uint16,mapKeyMinLen=1,mapKeyMaxLen=7,mapValueLengthPrefixType=uint32,mapValueMinLen=0,mapValueMaxLen=10"` + } + testAPI.RegisterTypeSettings(TestVectorMapStruct{}, serix.TypeSettings{}) + type MapStruct struct { MyMap MyMapType `serix:"0,lengthPrefixType=uint8,mapMinEntries=2,mapMaxEntries=4,mapMaxByteSize=50,mapKeyLengthPrefixType=uint16,mapKeyMinLen=2,mapKeyMaxLen=5,mapValueLengthPrefixType=uint32,mapValueMinLen=1,mapValueMaxLen=6"` } - - testAPI.RegisterTypeSettings(MapStruct{}, serix.TypeSettings{}.WithObjectType(uint8(0))) + testAPI.RegisterTypeSettings(MapStruct{}, serix.TypeSettings{}) tests := []deSerializeTest{ { name: "ok", - source: &MapStruct{ + source: &TestVectorMapStruct{ MyMap: map[string]string{ "k1": "v1", "k2": "v2", }, }, target: &MapStruct{}, - size: 22, - seriErr: nil, + size: 21, deSeriErr: nil, }, { name: "fail - not enough entries", - source: &MapStruct{ + source: &TestVectorMapStruct{ MyMap: map[string]string{ "k1": "v1", }, }, target: &MapStruct{}, size: 0, - seriErr: serix.ErrMapValidationMinElementsNotReached, - deSeriErr: nil, + deSeriErr: serix.ErrMapValidationMinElementsNotReached, }, { name: "fail - too many entries", - source: &MapStruct{ + source: &TestVectorMapStruct{ MyMap: map[string]string{ "k1": "v1", "k2": "v2", @@ -222,12 +387,11 @@ func TestSerixMap(t *testing.T) { }, target: &MapStruct{}, size: 0, - seriErr: serix.ErrMapValidationMaxElementsExceeded, - deSeriErr: nil, + deSeriErr: serix.ErrMapValidationMaxElementsExceeded, }, { name: "fail - too big", - source: &MapStruct{ + source: &TestVectorMapStruct{ MyMap: map[string]string{ "k1": "v1000", "k2": "v2000", @@ -237,12 +401,11 @@ func TestSerixMap(t *testing.T) { }, target: &MapStruct{}, size: 0, - seriErr: serix.ErrMapValidationMaxBytesExceeded, - deSeriErr: nil, + deSeriErr: serix.ErrMapValidationMaxBytesExceeded, }, { name: "fail - key too short", - source: &MapStruct{ + source: &TestVectorMapStruct{ MyMap: map[string]string{ "k1": "v1", "k": "v2", @@ -250,12 +413,11 @@ func TestSerixMap(t *testing.T) { }, target: &MapStruct{}, size: 0, - seriErr: serializer.ErrArrayValidationMinElementsNotReached, - deSeriErr: nil, + deSeriErr: serializer.ErrArrayValidationMinElementsNotReached, }, { name: "fail - key too long", - source: &MapStruct{ + source: &TestVectorMapStruct{ MyMap: map[string]string{ "k1": "v1", "k20000": "v2", @@ -263,12 +425,11 @@ func TestSerixMap(t *testing.T) { }, target: &MapStruct{}, size: 0, - seriErr: serializer.ErrArrayValidationMaxElementsExceeded, - deSeriErr: nil, + deSeriErr: serializer.ErrArrayValidationMaxElementsExceeded, }, { name: "fail - value too short", - source: &MapStruct{ + source: &TestVectorMapStruct{ MyMap: map[string]string{ "k1": "v1", "k2": "", @@ -276,12 +437,11 @@ func TestSerixMap(t *testing.T) { }, target: &MapStruct{}, size: 0, - seriErr: serializer.ErrArrayValidationMinElementsNotReached, - deSeriErr: nil, + deSeriErr: serializer.ErrArrayValidationMinElementsNotReached, }, { name: "fail - value too long", - source: &MapStruct{ + source: &TestVectorMapStruct{ MyMap: map[string]string{ "k1": "v1", "k2": "v200000", @@ -289,12 +449,11 @@ func TestSerixMap(t *testing.T) { }, target: &MapStruct{}, size: 0, - seriErr: serializer.ErrArrayValidationMaxElementsExceeded, - deSeriErr: nil, + deSeriErr: serializer.ErrArrayValidationMaxElementsExceeded, }, } for _, tt := range tests { - t.Run(tt.name, tt.deSerialize) + t.Run(tt.name, tt.run) } } From 183bec1f1644bf83d95bd50b2f161cb6a6b3bbaa Mon Sep 17 00:00:00 2001 From: muXxer Date: Tue, 7 Nov 2023 14:13:27 +0100 Subject: [PATCH 09/17] Remove serix "position" and "mapKey", use first position as key --- ds/set_impl.go | 2 +- serializer/serix/map_decode.go | 20 +++--- serializer/serix/map_encode.go | 26 ++++---- serializer/serix/map_encode_test.go | 68 ++++++++++----------- serializer/serix/serix.go | 56 +++++++++-------- serializer/serix/serix_serializable_test.go | 36 +++++------ serializer/serix/serix_test.go | 14 ++--- serializer/serix/type_settings.go | 30 ++++----- 8 files changed, 128 insertions(+), 124 deletions(-) diff --git a/ds/set_impl.go b/ds/set_impl.go index 27c2ea5f2..10af1df31 100644 --- a/ds/set_impl.go +++ b/ds/set_impl.go @@ -145,7 +145,7 @@ func (s *set[ElementType]) apply(mutations SetMutations[ElementType]) (appliedMu // //nolint:tagliatelle // heck knows why this linter fails here type readableSet[T comparable] struct { - *orderedmap.OrderedMap[T, types.Empty] `serix:"0"` + *orderedmap.OrderedMap[T, types.Empty] `serix:","` } // newReadableSet creates a new readable set with the given elements. diff --git a/serializer/serix/map_decode.go b/serializer/serix/map_decode.go index c47a55697..97402a459 100644 --- a/serializer/serix/map_decode.go +++ b/serializer/serix/map_decode.go @@ -102,13 +102,13 @@ func (api *API) mapDecodeBasedOnType(ctx context.Context, mapVal any, value refl return ierrors.Errorf("missing type settings for interface %s", valueType) } - mapKey := mapSliceArrayDefaultKey - if innerTS.mapKey != nil { - mapKey = *innerTS.mapKey + fieldKey := keyDefaultSliceArray + if innerTS.fieldKey != nil { + fieldKey = *innerTS.fieldKey } //nolint:forcetypeassert - fieldValStr := mapVal.(map[string]any)[mapKey].(string) + fieldValStr := mapVal.(map[string]any)[fieldKey].(string) byteSlice, err := DecodeHex(fieldValStr) if err != nil { return ierrors.Wrap(err, "failed to read byte slice from map") @@ -277,7 +277,7 @@ func (api *API) mapDecodeInterface( return ierrors.Errorf("non map[string]any in struct map decode, got %T instead", mapVal) } - objectCodeAny, has := m[mapTypeKeyName] + objectCodeAny, has := m[keyType] if !has { return ierrors.Errorf("no object type defined in map for interface %s", valueType) } @@ -323,7 +323,7 @@ func (api *API) mapDecodeStruct(ctx context.Context, mapVal any, value reflect.V if err != nil { return ierrors.WithStack(err) } - mapObjectCode, has := m[mapTypeKeyName] + mapObjectCode, has := m[keyType] if !has { return ierrors.Wrap(err, "missing type key in struct") } @@ -382,12 +382,12 @@ func (api *API) mapDecodeStructFields( continue } - key := mapStringKey(sField.name) - if sField.settings.ts.mapKey != nil { - key = sField.settings.ts.MustMapKey() + fieldKey := fieldKeyString(sField.name) + if sField.settings.ts.fieldKey != nil { + fieldKey = sField.settings.ts.MustFieldKey() } - mapVal, has := m[key] + mapVal, has := m[fieldKey] if !has { if sField.settings.isOptional || sField.settings.omitEmpty { continue diff --git a/serializer/serix/map_encode.go b/serializer/serix/map_encode.go index 23e5cf780..2a95727a0 100644 --- a/serializer/serix/map_encode.go +++ b/serializer/serix/map_encode.go @@ -16,10 +16,10 @@ import ( ) const ( - // the map key under which the object code is written. - mapTypeKeyName = "type" - // key used when no map key is defined for types which are slice/arrays of bytes. - mapSliceArrayDefaultKey = "data" + // the key under which the object code is written. + keyType = "type" + // the key used when no key is defined for types which are slice/arrays of bytes. + keyDefaultSliceArray = "data" ) var ( @@ -163,7 +163,7 @@ func (api *API) mapEncodeStruct( obj := orderedmap.New() if ts.ObjectType() != nil { - obj.Set(mapTypeKeyName, ts.ObjectType()) + obj.Set(keyType, ts.ObjectType()) } if err := api.mapEncodeStructFields(ctx, obj, value, valueType, opts); err != nil { return nil, ierrors.WithStack(err) @@ -215,8 +215,8 @@ func (api *API) mapEncodeStructFields( } switch { - case sField.settings.ts.mapKey != nil: - obj.Set(*sField.settings.ts.mapKey, eleOut) + case sField.settings.ts.fieldKey != nil: + obj.Set(*sField.settings.ts.fieldKey, eleOut) case sField.settings.nest: castedEleOut, ok := eleOut.(*orderedmap.OrderedMap) if !ok { @@ -227,7 +227,7 @@ func (api *API) mapEncodeStructFields( obj.Set(k, lo.Return1(castedEleOut.Get(k))) } default: - obj.Set(mapStringKey(sField.name), eleOut) + obj.Set(fieldKeyString(sField.name), eleOut) } } @@ -239,12 +239,12 @@ func (api *API) mapEncodeSlice(ctx context.Context, value reflect.Value, valueTy if ts.ObjectType() != nil { m := orderedmap.New() - m.Set(mapTypeKeyName, ts.ObjectType()) - mapKey := mapSliceArrayDefaultKey - if ts.mapKey != nil { - mapKey = *ts.mapKey + m.Set(keyType, ts.ObjectType()) + fieldKey := keyDefaultSliceArray + if ts.fieldKey != nil { + fieldKey = *ts.fieldKey } - m.Set(mapKey, EncodeHex(value.Bytes())) + m.Set(fieldKey, EncodeHex(value.Bytes())) return m, nil } diff --git a/serializer/serix/map_encode_test.go b/serializer/serix/map_encode_test.go index 12326faa9..70c0a1f6f 100644 --- a/serializer/serix/map_encode_test.go +++ b/serializer/serix/map_encode_test.go @@ -30,8 +30,8 @@ func must(err error) { type Identifier [blake2b.Size256]byte type serializableStruct struct { - bytes Identifier `serix:"0"` - index uint64 `serix:"1"` + bytes Identifier `serix:"bytes"` + index uint64 `serix:"index"` } func (s serializableStruct) EncodeJSON() (any, error) { @@ -76,19 +76,19 @@ func TestMapEncodeDecode(t *testing.T) { name: "basic types", paras: func() paras { type example struct { - Uint64 uint64 `serix:"0,mapKey=uint64"` - Uint32 uint32 `serix:"1,mapKey=uint32"` - Uint16 uint16 `serix:"2,mapKey=uint16"` - Uint8 uint8 `serix:"3,mapKey=uint8"` - Int64 int64 `serix:"4,mapKey=int64"` - Int32 int32 `serix:"5,mapKey=int32"` - Int16 int16 `serix:"6,mapKey=int16"` - Int8 int8 `serix:"7,mapKey=int8"` - ZeroInt32 int32 `serix:"8,mapKey=zeroInt32,omitempty"` - Float32 float32 `serix:"9,mapKey=float32"` - Float64 float64 `serix:"10,mapKey=float64"` - String string `serix:"11,mapKey=string"` - Bool bool `serix:"12,mapKey=bool"` + Uint64 uint64 `serix:"uint64"` + Uint32 uint32 `serix:"uint32"` + Uint16 uint16 `serix:"uint16"` + Uint8 uint8 `serix:"uint8"` + Int64 int64 `serix:"int64"` + Int32 int32 `serix:"int32"` + Int16 int16 `serix:"int16"` + Int8 int8 `serix:"int8"` + ZeroInt32 int32 `serix:"zeroInt32,omitempty"` + Float32 float32 `serix:"float32"` + Float64 float64 `serix:"float64"` + String string `serix:"string"` + Bool bool `serix:"bool"` } api := serix.NewAPI() @@ -133,7 +133,7 @@ func TestMapEncodeDecode(t *testing.T) { name: "big int", paras: func() paras { type example struct { - BigInt *big.Int `serix:"0,mapKey=bigInt"` + BigInt *big.Int `serix:"bigInt"` } api := serix.NewAPI() @@ -155,7 +155,7 @@ func TestMapEncodeDecode(t *testing.T) { name: "map", paras: func() paras { type example struct { - Map map[string]string `serix:"0,mapKey=map"` + Map map[string]string `serix:"map"` } api := serix.NewAPI() @@ -182,10 +182,10 @@ func TestMapEncodeDecode(t *testing.T) { paras: func() paras { type example struct { - ByteSlice []byte `serix:"0,mapKey=byteSlice"` - Array [5]byte `serix:"1,mapKey=array"` - SliceOfByteSlices [][]byte `serix:"3,mapKey=sliceOfByteSlices"` - SliceOfByteArrays [][3]byte `serix:"4,mapKey=sliceOfByteArrays"` + ByteSlice []byte `serix:"byteSlice"` + Array [5]byte `serix:"array"` + SliceOfByteSlices [][]byte `serix:"sliceOfByteSlices"` + SliceOfByteArrays [][3]byte `serix:"sliceOfByteArrays"` } api := serix.NewAPI() @@ -226,11 +226,11 @@ func TestMapEncodeDecode(t *testing.T) { paras: func() paras { type ( inner struct { - String string `serix:"0,mapKey=string"` + String string `serix:"string"` } example struct { - inner `serix:"0"` + inner `serix:"inner"` } ) @@ -258,19 +258,19 @@ func TestMapEncodeDecode(t *testing.T) { OtherObj [2]byte example struct { - Interface InterfaceType `serix:"0,mapKey=interface"` - Other *OtherObj `serix:"1,mapKey=other"` + Interface InterfaceType `serix:"interface"` + Other *OtherObj `serix:"other"` } ) api := serix.NewAPI() must(api.RegisterTypeSettings(example{}, serix.TypeSettings{}.WithObjectType(uint8(33)))) must(api.RegisterTypeSettings(InterfaceTypeImpl1{}, - serix.TypeSettings{}.WithObjectType(uint8(5)).WithMapKey("customInnerKey")), + serix.TypeSettings{}.WithObjectType(uint8(5)).WithFieldKey("customInnerKey")), ) must(api.RegisterInterfaceObjects((*InterfaceType)(nil), (*InterfaceTypeImpl1)(nil))) must(api.RegisterTypeSettings(OtherObj{}, - serix.TypeSettings{}.WithObjectType(uint8(2)).WithMapKey("otherObjKey")), + serix.TypeSettings{}.WithObjectType(uint8(2)).WithFieldKey("otherObjKey")), ) return paras{ @@ -299,14 +299,14 @@ func TestMapEncodeDecode(t *testing.T) { type ( Interface interface{} Impl1 struct { - String string `serix:"0,mapKey=string"` + String string `serix:"string"` } Impl2 struct { - Uint16 uint16 `serix:"0,mapKey=uint16"` + Uint16 uint16 `serix:"uint16"` } example struct { - Slice []Interface `serix:"0,mapKey=slice"` + Slice []Interface `serix:"slice"` } ) @@ -344,8 +344,8 @@ func TestMapEncodeDecode(t *testing.T) { name: "no map key", paras: func() paras { type example struct { - CaptainHook string `serix:"0"` - LiquidSoul int64 `serix:"1"` + CaptainHook string `serix:"captainHook"` + LiquidSoul int64 `serix:"liquidSoul"` } api := serix.NewAPI() @@ -369,7 +369,7 @@ func TestMapEncodeDecode(t *testing.T) { name: "time", paras: func() paras { type example struct { - CreationDate time.Time `serix:"0"` + CreationDate time.Time `serix:"creationDate"` } api := serix.NewAPI() @@ -396,7 +396,7 @@ func TestMapEncodeDecode(t *testing.T) { name: "serializable", paras: func() paras { type example struct { - Entries map[serializableStruct]struct{} `serix:"0"` + Entries map[serializableStruct]struct{} `serix:"entries"` } api := serix.NewAPI() diff --git a/serializer/serix/serix.go b/serializer/serix/serix.go index b270a267e..656efdc9c 100644 --- a/serializer/serix/serix.go +++ b/serializer/serix/serix.go @@ -3,12 +3,12 @@ Structs serialization/deserialization -In order for a field to be detected by serix it must have `serix` struct tag set with the position index: `serix:"0"`. -serix traverses all fields and handles them in the order specified in the struct tags. +In order for a field to be detected by serix it must have `serix` struct tag set with the name: `serix:"example" or empty name `serix:","`. +serix traverses all fields and handles them in the order specified in the struct. Apart from the required position you can provide the following settings to serix via struct tags: -"optional" - means that field might be nil. Only valid for pointers or interfaces: `serix:"1,optional"` -"lengthPrefixType=uint32" - provide serializer.SeriLengthPrefixType for that field: `serix:"2,lengthPrefixType=unint32"` -"nest" - handle embedded/anonymous field as a nested field: `serix:"3,nest"` +"optional" - means that field might be nil. Only valid for pointers or interfaces: `serix:"example,optional"` +"lengthPrefixType=uint32" - provide serializer.SeriLengthPrefixType for that field: `serix:"example,lengthPrefixType=unint32"` +"nest" - handle embedded/anonymous field as a nested field: `serix:"example,nest"` See serix_text.go for more detail. */ package serix @@ -698,30 +698,33 @@ func (api *API) parseStructType(structType reflect.Type) ([]structField, error) } structFields = make([]structField, 0, structType.NumField()) - seenPositions := make(map[int]struct{}) + + serixPosition := 0 for i := 0; i < structType.NumField(); i++ { field := structType.Field(i) + isUnexported := field.PkgPath != "" isEmbedded := field.Anonymous isStruct := isUnderlyingStruct(field.Type) isInterface := isUnderlyingInterface(field.Type) isEmbeddedStruct := isEmbedded && isStruct isEmbeddedInterface := isEmbedded && isInterface + if isUnexported && !isEmbeddedStruct { continue } + tag, ok := field.Tag.Lookup("serix") if !ok { continue } - tSettings, err := parseStructTag(tag) + + tSettings, err := parseStructTag(tag, serixPosition) if err != nil { return nil, ierrors.Wrapf(err, "failed to parse struct tag %s for field %s", tag, field.Name) } - if _, exists := seenPositions[tSettings.position]; exists { - return nil, ierrors.Errorf("struct field with duplicated position number %d", tSettings.position) - } - seenPositions[tSettings.position] = struct{}{} + serixPosition++ + if tSettings.isOptional { if field.Type.Kind() != reflect.Ptr && field.Type.Kind() != reflect.Interface { return nil, ierrors.Errorf( @@ -729,27 +732,32 @@ func (api *API) parseStructType(structType reflect.Type) ([]structField, error) "'optional' setting can only be used with pointers or interfaces, got %s", field.Name, field.Type.Kind()) } + if isEmbeddedStruct { return nil, ierrors.Errorf( "struct field %s is invalid: 'optional' setting can't be used with embedded structs", field.Name) } + if isEmbeddedInterface { return nil, ierrors.Errorf( "struct field %s is invalid: 'optional' setting can't be used with embedded interfaces", field.Name) } } + if tSettings.nest && isUnexported { return nil, ierrors.Errorf( "struct field %s is invalid: 'nest' setting can't be used with unexported types", field.Name) } + if !tSettings.nest && isEmbeddedInterface { return nil, ierrors.Errorf( "struct field %s is invalid: 'nest' setting needs to be used for embedded interfaces", field.Name) } + structFields = append(structFields, structField{ name: field.Name, isUnexported: isUnexported, @@ -820,21 +828,24 @@ func parseStructTagValuePrefixType(name string, keyValue []string, currentPart s return lengthPrefixType, nil } -func parseStructTag(tag string) (tagSettings, error) { +func parseStructTag(tag string, serixPosition int) (tagSettings, error) { if tag == "" { return tagSettings{}, ierrors.New("struct tag is empty") } + parts := strings.Split(tag, ",") - positionPart := parts[0] - position, err := strconv.Atoi(positionPart) - if err != nil { - return tagSettings{}, ierrors.Wrap(err, "failed to parse position number from the first part of the tag") + keyPart := parts[0] + + if strings.ContainsAny(keyPart, "=") { + return tagSettings{}, ierrors.Errorf("incorrect struct tag format: %s, must start with the field key or \",\"", tag) } + settings := tagSettings{} - settings.position = position + settings.position = serixPosition + settings.ts = settings.ts.WithFieldKey(keyPart) + parts = parts[1:] seenParts := map[string]struct{}{} - for _, currentPart := range parts { if _, ok := seenParts[currentPart]; ok { return tagSettings{}, ierrors.Errorf("duplicated tag part: %s", currentPart) @@ -852,13 +863,6 @@ func parseStructTag(tag string) (tagSettings, error) { case "omitempty": settings.omitEmpty = true - case "mapKey": - value, err := parseStructTagValue("mapKey", keyValue, currentPart) - if err != nil { - return tagSettings{}, err - } - settings.ts = settings.ts.WithMapKey(value) - case "minLen": value, err := parseStructTagValueUint("minLen", keyValue, currentPart) if err != nil { @@ -1028,6 +1032,6 @@ func getNumberTypeToConvert(kind reflect.Kind) (int, reflect.Type, reflect.Type) return bitSize, numberType, reflect.PointerTo(numberType) } -func mapStringKey(str string) string { +func fieldKeyString(str string) string { return strings.ToLower(str[:1]) + str[1:] } diff --git a/serializer/serix/serix_serializable_test.go b/serializer/serix/serix_serializable_test.go index 95a3c0651..82113b122 100644 --- a/serializer/serix/serix_serializable_test.go +++ b/serializer/serix/serix_serializable_test.go @@ -80,15 +80,15 @@ func (bs Bools) FromSerializables(seris serializer.Serializables) { } type SimpleStruct struct { - Bool bool `serix:"0"` - Uint uint64 `serix:"1"` - String string `serix:"2,lengthPrefixType=uint16"` - Bytes []byte `serix:"3,lengthPrefixType=uint32"` - BytesArray [16]byte `serix:"4"` - BigInt *big.Int `serix:"5"` - Time time.Time `serix:"6"` - Int uint64 `serix:"7"` - Float float64 `serix:"8"` + Bool bool `serix:"bool"` + Uint uint64 `serix:"uint"` + String string `serix:"string,lengthPrefixType=uint16"` + Bytes []byte `serix:"bytes,lengthPrefixType=uint32"` + BytesArray [16]byte `serix:"bytesarray"` + BigInt *big.Int `serix:"bigint"` + Time time.Time `serix:"time"` + Int uint64 `serix:"int"` + Float float64 `serix:"float"` } func NewSimpleStruct() SimpleStruct { @@ -146,7 +146,7 @@ type Interface interface { } type InterfaceImpl struct { - interfaceImpl `serix:"0"` + interfaceImpl `serix:"interfaceImpl"` } func (ii *InterfaceImpl) SetDeserializationContext(ctx context.Context) { @@ -154,8 +154,8 @@ func (ii *InterfaceImpl) SetDeserializationContext(ctx context.Context) { } type interfaceImpl struct { - A uint8 `serix:"0"` - B uint8 `serix:"1"` + A uint8 `serix:"a"` + B uint8 `serix:"b"` } func (ii *InterfaceImpl) Encode() ([]byte, error) { @@ -195,7 +195,7 @@ func (ii *InterfaceImpl) Serialize(deSeriMode serializer.DeSerializationMode, de } type StructWithInterface struct { - Interface Interface `serix:"0"` + Interface Interface `serix:"interface"` } func (si StructWithInterface) MarshalJSON() ([]byte, error) { @@ -221,7 +221,7 @@ func (si StructWithInterface) Serialize(deSeriMode serializer.DeSerializationMod } type StructWithOptionalField struct { - Optional *ExportedStruct `serix:"0,optional"` + Optional *ExportedStruct `serix:"optional,optional"` } func (so StructWithOptionalField) MarshalJSON() ([]byte, error) { @@ -247,8 +247,8 @@ func (so StructWithOptionalField) Serialize(deSeriMode serializer.DeSerializatio } type StructWithEmbeddedStructs struct { - unexportedStruct `serix:"0"` - ExportedStruct `serix:"1,nest"` + unexportedStruct `serix:"unexportedstruct"` + ExportedStruct `serix:"exportedstruct,nest"` } func (se StructWithEmbeddedStructs) MarshalJSON() ([]byte, error) { @@ -276,11 +276,11 @@ func (se StructWithEmbeddedStructs) Serialize(deSeriMode serializer.DeSerializat } type unexportedStruct struct { - Foo uint64 `serix:"0"` + Foo uint64 `serix:"foo"` } type ExportedStruct struct { - Bar uint64 `serix:"0"` + Bar uint64 `serix:"bar"` } func (e ExportedStruct) SetDeserializationContext(ctx context.Context) { diff --git a/serializer/serix/serix_test.go b/serializer/serix/serix_test.go index d12b37a28..79fdaa147 100644 --- a/serializer/serix/serix_test.go +++ b/serializer/serix/serix_test.go @@ -39,7 +39,7 @@ func TestMinMax(t *testing.T) { name: "ok - string in bounds", paras: func() paras { type example struct { - Str string `serix:"0,minLen=5,maxLen=10,lengthPrefixType=uint8"` + Str string `serix:"str,minLen=5,maxLen=10,lengthPrefixType=uint8"` } api := serix.NewAPI() @@ -57,7 +57,7 @@ func TestMinMax(t *testing.T) { name: "err - string out of bounds", paras: func() paras { type example struct { - Str string `serix:"0,minLen=5,maxLen=10,lengthPrefixType=uint8"` + Str string `serix:"str,minLen=5,maxLen=10,lengthPrefixType=uint8"` } api := serix.NewAPI() @@ -75,7 +75,7 @@ func TestMinMax(t *testing.T) { name: "ok - slice in bounds", paras: func() paras { type example struct { - Slice []byte `serix:"0,minLen=0,maxLen=10,lengthPrefixType=uint8"` + Slice []byte `serix:"slice,minLen=0,maxLen=10,lengthPrefixType=uint8"` } api := serix.NewAPI() @@ -93,7 +93,7 @@ func TestMinMax(t *testing.T) { name: "err - slice out of bounds", paras: func() paras { type example struct { - Slice []byte `serix:"0,minLen=0,maxLen=3,lengthPrefixType=uint8"` + Slice []byte `serix:"slice,minLen=0,maxLen=3,lengthPrefixType=uint8"` } api := serix.NewAPI() @@ -181,7 +181,7 @@ func TestSerixMapSerialize(t *testing.T) { type MyMapType map[string]string type MapStruct struct { - MyMap MyMapType `serix:"0,lengthPrefixType=uint8,mapMinEntries=2,mapMaxEntries=4,mapMaxByteSize=50,mapKeyLengthPrefixType=uint16,mapKeyMinLen=2,mapKeyMaxLen=5,mapValueLengthPrefixType=uint32,mapValueMinLen=1,mapValueMaxLen=6"` + MyMap MyMapType `serix:"myMap,lengthPrefixType=uint8,mapMinEntries=2,mapMaxEntries=4,mapMaxByteSize=50,mapKeyLengthPrefixType=uint16,mapKeyMinLen=2,mapKeyMaxLen=5,mapValueLengthPrefixType=uint32,mapValueMinLen=1,mapValueMaxLen=6"` } testAPI.RegisterTypeSettings(MapStruct{}, serix.TypeSettings{}) @@ -341,12 +341,12 @@ func TestSerixMapDeserialize(t *testing.T) { // used to create test data type TestVectorMapStruct struct { - MyMap MyMapType `serix:"0,lengthPrefixType=uint8,mapMinEntries=1,mapMaxEntries=5,mapMaxByteSize=100,mapKeyLengthPrefixType=uint16,mapKeyMinLen=1,mapKeyMaxLen=7,mapValueLengthPrefixType=uint32,mapValueMinLen=0,mapValueMaxLen=10"` + MyMap MyMapType `serix:"myMap,lengthPrefixType=uint8,mapMinEntries=1,mapMaxEntries=5,mapMaxByteSize=100,mapKeyLengthPrefixType=uint16,mapKeyMinLen=1,mapKeyMaxLen=7,mapValueLengthPrefixType=uint32,mapValueMinLen=0,mapValueMaxLen=10"` } testAPI.RegisterTypeSettings(TestVectorMapStruct{}, serix.TypeSettings{}) type MapStruct struct { - MyMap MyMapType `serix:"0,lengthPrefixType=uint8,mapMinEntries=2,mapMaxEntries=4,mapMaxByteSize=50,mapKeyLengthPrefixType=uint16,mapKeyMinLen=2,mapKeyMaxLen=5,mapValueLengthPrefixType=uint32,mapValueMinLen=1,mapValueMaxLen=6"` + MyMap MyMapType `serix:"myMap,lengthPrefixType=uint8,mapMinEntries=2,mapMaxEntries=4,mapMaxByteSize=50,mapKeyLengthPrefixType=uint16,mapKeyMinLen=2,mapKeyMaxLen=5,mapValueLengthPrefixType=uint32,mapValueMinLen=1,mapValueMaxLen=6"` } testAPI.RegisterTypeSettings(MapStruct{}, serix.TypeSettings{}) diff --git a/serializer/serix/type_settings.go b/serializer/serix/type_settings.go index 7009967bb..6a9c1092d 100644 --- a/serializer/serix/type_settings.go +++ b/serializer/serix/type_settings.go @@ -82,10 +82,10 @@ type MapRules struct { // So the precedence is the following 1<2<3. // See API.RegisterTypeSettings() and WithTypeSettings() for more detail. type TypeSettings struct { + fieldKey *string lengthPrefixType *LengthPrefixType objectType interface{} lexicalOrdering *bool - mapKey *string arrayRules *ArrayRules mapRules *MapRules } @@ -137,29 +137,29 @@ func (ts TypeSettings) LexicalOrdering() (val bool, set bool) { return *ts.lexicalOrdering, true } -// WithMapKey specifies the name for the map key. -func (ts TypeSettings) WithMapKey(name string) TypeSettings { - ts.mapKey = &name +// WithFieldKey specifies the key for the field. +func (ts TypeSettings) WithFieldKey(fieldKey string) TypeSettings { + ts.fieldKey = &fieldKey return ts } -// MapKey returns the map key name. -func (ts TypeSettings) MapKey() (string, bool) { - if ts.mapKey == nil { +// FieldKey returns the key for the field. +func (ts TypeSettings) FieldKey() (string, bool) { + if ts.fieldKey == nil { return "", false } - return *ts.mapKey, true + return *ts.fieldKey, true } -// MustMapKey must return a map key name. -func (ts TypeSettings) MustMapKey() string { - if ts.mapKey == nil { - panic("no map key set") +// MustFieldKey must return a key for the field. +func (ts TypeSettings) MustFieldKey() string { + if ts.fieldKey == nil { + panic("no field key set") } - return *ts.mapKey + return *ts.fieldKey } // WithArrayRules specifies serializer.ArrayRules. @@ -371,8 +371,8 @@ func (ts TypeSettings) merge(other TypeSettings) TypeSettings { if ts.arrayRules == nil { ts.arrayRules = other.arrayRules } - if ts.mapKey == nil { - ts.mapKey = other.mapKey + if ts.fieldKey == nil { + ts.fieldKey = other.fieldKey } if ts.mapRules == nil { ts.mapRules = other.mapRules From 0548cbfbbd10eb88ab50c2684ff8689af684116e Mon Sep 17 00:00:00 2001 From: muXxer Date: Tue, 7 Nov 2023 18:35:11 +0100 Subject: [PATCH 10/17] Rename struct tags and unify map and array rules --- serializer/serix/decode.go | 2 +- serializer/serix/encode.go | 5 +- serializer/serix/map_decode.go | 4 +- serializer/serix/map_encode.go | 6 +- serializer/serix/serix.go | 72 ++++++---------- serializer/serix/serix_serializable_test.go | 4 +- serializer/serix/serix_test.go | 94 ++++++++++++++++++--- serializer/serix/type_settings.go | 74 ++++++---------- 8 files changed, 143 insertions(+), 118 deletions(-) diff --git a/serializer/serix/decode.go b/serializer/serix/decode.go index cd785ba53..1a6f1b545 100644 --- a/serializer/serix/decode.go +++ b/serializer/serix/decode.go @@ -472,7 +472,7 @@ func (api *API) decodeMap(ctx context.Context, b []byte, value reflect.Value, return consumedBytes, err } - if err := api.checkMapMinMaxBounds(value.Len(), ts); err != nil { + if err := api.checkMinMaxBounds(value, ts); err != nil { return consumedBytes, err } diff --git a/serializer/serix/encode.go b/serializer/serix/encode.go index 51e873013..759c047b3 100644 --- a/serializer/serix/encode.go +++ b/serializer/serix/encode.go @@ -342,13 +342,12 @@ func (api *API) encodeMapKVPair(ctx context.Context, key, val reflect.Value, ts func (api *API) encodeMap(ctx context.Context, value reflect.Value, valueType reflect.Type, ts TypeSettings, opts *options) ([]byte, error) { - size := value.Len() - if err := api.checkMapMinMaxBounds(size, ts); err != nil { + if err := api.checkMinMaxBounds(value, ts); err != nil { return nil, err } - data := make([][]byte, size) + data := make([][]byte, value.Len()) iter := value.MapRange() for i := 0; iter.Next(); i++ { key := iter.Key() diff --git a/serializer/serix/map_decode.go b/serializer/serix/map_decode.go index 97402a459..282c404ac 100644 --- a/serializer/serix/map_decode.go +++ b/serializer/serix/map_decode.go @@ -382,7 +382,7 @@ func (api *API) mapDecodeStructFields( continue } - fieldKey := fieldKeyString(sField.name) + fieldKey := FieldKeyString(sField.name) if sField.settings.ts.fieldKey != nil { fieldKey = sField.settings.ts.MustFieldKey() } @@ -484,7 +484,7 @@ func (api *API) mapDecodeMap(ctx context.Context, mapVal any, value reflect.Valu value.SetMapIndex(keyValue, elemValue) } - if err := api.checkMapMinMaxBounds(value.Len(), ts); err != nil { + if err := api.checkMinMaxBounds(value, ts); err != nil { return err } diff --git a/serializer/serix/map_encode.go b/serializer/serix/map_encode.go index 2a95727a0..79dfe75b1 100644 --- a/serializer/serix/map_encode.go +++ b/serializer/serix/map_encode.go @@ -227,7 +227,7 @@ func (api *API) mapEncodeStructFields( obj.Set(k, lo.Return1(castedEleOut.Get(k))) } default: - obj.Set(fieldKeyString(sField.name), eleOut) + obj.Set(FieldKeyString(sField.name), eleOut) } } @@ -299,9 +299,7 @@ func (api *API) mapEncodeMapKVPair(ctx context.Context, key, val reflect.Value, } func (api *API) mapEncodeMap(ctx context.Context, value reflect.Value, ts TypeSettings, opts *options) (*orderedmap.OrderedMap, error) { - size := value.Len() - - if err := api.checkMapMinMaxBounds(size, ts); err != nil { + if err := api.checkMinMaxBounds(value, ts); err != nil { return nil, err } diff --git a/serializer/serix/serix.go b/serializer/serix/serix.go index 656efdc9c..460a37b70 100644 --- a/serializer/serix/serix.go +++ b/serializer/serix/serix.go @@ -7,7 +7,7 @@ In order for a field to be detected by serix it must have `serix` struct tag set serix traverses all fields and handles them in the order specified in the struct. Apart from the required position you can provide the following settings to serix via struct tags: "optional" - means that field might be nil. Only valid for pointers or interfaces: `serix:"example,optional"` -"lengthPrefixType=uint32" - provide serializer.SeriLengthPrefixType for that field: `serix:"example,lengthPrefixType=unint32"` +"lenPrefix=uint32" - provide serializer.SeriLengthPrefixType for that field: `serix:"example,lenPrefix=unint32"` "nest" - handle embedded/anonymous field as a nested field: `serix:"example,nest"` See serix_text.go for more detail. */ @@ -31,10 +31,6 @@ import ( ) var ( - // ErrMapValidationMinElementsNotReached gets returned if the count of elements is too small. - ErrMapValidationMinElementsNotReached = ierrors.New("min count of elements within the map not reached") - // ErrMapValidationMaxElementsExceeded gets returned if the count of elements is too big. - ErrMapValidationMaxElementsExceeded = ierrors.New("max count of elements within the map exceeded") // ErrMapValidationMaxBytesExceeded gets returned if the serialized byte size of the map is too big. ErrMapValidationMaxBytesExceeded = ierrors.New("max bytes size of the map exceeded") // ErrMapValidationViolatesUniqueness gets returned if the map elements are not unique. @@ -204,21 +200,6 @@ func (api *API) checkMinMaxBounds(v reflect.Value, ts TypeSettings) error { return nil } -// checkMapMinMaxBounds checks whether the given map is within its defined bounds in case it has defined map rules. -func (api *API) checkMapMinMaxBounds(length int, ts TypeSettings) error { - if ts.mapRules != nil { - switch { - case ts.mapRules.MaxEntries > 0 && uint(length) > ts.mapRules.MaxEntries: - return ierrors.Wrapf(ErrMapValidationMaxElementsExceeded, "map (len %d) exceeds max length of %d ", length, ts.mapRules.MaxEntries) - - case ts.mapRules.MinEntries > 0 && uint(length) < ts.mapRules.MinEntries: - return ierrors.Wrapf(ErrMapValidationMinElementsNotReached, "map (len %d) is less than min length of %d ", length, ts.mapRules.MinEntries) - } - } - - return nil -} - // checkMapMaxByteSize checks whether the given map is within its defined max byte size in case it has defined map rules. func (api *API) checkMapMaxByteSize(byteSize int, ts TypeSettings) error { if ts.mapRules != nil && ts.mapRules.MaxByteSize > 0 && byteSize > int(ts.mapRules.MaxByteSize) { @@ -863,6 +844,13 @@ func parseStructTag(tag string, serixPosition int) (tagSettings, error) { case "omitempty": settings.omitEmpty = true + case "lenPrefix": + value, err := parseStructTagValuePrefixType("lenPrefix", keyValue, currentPart) + if err != nil { + return tagSettings{}, err + } + settings.ts = settings.ts.WithLengthPrefixType(value) + case "minLen": value, err := parseStructTagValueUint("minLen", keyValue, currentPart) if err != nil { @@ -877,27 +865,6 @@ func parseStructTag(tag string, serixPosition int) (tagSettings, error) { } settings.ts = settings.ts.WithMaxLen(value) - case "lengthPrefixType": - value, err := parseStructTagValuePrefixType("lengthPrefixType", keyValue, currentPart) - if err != nil { - return tagSettings{}, err - } - settings.ts = settings.ts.WithLengthPrefixType(value) - - case "mapMinEntries": - value, err := parseStructTagValueUint("mapMinEntries", keyValue, currentPart) - if err != nil { - return tagSettings{}, err - } - settings.ts = settings.ts.WithMapMinEntries(value) - - case "mapMaxEntries": - value, err := parseStructTagValueUint("mapMaxEntries", keyValue, currentPart) - if err != nil { - return tagSettings{}, err - } - settings.ts = settings.ts.WithMapMaxEntries(value) - case "mapMaxByteSize": value, err := parseStructTagValueUint("mapMaxByteSize", keyValue, currentPart) if err != nil { @@ -905,8 +872,8 @@ func parseStructTag(tag string, serixPosition int) (tagSettings, error) { } settings.ts = settings.ts.WithMapMaxByteSize(value) - case "mapKeyLengthPrefixType": - value, err := parseStructTagValuePrefixType("mapKeyLengthPrefixType", keyValue, currentPart) + case "mapKeyLenPrefix": + value, err := parseStructTagValuePrefixType("mapKeyLenPrefix", keyValue, currentPart) if err != nil { return tagSettings{}, err } @@ -926,8 +893,8 @@ func parseStructTag(tag string, serixPosition int) (tagSettings, error) { } settings.ts = settings.ts.WithMapKeyMaxLen(value) - case "mapValueLengthPrefixType": - value, err := parseStructTagValuePrefixType("mapValueLengthPrefixType", keyValue, currentPart) + case "mapValueLenPrefix": + value, err := parseStructTagValuePrefixType("mapValueLenPrefix", keyValue, currentPart) if err != nil { return tagSettings{}, err } @@ -950,6 +917,7 @@ func parseStructTag(tag string, serixPosition int) (tagSettings, error) { default: return tagSettings{}, ierrors.Errorf("unknown tag part: %s", currentPart) } + seenParts[partName] = struct{}{} } @@ -1032,6 +1000,18 @@ func getNumberTypeToConvert(kind reflect.Kind) (int, reflect.Type, reflect.Type) return bitSize, numberType, reflect.PointerTo(numberType) } -func fieldKeyString(str string) string { +// FieldKeyString converts the given string to camelCase. +// Special keywords like ID or URL are converted to only first letter upper case. +func FieldKeyString(str string) string { + for _, keyword := range []string{"ID", "URL"} { + if !strings.Contains(str, keyword) { + continue + } + + // first keyword letter upper case, rest lower case + str = strings.ReplaceAll(str, keyword, string(keyword[0])+strings.ToLower(keyword)[1:]) + } + + // first letter lower case return strings.ToLower(str[:1]) + str[1:] } diff --git a/serializer/serix/serix_serializable_test.go b/serializer/serix/serix_serializable_test.go index 82113b122..278149eee 100644 --- a/serializer/serix/serix_serializable_test.go +++ b/serializer/serix/serix_serializable_test.go @@ -82,8 +82,8 @@ func (bs Bools) FromSerializables(seris serializer.Serializables) { type SimpleStruct struct { Bool bool `serix:"bool"` Uint uint64 `serix:"uint"` - String string `serix:"string,lengthPrefixType=uint16"` - Bytes []byte `serix:"bytes,lengthPrefixType=uint32"` + String string `serix:"string,lenPrefix=uint16"` + Bytes []byte `serix:"bytes,lenPrefix=uint32"` BytesArray [16]byte `serix:"bytesarray"` BigInt *big.Int `serix:"bigint"` Time time.Time `serix:"time"` diff --git a/serializer/serix/serix_test.go b/serializer/serix/serix_test.go index 79fdaa147..858499e26 100644 --- a/serializer/serix/serix_test.go +++ b/serializer/serix/serix_test.go @@ -39,7 +39,7 @@ func TestMinMax(t *testing.T) { name: "ok - string in bounds", paras: func() paras { type example struct { - Str string `serix:"str,minLen=5,maxLen=10,lengthPrefixType=uint8"` + Str string `serix:"str,minLen=5,maxLen=10,lenPrefix=uint8"` } api := serix.NewAPI() @@ -57,7 +57,7 @@ func TestMinMax(t *testing.T) { name: "err - string out of bounds", paras: func() paras { type example struct { - Str string `serix:"str,minLen=5,maxLen=10,lengthPrefixType=uint8"` + Str string `serix:"str,minLen=5,maxLen=10,lenPrefix=uint8"` } api := serix.NewAPI() @@ -75,7 +75,7 @@ func TestMinMax(t *testing.T) { name: "ok - slice in bounds", paras: func() paras { type example struct { - Slice []byte `serix:"slice,minLen=0,maxLen=10,lengthPrefixType=uint8"` + Slice []byte `serix:"slice,minLen=0,maxLen=10,lenPrefix=uint8"` } api := serix.NewAPI() @@ -93,7 +93,7 @@ func TestMinMax(t *testing.T) { name: "err - slice out of bounds", paras: func() paras { type example struct { - Slice []byte `serix:"slice,minLen=0,maxLen=3,lengthPrefixType=uint8"` + Slice []byte `serix:"slice,minLen=0,maxLen=3,lenPrefix=uint8"` } api := serix.NewAPI() @@ -181,7 +181,7 @@ func TestSerixMapSerialize(t *testing.T) { type MyMapType map[string]string type MapStruct struct { - MyMap MyMapType `serix:"myMap,lengthPrefixType=uint8,mapMinEntries=2,mapMaxEntries=4,mapMaxByteSize=50,mapKeyLengthPrefixType=uint16,mapKeyMinLen=2,mapKeyMaxLen=5,mapValueLengthPrefixType=uint32,mapValueMinLen=1,mapValueMaxLen=6"` + MyMap MyMapType `serix:"myMap,lenPrefix=uint8,minLen=2,maxLen=4,mapMaxByteSize=50,mapKeyLenPrefix=uint16,mapKeyMinLen=2,mapKeyMaxLen=5,mapValueLenPrefix=uint32,mapValueMinLen=1,mapValueMaxLen=6"` } testAPI.RegisterTypeSettings(MapStruct{}, serix.TypeSettings{}) @@ -207,7 +207,7 @@ func TestSerixMapSerialize(t *testing.T) { }, target: &MapStruct{}, size: 0, - seriErr: serix.ErrMapValidationMinElementsNotReached, + seriErr: serializer.ErrArrayValidationMinElementsNotReached, }, { name: "fail - too many entries", @@ -222,7 +222,7 @@ func TestSerixMapSerialize(t *testing.T) { }, target: &MapStruct{}, size: 0, - seriErr: serix.ErrMapValidationMaxElementsExceeded, + seriErr: serializer.ErrArrayValidationMaxElementsExceeded, }, { name: "fail - too big", @@ -341,12 +341,12 @@ func TestSerixMapDeserialize(t *testing.T) { // used to create test data type TestVectorMapStruct struct { - MyMap MyMapType `serix:"myMap,lengthPrefixType=uint8,mapMinEntries=1,mapMaxEntries=5,mapMaxByteSize=100,mapKeyLengthPrefixType=uint16,mapKeyMinLen=1,mapKeyMaxLen=7,mapValueLengthPrefixType=uint32,mapValueMinLen=0,mapValueMaxLen=10"` + MyMap MyMapType `serix:"myMap,lenPrefix=uint8,minLen=1,maxLen=5,mapMaxByteSize=100,mapKeyLenPrefix=uint16,mapKeyMinLen=1,mapKeyMaxLen=7,mapValueLenPrefix=uint32,mapValueMinLen=0,mapValueMaxLen=10"` } testAPI.RegisterTypeSettings(TestVectorMapStruct{}, serix.TypeSettings{}) type MapStruct struct { - MyMap MyMapType `serix:"myMap,lengthPrefixType=uint8,mapMinEntries=2,mapMaxEntries=4,mapMaxByteSize=50,mapKeyLengthPrefixType=uint16,mapKeyMinLen=2,mapKeyMaxLen=5,mapValueLengthPrefixType=uint32,mapValueMinLen=1,mapValueMaxLen=6"` + MyMap MyMapType `serix:"myMap,lenPrefix=uint8,minLen=2,maxLen=4,mapMaxByteSize=50,mapKeyLenPrefix=uint16,mapKeyMinLen=2,mapKeyMaxLen=5,mapValueLenPrefix=uint32,mapValueMinLen=1,mapValueMaxLen=6"` } testAPI.RegisterTypeSettings(MapStruct{}, serix.TypeSettings{}) @@ -372,7 +372,7 @@ func TestSerixMapDeserialize(t *testing.T) { }, target: &MapStruct{}, size: 0, - deSeriErr: serix.ErrMapValidationMinElementsNotReached, + deSeriErr: serializer.ErrArrayValidationMinElementsNotReached, }, { name: "fail - too many entries", @@ -387,7 +387,7 @@ func TestSerixMapDeserialize(t *testing.T) { }, target: &MapStruct{}, size: 0, - deSeriErr: serix.ErrMapValidationMaxElementsExceeded, + deSeriErr: serializer.ErrArrayValidationMaxElementsExceeded, }, { name: "fail - too big", @@ -457,3 +457,75 @@ func TestSerixMapDeserialize(t *testing.T) { t.Run(tt.name, tt.run) } } + +func TestSerixFieldKeyString(t *testing.T) { + type test struct { + name string + source string + target string + } + + tests := []*test{ + { + name: "single char", + source: "A", + target: "a", + }, + { + name: "all upper case", + source: "MYTEST", + target: "mYTEST", + }, + { + name: "all lower case", + source: "mytest", + target: "mytest", + }, + { + name: "mixed case", + source: "MyTest", + target: "myTest", + }, + { + name: "mixed case with numbers", + source: "MyTest123", + target: "myTest123", + }, + { + name: "mixed case with numbers and underscore", + source: "MyTest_123", + target: "myTest_123", + }, + { + name: "mixed case with numbers and underscore and dash", + source: "MyTest_123-", + target: "myTest_123-", + }, + { + name: "mixed case with special keyword 'id'", + source: "MyTestID", + target: "myTestId", + }, + { + name: "mixed case with special keyword 'URL'", + source: "MyTestURL", + target: "myTestUrl", + }, + { + name: "mixed case with special keyword 'ID' but lowercase", + source: "MyTestid", + target: "myTestid", + }, + { + name: "mixed case with special keyword 'URL' but lowercase", + source: "MyTesturl", + target: "myTesturl", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.target, serix.FieldKeyString(tt.source)) + }) + } +} diff --git a/serializer/serix/type_settings.go b/serializer/serix/type_settings.go index 6a9c1092d..90488b6d2 100644 --- a/serializer/serix/type_settings.go +++ b/serializer/serix/type_settings.go @@ -59,10 +59,6 @@ func (m *MapElementRules) ToTypeSettings() TypeSettings { // MapRules defines rules around a to be deserialized map. type MapRules struct { - // MinEntries defines the min entries for the map. - MinEntries uint - // MaxEntries defines the max entries for the map. 0 means unbounded. - MaxEntries uint // MaxByteSize defines the max serialized byte size for the map. 0 means unbounded. MaxByteSize uint @@ -90,6 +86,31 @@ type TypeSettings struct { mapRules *MapRules } +// WithFieldKey specifies the key for the field. +func (ts TypeSettings) WithFieldKey(fieldKey string) TypeSettings { + ts.fieldKey = &fieldKey + + return ts +} + +// FieldKey returns the key for the field. +func (ts TypeSettings) FieldKey() (string, bool) { + if ts.fieldKey == nil { + return "", false + } + + return *ts.fieldKey, true +} + +// MustFieldKey must return a key for the field. +func (ts TypeSettings) MustFieldKey() string { + if ts.fieldKey == nil { + panic("no field key set") + } + + return *ts.fieldKey +} + // WithLengthPrefixType specifies LengthPrefixType. func (ts TypeSettings) WithLengthPrefixType(lpt LengthPrefixType) TypeSettings { ts.lengthPrefixType = &lpt @@ -137,31 +158,6 @@ func (ts TypeSettings) LexicalOrdering() (val bool, set bool) { return *ts.lexicalOrdering, true } -// WithFieldKey specifies the key for the field. -func (ts TypeSettings) WithFieldKey(fieldKey string) TypeSettings { - ts.fieldKey = &fieldKey - - return ts -} - -// FieldKey returns the key for the field. -func (ts TypeSettings) FieldKey() (string, bool) { - if ts.fieldKey == nil { - return "", false - } - - return *ts.fieldKey, true -} - -// MustFieldKey must return a key for the field. -func (ts TypeSettings) MustFieldKey() string { - if ts.fieldKey == nil { - panic("no field key set") - } - - return *ts.fieldKey -} - // WithArrayRules specifies serializer.ArrayRules. func (ts TypeSettings) WithArrayRules(rules *ArrayRules) TypeSettings { ts.arrayRules = rules @@ -238,26 +234,6 @@ func (ts TypeSettings) MapRules() *MapRules { return ts.mapRules } -// WithMapMinEntries specifies the min entries for the map. -func (ts TypeSettings) WithMapMinEntries(l uint) TypeSettings { - if ts.mapRules == nil { - ts.mapRules = new(MapRules) - } - ts.mapRules.MinEntries = l - - return ts -} - -// WithMapMaxEntries specifies the max entries for the map. -func (ts TypeSettings) WithMapMaxEntries(l uint) TypeSettings { - if ts.mapRules == nil { - ts.mapRules = new(MapRules) - } - ts.mapRules.MaxEntries = l - - return ts -} - // WithMapMaxByteSize specifies max serialized byte size for the map. 0 means unbounded. func (ts TypeSettings) WithMapMaxByteSize(l uint) TypeSettings { if ts.mapRules == nil { From 4628d1113731cadc412fc0b723f20779ff057de6 Mon Sep 17 00:00:00 2001 From: muXxer Date: Wed, 8 Nov 2023 10:11:06 +0100 Subject: [PATCH 11/17] Allow empty serix tags --- serializer/serix/serix.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/serializer/serix/serix.go b/serializer/serix/serix.go index 460a37b70..df71d5d88 100644 --- a/serializer/serix/serix.go +++ b/serializer/serix/serix.go @@ -700,9 +700,9 @@ func (api *API) parseStructType(structType reflect.Type) ([]structField, error) continue } - tSettings, err := parseStructTag(tag, serixPosition) + tSettings, err := parseSerixSettings(tag, serixPosition) if err != nil { - return nil, ierrors.Wrapf(err, "failed to parse struct tag %s for field %s", tag, field.Name) + return nil, ierrors.Wrapf(err, "failed to parse serix struct tag for field %s", field.Name) } serixPosition++ @@ -809,9 +809,13 @@ func parseStructTagValuePrefixType(name string, keyValue []string, currentPart s return lengthPrefixType, nil } -func parseStructTag(tag string, serixPosition int) (tagSettings, error) { +func parseSerixSettings(tag string, serixPosition int) (tagSettings, error) { + settings := tagSettings{} + settings.position = serixPosition + if tag == "" { - return tagSettings{}, ierrors.New("struct tag is empty") + // empty struct tags are allowed + return settings, nil } parts := strings.Split(tag, ",") @@ -821,8 +825,6 @@ func parseStructTag(tag string, serixPosition int) (tagSettings, error) { return tagSettings{}, ierrors.Errorf("incorrect struct tag format: %s, must start with the field key or \",\"", tag) } - settings := tagSettings{} - settings.position = serixPosition settings.ts = settings.ts.WithFieldKey(keyPart) parts = parts[1:] From eb6153bcc6d48f235d80049d5b92f22fbd587827 Mon Sep 17 00:00:00 2001 From: muXxer Date: Wed, 8 Nov 2023 10:11:26 +0100 Subject: [PATCH 12/17] Address review comments --- ds/set_impl.go | 2 +- serializer/error.go | 4 +- serializer/serializer.go | 8 +-- serializer/serix/decode_test.go | 2 +- serializer/serix/encode_test.go | 2 +- serializer/serix/map_encode_test.go | 64 ++++++++++----------- serializer/serix/serix.go | 57 ++++++++++++++++-- serializer/serix/serix_serializable_test.go | 36 ++++++------ serializer/serix/serix_test.go | 14 ++--- 9 files changed, 118 insertions(+), 71 deletions(-) diff --git a/ds/set_impl.go b/ds/set_impl.go index 10af1df31..9cf3b4f70 100644 --- a/ds/set_impl.go +++ b/ds/set_impl.go @@ -145,7 +145,7 @@ func (s *set[ElementType]) apply(mutations SetMutations[ElementType]) (appliedMu // //nolint:tagliatelle // heck knows why this linter fails here type readableSet[T comparable] struct { - *orderedmap.OrderedMap[T, types.Empty] `serix:","` + *orderedmap.OrderedMap[T, types.Empty] `serix:""` } // newReadableSet creates a new readable set with the given elements. diff --git a/serializer/error.go b/serializer/error.go index b4eae64a0..e5c32ca38 100644 --- a/serializer/error.go +++ b/serializer/error.go @@ -33,9 +33,9 @@ var ( // ErrDeserializationLengthInvalid gets returned if a length denotation exceeds a specified limit. ErrDeserializationLengthInvalid = ierrors.New("length denotation invalid") // ErrDeserializationLengthMinNotReached gets returned if a length denotation is less than a specified limit. - ErrDeserializationLengthMinNotReached = ierrors.New("min length not reached") + ErrDeserializationLengthMinNotReached = ierrors.Wrap(ErrDeserializationLengthInvalid, "min length not reached") // ErrDeserializationLengthMaxExceeded gets returned if a length denotation is more than a specified limit. - ErrDeserializationLengthMaxExceeded = ierrors.New("max length exceeded") + ErrDeserializationLengthMaxExceeded = ierrors.Wrap(ErrDeserializationLengthInvalid, "max length exceeded") // ErrDeserializationNotAllConsumed gets returned if not all bytes were consumed during deserialization of a given type. ErrDeserializationNotAllConsumed = ierrors.New("not all data has been consumed but should have been") // ErrUint256NumNegative gets returned if a supposed uint256 has a sign bit. diff --git a/serializer/serializer.go b/serializer/serializer.go index 96623b8d9..e950153a8 100644 --- a/serializer/serializer.go +++ b/serializer/serializer.go @@ -776,9 +776,9 @@ func (d *Deserializer) ReadVariableByteSlice(slice *[]byte, lenType SeriLengthPr switch { case maxLen > 0 && sliceLength > maxLen: - d.err = errProducer(ierrors.Join(ErrDeserializationLengthInvalid, ierrors.Wrapf(ErrDeserializationLengthMaxExceeded, "denoted %d bytes, max allowed %d ", sliceLength, maxLen))) + d.err = errProducer(ierrors.Wrapf(ErrDeserializationLengthMaxExceeded, "denoted %d bytes, max allowed %d ", sliceLength, maxLen)) case minLen > 0 && sliceLength < minLen: - d.err = errProducer(ierrors.Join(ErrDeserializationLengthInvalid, ierrors.Wrapf(ErrDeserializationLengthMinNotReached, "denoted %d bytes, min required %d ", sliceLength, minLen))) + d.err = errProducer(ierrors.Wrapf(ErrDeserializationLengthMinNotReached, "denoted %d bytes, min required %d ", sliceLength, minLen)) } dest := make([]byte, sliceLength) @@ -1158,9 +1158,9 @@ func (d *Deserializer) ReadString(s *string, lenType SeriLengthPrefixType, errPr switch { case maxLen > 0 && strLen > maxLen: - d.err = errProducer(ierrors.Join(ErrDeserializationLengthInvalid, ierrors.Wrapf(ErrDeserializationLengthMaxExceeded, "string defined to be of %d bytes length but max %d is allowed", strLen, maxLen))) + d.err = errProducer(ierrors.Wrapf(ErrDeserializationLengthMaxExceeded, "string defined to be of %d bytes length but max %d is allowed", strLen, maxLen)) case minLen > 0 && strLen < minLen: - d.err = errProducer(ierrors.Join(ErrDeserializationLengthInvalid, ierrors.Wrapf(ErrDeserializationLengthMinNotReached, "string defined to be of %d bytes length but min %d is required", strLen, minLen))) + d.err = errProducer(ierrors.Wrapf(ErrDeserializationLengthMinNotReached, "string defined to be of %d bytes length but min %d is required", strLen, minLen)) } if len(d.src[d.offset:]) < strLen { diff --git a/serializer/serix/decode_test.go b/serializer/serix/decode_test.go index 7be0a00a6..9d004c42e 100644 --- a/serializer/serix/decode_test.go +++ b/serializer/serix/decode_test.go @@ -143,7 +143,7 @@ func TestDecode_ArrayRules(t *testing.T) { ts := serix.TypeSettings{}.WithLengthPrefixType(boolsLenType).WithArrayRules(rules) bytesRead, err := testAPI.Decode(ctx, bytes, testObj, serix.WithValidation(), serix.WithTypeSettings(ts)) require.Zero(t, bytesRead) - assert.Contains(t, err.Error(), serializer.ErrArrayValidationMinElementsNotReached.Error()) + require.ErrorIs(t, err, serializer.ErrArrayValidationMinElementsNotReached) } func testDecode(t testing.TB, ctx context.Context, expected serializer.Serializable, opts ...serix.Option) { diff --git a/serializer/serix/encode_test.go b/serializer/serix/encode_test.go index c06948327..588ee3029 100644 --- a/serializer/serix/encode_test.go +++ b/serializer/serix/encode_test.go @@ -96,7 +96,7 @@ func TestEncode_ArrayRules(t *testing.T) { ts := serix.TypeSettings{}.WithLengthPrefixType(serix.LengthPrefixTypeAsUint32).WithArrayRules(rules) got, err := testAPI.Encode(ctx, testObj, serix.WithValidation(), serix.WithTypeSettings(ts)) require.Nil(t, got) - assert.Contains(t, err.Error(), serializer.ErrArrayValidationMinElementsNotReached.Error()) + require.ErrorIs(t, err, serializer.ErrArrayValidationMinElementsNotReached) } func testEncode(t testing.TB, testObj serializer.Serializable, opts ...serix.Option) { diff --git a/serializer/serix/map_encode_test.go b/serializer/serix/map_encode_test.go index 70c0a1f6f..b02961fb2 100644 --- a/serializer/serix/map_encode_test.go +++ b/serializer/serix/map_encode_test.go @@ -30,8 +30,8 @@ func must(err error) { type Identifier [blake2b.Size256]byte type serializableStruct struct { - bytes Identifier `serix:"bytes"` - index uint64 `serix:"index"` + bytes Identifier `serix:""` + index uint64 `serix:""` } func (s serializableStruct) EncodeJSON() (any, error) { @@ -76,19 +76,19 @@ func TestMapEncodeDecode(t *testing.T) { name: "basic types", paras: func() paras { type example struct { - Uint64 uint64 `serix:"uint64"` - Uint32 uint32 `serix:"uint32"` - Uint16 uint16 `serix:"uint16"` - Uint8 uint8 `serix:"uint8"` - Int64 int64 `serix:"int64"` - Int32 int32 `serix:"int32"` - Int16 int16 `serix:"int16"` - Int8 int8 `serix:"int8"` - ZeroInt32 int32 `serix:"zeroInt32,omitempty"` - Float32 float32 `serix:"float32"` - Float64 float64 `serix:"float64"` - String string `serix:"string"` - Bool bool `serix:"bool"` + Uint64 uint64 `serix:""` + Uint32 uint32 `serix:""` + Uint16 uint16 `serix:""` + Uint8 uint8 `serix:""` + Int64 int64 `serix:""` + Int32 int32 `serix:""` + Int16 int16 `serix:""` + Int8 int8 `serix:""` + ZeroInt32 int32 `serix:",omitempty"` + Float32 float32 `serix:""` + Float64 float64 `serix:""` + String string `serix:""` + Bool bool `serix:""` } api := serix.NewAPI() @@ -133,7 +133,7 @@ func TestMapEncodeDecode(t *testing.T) { name: "big int", paras: func() paras { type example struct { - BigInt *big.Int `serix:"bigInt"` + BigInt *big.Int `serix:""` } api := serix.NewAPI() @@ -155,7 +155,7 @@ func TestMapEncodeDecode(t *testing.T) { name: "map", paras: func() paras { type example struct { - Map map[string]string `serix:"map"` + Map map[string]string `serix:""` } api := serix.NewAPI() @@ -182,10 +182,10 @@ func TestMapEncodeDecode(t *testing.T) { paras: func() paras { type example struct { - ByteSlice []byte `serix:"byteSlice"` - Array [5]byte `serix:"array"` - SliceOfByteSlices [][]byte `serix:"sliceOfByteSlices"` - SliceOfByteArrays [][3]byte `serix:"sliceOfByteArrays"` + ByteSlice []byte `serix:""` + Array [5]byte `serix:""` + SliceOfByteSlices [][]byte `serix:""` + SliceOfByteArrays [][3]byte `serix:""` } api := serix.NewAPI() @@ -226,11 +226,11 @@ func TestMapEncodeDecode(t *testing.T) { paras: func() paras { type ( inner struct { - String string `serix:"string"` + String string `serix:""` } example struct { - inner `serix:"inner"` + inner `serix:""` } ) @@ -258,8 +258,8 @@ func TestMapEncodeDecode(t *testing.T) { OtherObj [2]byte example struct { - Interface InterfaceType `serix:"interface"` - Other *OtherObj `serix:"other"` + Interface InterfaceType `serix:""` + Other *OtherObj `serix:""` } ) @@ -299,14 +299,14 @@ func TestMapEncodeDecode(t *testing.T) { type ( Interface interface{} Impl1 struct { - String string `serix:"string"` + String string `serix:""` } Impl2 struct { - Uint16 uint16 `serix:"uint16"` + Uint16 uint16 `serix:""` } example struct { - Slice []Interface `serix:"slice"` + Slice []Interface `serix:""` } ) @@ -344,8 +344,8 @@ func TestMapEncodeDecode(t *testing.T) { name: "no map key", paras: func() paras { type example struct { - CaptainHook string `serix:"captainHook"` - LiquidSoul int64 `serix:"liquidSoul"` + CaptainHook string `serix:""` + LiquidSoul int64 `serix:""` } api := serix.NewAPI() @@ -369,7 +369,7 @@ func TestMapEncodeDecode(t *testing.T) { name: "time", paras: func() paras { type example struct { - CreationDate time.Time `serix:"creationDate"` + CreationDate time.Time `serix:""` } api := serix.NewAPI() @@ -396,7 +396,7 @@ func TestMapEncodeDecode(t *testing.T) { name: "serializable", paras: func() paras { type example struct { - Entries map[serializableStruct]struct{} `serix:"entries"` + Entries map[serializableStruct]struct{} `serix:""` } api := serix.NewAPI() diff --git a/serializer/serix/serix.go b/serializer/serix/serix.go index df71d5d88..3d70a7c09 100644 --- a/serializer/serix/serix.go +++ b/serializer/serix/serix.go @@ -3,12 +3,59 @@ Structs serialization/deserialization -In order for a field to be detected by serix it must have `serix` struct tag set with the name: `serix:"example" or empty name `serix:","`. +In order for a field to be detected by serix it must have `serix:""` struct tag. +The first part in the tag is the key used for json serialization. +If the name is empty, serix uses the field name in camel case (Exceptions: "ID" => "Id", "URL" => "Url"). + +Examples: + - `serix:"" + - `serix:"example" + - `serix:","` + serix traverses all fields and handles them in the order specified in the struct. -Apart from the required position you can provide the following settings to serix via struct tags: -"optional" - means that field might be nil. Only valid for pointers or interfaces: `serix:"example,optional"` -"lenPrefix=uint32" - provide serializer.SeriLengthPrefixType for that field: `serix:"example,lenPrefix=unint32"` -"nest" - handle embedded/anonymous field as a nested field: `serix:"example,nest"` +You can provide the following settings to serix via struct tags: + + - "optional": means the field might be nil. Only valid for pointers or interfaces. + It will be prepended with the serialized size of the field. + `serix:"example,optional"` + + - "nest": handle embedded/anonymous field as a nested field + `serix:"example,nest"` + + - "omitempty": omit the field in json serialization if it's empty + `serix:"example,omitempty"` + + - "lenPrefix": provide serializer.SeriLengthPrefixType for that field (string, slice, map) + `serix:"example,lenPrefix=uint32"` + + - "minLen": minimum length for that field (string, slice, map) + `serix:"example,minLen=2"` + + - "maxLen": maximum length for that field (string, slice, map) + `serix:"example,maxLen=5"` + + - "mapMaxByteSize": maximum serialized byte size for that map + `serix:"example,mapMaxByteSize=100"` + + - "mapKeyLenPrefix": provide serializer.SeriLengthPrefixType for the keys of that map + `serix:"example,mapKeyLenPrefix=uint32"` + + - "mapKeyMinLen": minimum length for the keys of that map + `serix:"example,mapKeyMinLen=2"` + + - "mapKeyMaxLen": maximum length for the keys of that map + `serix:"example,mapKeyMaxLen=5"` + + - "mapValueLenPrefix": provide serializer.SeriLengthPrefixType for the values of that map + `serix:"example,mapValueLenPrefix=uint32"` + + - "mapValueMinLen": minimum length for the values of that map + `serix:"example,mapValueMinLen=2"` + + - "mapValueMaxLen": maximum length for the values of that map + `serix:"example,mapValueMaxLen=5"` + + See serix_text.go for more detail. */ package serix diff --git a/serializer/serix/serix_serializable_test.go b/serializer/serix/serix_serializable_test.go index 278149eee..d5d733f6f 100644 --- a/serializer/serix/serix_serializable_test.go +++ b/serializer/serix/serix_serializable_test.go @@ -80,15 +80,15 @@ func (bs Bools) FromSerializables(seris serializer.Serializables) { } type SimpleStruct struct { - Bool bool `serix:"bool"` - Uint uint64 `serix:"uint"` - String string `serix:"string,lenPrefix=uint16"` - Bytes []byte `serix:"bytes,lenPrefix=uint32"` - BytesArray [16]byte `serix:"bytesarray"` - BigInt *big.Int `serix:"bigint"` - Time time.Time `serix:"time"` - Int uint64 `serix:"int"` - Float float64 `serix:"float"` + Bool bool `serix:""` + Uint uint64 `serix:""` + String string `serix:",lenPrefix=uint16"` + Bytes []byte `serix:",lenPrefix=uint32"` + BytesArray [16]byte `serix:""` + BigInt *big.Int `serix:""` + Time time.Time `serix:""` + Int uint64 `serix:""` + Float float64 `serix:""` } func NewSimpleStruct() SimpleStruct { @@ -146,7 +146,7 @@ type Interface interface { } type InterfaceImpl struct { - interfaceImpl `serix:"interfaceImpl"` + interfaceImpl `serix:""` } func (ii *InterfaceImpl) SetDeserializationContext(ctx context.Context) { @@ -154,8 +154,8 @@ func (ii *InterfaceImpl) SetDeserializationContext(ctx context.Context) { } type interfaceImpl struct { - A uint8 `serix:"a"` - B uint8 `serix:"b"` + A uint8 `serix:""` + B uint8 `serix:""` } func (ii *InterfaceImpl) Encode() ([]byte, error) { @@ -195,7 +195,7 @@ func (ii *InterfaceImpl) Serialize(deSeriMode serializer.DeSerializationMode, de } type StructWithInterface struct { - Interface Interface `serix:"interface"` + Interface Interface `serix:""` } func (si StructWithInterface) MarshalJSON() ([]byte, error) { @@ -221,7 +221,7 @@ func (si StructWithInterface) Serialize(deSeriMode serializer.DeSerializationMod } type StructWithOptionalField struct { - Optional *ExportedStruct `serix:"optional,optional"` + Optional *ExportedStruct `serix:",optional"` } func (so StructWithOptionalField) MarshalJSON() ([]byte, error) { @@ -247,8 +247,8 @@ func (so StructWithOptionalField) Serialize(deSeriMode serializer.DeSerializatio } type StructWithEmbeddedStructs struct { - unexportedStruct `serix:"unexportedstruct"` - ExportedStruct `serix:"exportedstruct,nest"` + unexportedStruct `serix:""` + ExportedStruct `serix:",nest"` } func (se StructWithEmbeddedStructs) MarshalJSON() ([]byte, error) { @@ -276,11 +276,11 @@ func (se StructWithEmbeddedStructs) Serialize(deSeriMode serializer.DeSerializat } type unexportedStruct struct { - Foo uint64 `serix:"foo"` + Foo uint64 `serix:""` } type ExportedStruct struct { - Bar uint64 `serix:"bar"` + Bar uint64 `serix:""` } func (e ExportedStruct) SetDeserializationContext(ctx context.Context) { diff --git a/serializer/serix/serix_test.go b/serializer/serix/serix_test.go index 858499e26..d0bc89be3 100644 --- a/serializer/serix/serix_test.go +++ b/serializer/serix/serix_test.go @@ -39,7 +39,7 @@ func TestMinMax(t *testing.T) { name: "ok - string in bounds", paras: func() paras { type example struct { - Str string `serix:"str,minLen=5,maxLen=10,lenPrefix=uint8"` + Str string `serix:",minLen=5,maxLen=10,lenPrefix=uint8"` } api := serix.NewAPI() @@ -57,7 +57,7 @@ func TestMinMax(t *testing.T) { name: "err - string out of bounds", paras: func() paras { type example struct { - Str string `serix:"str,minLen=5,maxLen=10,lenPrefix=uint8"` + Str string `serix:",minLen=5,maxLen=10,lenPrefix=uint8"` } api := serix.NewAPI() @@ -75,7 +75,7 @@ func TestMinMax(t *testing.T) { name: "ok - slice in bounds", paras: func() paras { type example struct { - Slice []byte `serix:"slice,minLen=0,maxLen=10,lenPrefix=uint8"` + Slice []byte `serix:",minLen=0,maxLen=10,lenPrefix=uint8"` } api := serix.NewAPI() @@ -93,7 +93,7 @@ func TestMinMax(t *testing.T) { name: "err - slice out of bounds", paras: func() paras { type example struct { - Slice []byte `serix:"slice,minLen=0,maxLen=3,lenPrefix=uint8"` + Slice []byte `serix:",minLen=0,maxLen=3,lenPrefix=uint8"` } api := serix.NewAPI() @@ -181,7 +181,7 @@ func TestSerixMapSerialize(t *testing.T) { type MyMapType map[string]string type MapStruct struct { - MyMap MyMapType `serix:"myMap,lenPrefix=uint8,minLen=2,maxLen=4,mapMaxByteSize=50,mapKeyLenPrefix=uint16,mapKeyMinLen=2,mapKeyMaxLen=5,mapValueLenPrefix=uint32,mapValueMinLen=1,mapValueMaxLen=6"` + MyMap MyMapType `serix:",lenPrefix=uint8,minLen=2,maxLen=4,mapMaxByteSize=50,mapKeyLenPrefix=uint16,mapKeyMinLen=2,mapKeyMaxLen=5,mapValueLenPrefix=uint32,mapValueMinLen=1,mapValueMaxLen=6"` } testAPI.RegisterTypeSettings(MapStruct{}, serix.TypeSettings{}) @@ -341,12 +341,12 @@ func TestSerixMapDeserialize(t *testing.T) { // used to create test data type TestVectorMapStruct struct { - MyMap MyMapType `serix:"myMap,lenPrefix=uint8,minLen=1,maxLen=5,mapMaxByteSize=100,mapKeyLenPrefix=uint16,mapKeyMinLen=1,mapKeyMaxLen=7,mapValueLenPrefix=uint32,mapValueMinLen=0,mapValueMaxLen=10"` + MyMap MyMapType `serix:",lenPrefix=uint8,minLen=1,maxLen=5,mapMaxByteSize=100,mapKeyLenPrefix=uint16,mapKeyMinLen=1,mapKeyMaxLen=7,mapValueLenPrefix=uint32,mapValueMinLen=0,mapValueMaxLen=10"` } testAPI.RegisterTypeSettings(TestVectorMapStruct{}, serix.TypeSettings{}) type MapStruct struct { - MyMap MyMapType `serix:"myMap,lenPrefix=uint8,minLen=2,maxLen=4,mapMaxByteSize=50,mapKeyLenPrefix=uint16,mapKeyMinLen=2,mapKeyMaxLen=5,mapValueLenPrefix=uint32,mapValueMinLen=1,mapValueMaxLen=6"` + MyMap MyMapType `serix:",lenPrefix=uint8,minLen=2,maxLen=4,mapMaxByteSize=50,mapKeyLenPrefix=uint16,mapKeyMinLen=2,mapKeyMaxLen=5,mapValueLenPrefix=uint32,mapValueMinLen=1,mapValueMaxLen=6"` } testAPI.RegisterTypeSettings(MapStruct{}, serix.TypeSettings{}) From d39d68b77f0c3441067b5830c3749fe700ffbb68 Mon Sep 17 00:00:00 2001 From: muXxer Date: Wed, 8 Nov 2023 10:11:43 +0100 Subject: [PATCH 13/17] Allow embedded interfaces --- serializer/serix/serix.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serializer/serix/serix.go b/serializer/serix/serix.go index 3d70a7c09..a28153a23 100644 --- a/serializer/serix/serix.go +++ b/serializer/serix/serix.go @@ -738,7 +738,7 @@ func (api *API) parseStructType(structType reflect.Type) ([]structField, error) isEmbeddedStruct := isEmbedded && isStruct isEmbeddedInterface := isEmbedded && isInterface - if isUnexported && !isEmbeddedStruct { + if isUnexported && !isEmbeddedStruct && !isEmbeddedInterface { continue } From af9e771c1c6081969abfb714e249994586ebe389 Mon Sep 17 00:00:00 2001 From: muXxer Date: Wed, 8 Nov 2023 11:41:58 +0100 Subject: [PATCH 14/17] Rename nest to inlined --- serializer/serix/decode.go | 2 +- serializer/serix/encode.go | 2 +- serializer/serix/map_decode.go | 6 +++--- serializer/serix/map_encode.go | 6 +++--- serializer/serix/serix.go | 18 +++++++++--------- serializer/serix/serix_serializable_test.go | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/serializer/serix/decode.go b/serializer/serix/decode.go index 1a6f1b545..203ccb70a 100644 --- a/serializer/serix/decode.go +++ b/serializer/serix/decode.go @@ -281,7 +281,7 @@ func (api *API) decodeStructFields( for _, sField := range structFields { fieldValue := value.Field(sField.index) - if sField.isEmbedded && !sField.settings.nest { + if sField.isEmbedded && !sField.settings.inlined { fieldType := sField.fType if fieldType.Kind() == reflect.Ptr { if fieldValue.IsNil() { diff --git a/serializer/serix/encode.go b/serializer/serix/encode.go index 759c047b3..0ef095dd6 100644 --- a/serializer/serix/encode.go +++ b/serializer/serix/encode.go @@ -203,7 +203,7 @@ func (api *API) encodeStructFields( for _, sField := range structFields { fieldValue := value.Field(sField.index) - if sField.isEmbedded && !sField.settings.nest { + if sField.isEmbedded && !sField.settings.inlined { fieldType := sField.fType if fieldValue.Kind() == reflect.Ptr { if fieldValue.IsNil() { diff --git a/serializer/serix/map_decode.go b/serializer/serix/map_decode.go index 282c404ac..794f11d16 100644 --- a/serializer/serix/map_decode.go +++ b/serializer/serix/map_decode.go @@ -352,7 +352,7 @@ func (api *API) mapDecodeStructFields( for _, sField := range structFields { fieldValue := structVal.Field(sField.index) - if sField.isEmbedded && !sField.settings.nest { + if sField.isEmbedded && !sField.settings.inlined { fieldType := sField.fType if fieldType.Kind() == reflect.Ptr { if fieldValue.IsNil() { @@ -374,9 +374,9 @@ func (api *API) mapDecodeStructFields( continue } - if sField.settings.nest { + if sField.settings.inlined { if err := api.mapDecode(ctx, m, fieldValue, sField.settings.ts, opts); err != nil { - return ierrors.Wrapf(err, "failed to deserialize nested struct field %s", sField.name) + return ierrors.Wrapf(err, "failed to deserialize inlined struct field %s", sField.name) } continue diff --git a/serializer/serix/map_encode.go b/serializer/serix/map_encode.go index 79dfe75b1..d27ec607c 100644 --- a/serializer/serix/map_encode.go +++ b/serializer/serix/map_encode.go @@ -182,7 +182,7 @@ func (api *API) mapEncodeStructFields( for _, sField := range structFields { fieldValue := value.Field(sField.index) - if sField.isEmbedded && !sField.settings.nest { + if sField.isEmbedded && !sField.settings.inlined { fieldType := sField.fType if fieldValue.Kind() == reflect.Ptr { if fieldValue.IsNil() { @@ -217,10 +217,10 @@ func (api *API) mapEncodeStructFields( switch { case sField.settings.ts.fieldKey != nil: obj.Set(*sField.settings.ts.fieldKey, eleOut) - case sField.settings.nest: + case sField.settings.inlined: castedEleOut, ok := eleOut.(*orderedmap.OrderedMap) if !ok { - return ierrors.Errorf("failed to cast nested struct field %s to map", sField.name) + return ierrors.Errorf("failed to cast inlined struct field %s to map", sField.name) } for _, k := range castedEleOut.Keys() { diff --git a/serializer/serix/serix.go b/serializer/serix/serix.go index a28153a23..0e6c37e1b 100644 --- a/serializer/serix/serix.go +++ b/serializer/serix/serix.go @@ -19,8 +19,8 @@ You can provide the following settings to serix via struct tags: It will be prepended with the serialized size of the field. `serix:"example,optional"` - - "nest": handle embedded/anonymous field as a nested field - `serix:"example,nest"` + - "inlined": handle embedded/anonymous field as a nested field + `serix:"example,inlined"` - "omitempty": omit the field in json serialization if it's empty `serix:"example,omitempty"` @@ -712,7 +712,7 @@ type structField struct { type tagSettings struct { position int isOptional bool - nest bool + inlined bool omitEmpty bool ts TypeSettings } @@ -774,15 +774,15 @@ func (api *API) parseStructType(structType reflect.Type) ([]structField, error) } } - if tSettings.nest && isUnexported { + if tSettings.inlined && isUnexported { return nil, ierrors.Errorf( - "struct field %s is invalid: 'nest' setting can't be used with unexported types", + "struct field %s is invalid: 'inlined' setting can't be used with unexported types", field.Name) } - if !tSettings.nest && isEmbeddedInterface { + if !tSettings.inlined && isEmbeddedInterface { return nil, ierrors.Errorf( - "struct field %s is invalid: 'nest' setting needs to be used for embedded interfaces", + "struct field %s is invalid: 'inlined' setting needs to be used for embedded interfaces", field.Name) } @@ -887,8 +887,8 @@ func parseSerixSettings(tag string, serixPosition int) (tagSettings, error) { case "optional": settings.isOptional = true - case "nest": - settings.nest = true + case "inlined": + settings.inlined = true case "omitempty": settings.omitEmpty = true diff --git a/serializer/serix/serix_serializable_test.go b/serializer/serix/serix_serializable_test.go index d5d733f6f..ccfa8174c 100644 --- a/serializer/serix/serix_serializable_test.go +++ b/serializer/serix/serix_serializable_test.go @@ -248,7 +248,7 @@ func (so StructWithOptionalField) Serialize(deSeriMode serializer.DeSerializatio type StructWithEmbeddedStructs struct { unexportedStruct `serix:""` - ExportedStruct `serix:",nest"` + ExportedStruct `serix:",inlined"` } func (se StructWithEmbeddedStructs) MarshalJSON() ([]byte, error) { From b766b275dbee93f599589bf98a17b907e002b20b Mon Sep 17 00:00:00 2001 From: muXxer Date: Wed, 8 Nov 2023 12:05:58 +0100 Subject: [PATCH 15/17] Add MaxByteSize check for all serix types --- serializer/serix/decode.go | 15 ++++--- serializer/serix/encode.go | 15 ++++--- serializer/serix/map_decode.go | 36 +++++++++++------ serializer/serix/map_encode.go | 24 +++++++---- serializer/serix/serix.go | 44 +++++++++----------- serializer/serix/serix_test.go | 10 ++--- serializer/serix/type_settings.go | 67 ++++++++++++++++--------------- 7 files changed, 117 insertions(+), 94 deletions(-) diff --git a/serializer/serix/decode.go b/serializer/serix/decode.go index 203ccb70a..61fd153ce 100644 --- a/serializer/serix/decode.go +++ b/serializer/serix/decode.go @@ -86,10 +86,15 @@ func (api *API) decode(ctx context.Context, b []byte, value reflect.Value, ts Ty return 0, ierrors.WithStack(err) } } + if opts.validation { if err := api.callSyntacticValidator(ctx, value, valueType); err != nil { return 0, ierrors.Wrap(err, "post-deserialization validation failed") } + + if err := api.checkMaxByteSize(bytesRead, ts); err != nil { + return bytesRead, err + } } return bytesRead, nil @@ -472,12 +477,10 @@ func (api *API) decodeMap(ctx context.Context, b []byte, value reflect.Value, return consumedBytes, err } - if err := api.checkMinMaxBounds(value, ts); err != nil { - return consumedBytes, err - } - - if err := api.checkMapMaxByteSize(consumedBytes, ts); err != nil { - return consumedBytes, err + if opts.validation { + if err := api.checkMinMaxBounds(value, ts); err != nil { + return consumedBytes, err + } } return consumedBytes, nil diff --git a/serializer/serix/encode.go b/serializer/serix/encode.go index 0ef095dd6..36ce22a07 100644 --- a/serializer/serix/encode.go +++ b/serializer/serix/encode.go @@ -52,10 +52,15 @@ func (api *API) encode(ctx context.Context, value reflect.Value, ts TypeSettings return nil, ierrors.WithStack(err) } } + if opts.validation { if err = api.callBytesValidator(ctx, valueType, b); err != nil { return nil, ierrors.Wrap(err, "post-serialization validation failed") } + + if err := api.checkMaxByteSize(len(b), ts); err != nil { + return nil, err + } } return b, nil @@ -343,8 +348,10 @@ func (api *API) encodeMapKVPair(ctx context.Context, key, val reflect.Value, ts func (api *API) encodeMap(ctx context.Context, value reflect.Value, valueType reflect.Type, ts TypeSettings, opts *options) ([]byte, error) { - if err := api.checkMinMaxBounds(value, ts); err != nil { - return nil, err + if opts.validation { + if err := api.checkMinMaxBounds(value, ts); err != nil { + return nil, err + } } data := make([][]byte, value.Len()) @@ -365,10 +372,6 @@ func (api *API) encodeMap(ctx context.Context, value reflect.Value, valueType re return nil, err } - if err := api.checkMapMaxByteSize(len(bytes), ts); err != nil { - return nil, err - } - return bytes, nil } diff --git a/serializer/serix/map_decode.go b/serializer/serix/map_decode.go index 794f11d16..bc1e12947 100644 --- a/serializer/serix/map_decode.go +++ b/serializer/serix/map_decode.go @@ -40,8 +40,10 @@ func (api *API) mapDecode(ctx context.Context, mapVal any, value reflect.Value, } } - if err := api.checkMapSerializedSize(ctx, value, ts, opts); err != nil { - return err + if opts.validation { + if err := api.checkSerializedSize(ctx, value, ts, opts); err != nil { + return err + } } return nil @@ -114,8 +116,10 @@ func (api *API) mapDecodeBasedOnType(ctx context.Context, mapVal any, value refl return ierrors.Wrap(err, "failed to read byte slice from map") } - if err := api.checkMinMaxBoundsLength(len(byteSlice), ts); err != nil { - return ierrors.Wrapf(err, "can't deserialize '%s' type", value.Kind()) + if opts.validation { + if err := api.checkMinMaxBoundsLength(len(byteSlice), ts); err != nil { + return ierrors.Wrapf(err, "can't deserialize '%s' type", value.Kind()) + } } copy(sliceValue.Bytes(), byteSlice) @@ -165,8 +169,10 @@ func (api *API) mapDecodeBasedOnType(ctx context.Context, mapVal any, value refl addrValue := value.Addr().Convert(reflect.TypeOf((*string)(nil))) addrValue.Elem().Set(reflect.ValueOf(mapVal)) - if err := api.checkMinMaxBoundsLength(len(str), ts); err != nil { - return ierrors.Wrapf(err, "can't deserialize '%s' type", value.Kind()) + if opts.validation { + if err := api.checkMinMaxBoundsLength(len(str), ts); err != nil { + return ierrors.Wrapf(err, "can't deserialize '%s' type", value.Kind()) + } } return nil @@ -414,8 +420,10 @@ func (api *API) mapDecodeSlice(ctx context.Context, mapVal any, value reflect.Va return ierrors.Wrap(err, "failed to read byte slice from map") } - if err := api.checkMinMaxBoundsLength(len(byteSlice), ts); err != nil { - return ierrors.Wrapf(err, "can't deserialize '%s' type", value.Kind()) + if opts.validation { + if err := api.checkMinMaxBoundsLength(len(byteSlice), ts); err != nil { + return ierrors.Wrapf(err, "can't deserialize '%s' type", value.Kind()) + } } addrValue := value.Addr().Convert(reflect.TypeOf((*[]byte)(nil))) @@ -433,8 +441,10 @@ func (api *API) mapDecodeSlice(ctx context.Context, mapVal any, value reflect.Va value.Set(reflect.Append(value, elemValue)) } - if err := api.checkMinMaxBounds(value, ts); err != nil { - return ierrors.Wrapf(err, "can't serialize '%s' type", value.Kind()) + if opts.validation { + if err := api.checkMinMaxBounds(value, ts); err != nil { + return ierrors.Wrapf(err, "can't serialize '%s' type", value.Kind()) + } } // check if the slice is a nil pointer to the slice type (in case the sliceLength is zero and the slice was not initialized before) @@ -484,8 +494,10 @@ func (api *API) mapDecodeMap(ctx context.Context, mapVal any, value reflect.Valu value.SetMapIndex(keyValue, elemValue) } - if err := api.checkMinMaxBounds(value, ts); err != nil { - return err + if opts.validation { + if err := api.checkMinMaxBounds(value, ts); err != nil { + return err + } } return nil diff --git a/serializer/serix/map_encode.go b/serializer/serix/map_encode.go index d27ec607c..b86dfc8cb 100644 --- a/serializer/serix/map_encode.go +++ b/serializer/serix/map_encode.go @@ -36,8 +36,10 @@ func (api *API) mapEncode(ctx context.Context, value reflect.Value, ts TypeSetti } } - if err := api.checkMapSerializedSize(ctx, value, ts, opts); err != nil { - return nil, err + if opts.validation { + if err := api.checkSerializedSize(ctx, value, ts, opts); err != nil { + return nil, err + } } if serializable, ok := valueI.(SerializableJSON); ok { @@ -101,8 +103,10 @@ func (api *API) mapEncodeBasedOnType( return nil, ErrNonUTF8String } - if err := api.checkMinMaxBoundsLength(len(str), ts); err != nil { - return nil, ierrors.Wrapf(err, "can't serialize '%s' type", value.Kind()) + if opts.validation { + if err := api.checkMinMaxBoundsLength(len(str), ts); err != nil { + return nil, ierrors.Wrapf(err, "can't serialize '%s' type", value.Kind()) + } } return value.String(), nil @@ -250,8 +254,10 @@ func (api *API) mapEncodeSlice(ctx context.Context, value reflect.Value, valueTy } if valueType.AssignableTo(bytesType) { - if err := api.checkMinMaxBoundsLength(len(value.Bytes()), ts); err != nil { - return nil, ierrors.Wrapf(err, "can't serialize '%s' type", value.Kind()) + if opts.validation { + if err := api.checkMinMaxBoundsLength(len(value.Bytes()), ts); err != nil { + return nil, ierrors.Wrapf(err, "can't serialize '%s' type", value.Kind()) + } } return EncodeHex(value.Bytes()), nil @@ -299,8 +305,10 @@ func (api *API) mapEncodeMapKVPair(ctx context.Context, key, val reflect.Value, } func (api *API) mapEncodeMap(ctx context.Context, value reflect.Value, ts TypeSettings, opts *options) (*orderedmap.OrderedMap, error) { - if err := api.checkMinMaxBounds(value, ts); err != nil { - return nil, err + if opts.validation { + if err := api.checkMinMaxBounds(value, ts); err != nil { + return nil, err + } } m := orderedmap.New() diff --git a/serializer/serix/serix.go b/serializer/serix/serix.go index 0e6c37e1b..0e1ceaf62 100644 --- a/serializer/serix/serix.go +++ b/serializer/serix/serix.go @@ -25,6 +25,9 @@ You can provide the following settings to serix via struct tags: - "omitempty": omit the field in json serialization if it's empty `serix:"example,omitempty"` + - "maxByteSize": maximum serialized byte size for that field + `serix:"example,maxByteSize=100"` + - "lenPrefix": provide serializer.SeriLengthPrefixType for that field (string, slice, map) `serix:"example,lenPrefix=uint32"` @@ -34,9 +37,6 @@ You can provide the following settings to serix via struct tags: - "maxLen": maximum length for that field (string, slice, map) `serix:"example,maxLen=5"` - - "mapMaxByteSize": maximum serialized byte size for that map - `serix:"example,mapMaxByteSize=100"` - - "mapKeyLenPrefix": provide serializer.SeriLengthPrefixType for the keys of that map `serix:"example,mapKeyLenPrefix=uint32"` @@ -78,8 +78,8 @@ import ( ) var ( - // ErrMapValidationMaxBytesExceeded gets returned if the serialized byte size of the map is too big. - ErrMapValidationMaxBytesExceeded = ierrors.New("max bytes size of the map exceeded") + // ErrValidationMaxBytesExceeded gets returned if the serialized byte size of the object too big. + ErrValidationMaxBytesExceeded = ierrors.New("max bytes size exceeded") // ErrMapValidationViolatesUniqueness gets returned if the map elements are not unique. ErrMapValidationViolatesUniqueness = ierrors.New("map elements must be unique") ) @@ -247,30 +247,26 @@ func (api *API) checkMinMaxBounds(v reflect.Value, ts TypeSettings) error { return nil } -// checkMapMaxByteSize checks whether the given map is within its defined max byte size in case it has defined map rules. -func (api *API) checkMapMaxByteSize(byteSize int, ts TypeSettings) error { - if ts.mapRules != nil && ts.mapRules.MaxByteSize > 0 && byteSize > int(ts.mapRules.MaxByteSize) { - return ierrors.Wrapf(ErrMapValidationMaxBytesExceeded, "map (len %d) exceeds max bytes of %d ", byteSize, ts.mapRules.MaxByteSize) +// checkMaxByteSize checks whether the given type is within its defined size in case it has a max byte size. +func (api *API) checkMaxByteSize(byteSize int, ts TypeSettings) error { + if ts.maxByteSize > 0 && byteSize > int(ts.maxByteSize) { + return ierrors.Wrapf(ErrValidationMaxBytesExceeded, "serialized size (%d) exceeds max byte size of %d ", byteSize, ts.maxByteSize) } return nil } -func (api *API) checkMapSerializedSize(ctx context.Context, value reflect.Value, ts TypeSettings, opts *options) error { - if ts.mapRules == nil || ts.mapRules.MaxByteSize == 0 { +func (api *API) checkSerializedSize(ctx context.Context, value reflect.Value, ts TypeSettings, opts *options) error { + if ts.maxByteSize == 0 { return nil } - if value.Kind() != reflect.Map { - return ierrors.Errorf("can't get map serialized size: value is not a map, got %s", value.Kind()) - } - bytes, err := api.encode(ctx, value, ts, opts) if err != nil { - return ierrors.Wrapf(err, "can't get map serialized size: failed to encode map") + return ierrors.Wrapf(err, "can't get serialized size: failed to encode '%s' type", value.Kind()) } - return api.checkMapMaxByteSize(len(bytes), ts) + return api.checkMaxByteSize(len(bytes), ts) } // Encode serializes the provided object obj into bytes. @@ -893,6 +889,13 @@ func parseSerixSettings(tag string, serixPosition int) (tagSettings, error) { case "omitempty": settings.omitEmpty = true + case "maxByteSize": + value, err := parseStructTagValueUint("maxByteSize", keyValue, currentPart) + if err != nil { + return tagSettings{}, err + } + settings.ts = settings.ts.WithMaxByteSize(value) + case "lenPrefix": value, err := parseStructTagValuePrefixType("lenPrefix", keyValue, currentPart) if err != nil { @@ -914,13 +917,6 @@ func parseSerixSettings(tag string, serixPosition int) (tagSettings, error) { } settings.ts = settings.ts.WithMaxLen(value) - case "mapMaxByteSize": - value, err := parseStructTagValueUint("mapMaxByteSize", keyValue, currentPart) - if err != nil { - return tagSettings{}, err - } - settings.ts = settings.ts.WithMapMaxByteSize(value) - case "mapKeyLenPrefix": value, err := parseStructTagValuePrefixType("mapKeyLenPrefix", keyValue, currentPart) if err != nil { diff --git a/serializer/serix/serix_test.go b/serializer/serix/serix_test.go index d0bc89be3..dd5ce3565 100644 --- a/serializer/serix/serix_test.go +++ b/serializer/serix/serix_test.go @@ -181,7 +181,7 @@ func TestSerixMapSerialize(t *testing.T) { type MyMapType map[string]string type MapStruct struct { - MyMap MyMapType `serix:",lenPrefix=uint8,minLen=2,maxLen=4,mapMaxByteSize=50,mapKeyLenPrefix=uint16,mapKeyMinLen=2,mapKeyMaxLen=5,mapValueLenPrefix=uint32,mapValueMinLen=1,mapValueMaxLen=6"` + MyMap MyMapType `serix:",lenPrefix=uint8,minLen=2,maxLen=4,maxByteSize=50,mapKeyLenPrefix=uint16,mapKeyMinLen=2,mapKeyMaxLen=5,mapValueLenPrefix=uint32,mapValueMinLen=1,mapValueMaxLen=6"` } testAPI.RegisterTypeSettings(MapStruct{}, serix.TypeSettings{}) @@ -236,7 +236,7 @@ func TestSerixMapSerialize(t *testing.T) { }, target: &MapStruct{}, size: 0, - seriErr: serix.ErrMapValidationMaxBytesExceeded, + seriErr: serix.ErrValidationMaxBytesExceeded, }, { name: "fail - key too short", @@ -341,12 +341,12 @@ func TestSerixMapDeserialize(t *testing.T) { // used to create test data type TestVectorMapStruct struct { - MyMap MyMapType `serix:",lenPrefix=uint8,minLen=1,maxLen=5,mapMaxByteSize=100,mapKeyLenPrefix=uint16,mapKeyMinLen=1,mapKeyMaxLen=7,mapValueLenPrefix=uint32,mapValueMinLen=0,mapValueMaxLen=10"` + MyMap MyMapType `serix:",lenPrefix=uint8,minLen=1,maxLen=5,maxByteSize=100,mapKeyLenPrefix=uint16,mapKeyMinLen=1,mapKeyMaxLen=7,mapValueLenPrefix=uint32,mapValueMinLen=0,mapValueMaxLen=10"` } testAPI.RegisterTypeSettings(TestVectorMapStruct{}, serix.TypeSettings{}) type MapStruct struct { - MyMap MyMapType `serix:",lenPrefix=uint8,minLen=2,maxLen=4,mapMaxByteSize=50,mapKeyLenPrefix=uint16,mapKeyMinLen=2,mapKeyMaxLen=5,mapValueLenPrefix=uint32,mapValueMinLen=1,mapValueMaxLen=6"` + MyMap MyMapType `serix:",lenPrefix=uint8,minLen=2,maxLen=4,maxByteSize=50,mapKeyLenPrefix=uint16,mapKeyMinLen=2,mapKeyMaxLen=5,mapValueLenPrefix=uint32,mapValueMinLen=1,mapValueMaxLen=6"` } testAPI.RegisterTypeSettings(MapStruct{}, serix.TypeSettings{}) @@ -401,7 +401,7 @@ func TestSerixMapDeserialize(t *testing.T) { }, target: &MapStruct{}, size: 0, - deSeriErr: serix.ErrMapValidationMaxBytesExceeded, + deSeriErr: serix.ErrValidationMaxBytesExceeded, }, { name: "fail - key too short", diff --git a/serializer/serix/type_settings.go b/serializer/serix/type_settings.go index 90488b6d2..d529d52ba 100644 --- a/serializer/serix/type_settings.go +++ b/serializer/serix/type_settings.go @@ -59,9 +59,6 @@ func (m *MapElementRules) ToTypeSettings() TypeSettings { // MapRules defines rules around a to be deserialized map. type MapRules struct { - // MaxByteSize defines the max serialized byte size for the map. 0 means unbounded. - MaxByteSize uint - // KeyRules define the rules applied to the keys of the map. KeyRules *MapElementRules // ValueRules define the rules applied to the values of the map. @@ -78,12 +75,19 @@ type MapRules struct { // So the precedence is the following 1<2<3. // See API.RegisterTypeSettings() and WithTypeSettings() for more detail. type TypeSettings struct { - fieldKey *string + // fieldKey defines the key for the field used in json serialization. + fieldKey *string + // objectType defines the object type. It can be either uint8 or uint32 number. + objectType interface{} + // maxByteSize defines the max serialized byte size. 0 means unbounded. + maxByteSize uint + // lengthPrefixType defines the type of the value denoting the length of a collection. lengthPrefixType *LengthPrefixType - objectType interface{} - lexicalOrdering *bool - arrayRules *ArrayRules - mapRules *MapRules + // lexicalOrdering defines whether the collection must be lexically ordered during serialization. + lexicalOrdering *bool + // arrayRules defines rules around a to be deserialized array. + arrayRules *ArrayRules + mapRules *MapRules } // WithFieldKey specifies the key for the field. @@ -111,6 +115,28 @@ func (ts TypeSettings) MustFieldKey() string { return *ts.fieldKey } +// WithObjectType specifies the object type. It can be either uint8 or uint32 number. +// The object type holds two meanings: the actual code (number) and the serializer.TypeDenotationType like uint8 or uint32. +// serix uses object type to actually encode the number +// and to know its serializer.TypeDenotationType to be able to decode it. +func (ts TypeSettings) WithObjectType(t interface{}) TypeSettings { + ts.objectType = t + + return ts +} + +// ObjectType returns the object type as an uint8 or uint32 number. +func (ts TypeSettings) ObjectType() interface{} { + return ts.objectType +} + +// WithMaxByteSize specifies max serialized byte size for the type. 0 means unbounded. +func (ts TypeSettings) WithMaxByteSize(l uint) TypeSettings { + ts.maxByteSize = l + + return ts +} + // WithLengthPrefixType specifies LengthPrefixType. func (ts TypeSettings) WithLengthPrefixType(lpt LengthPrefixType) TypeSettings { ts.lengthPrefixType = &lpt @@ -127,21 +153,6 @@ func (ts TypeSettings) LengthPrefixType() (LengthPrefixType, bool) { return *ts.lengthPrefixType, true } -// WithObjectType specifies the object type. It can be either uint8 or uint32 number. -// The object type holds two meanings: the actual code (number) and the serializer.TypeDenotationType like uint8 or uint32. -// serix uses object type to actually encode the number -// and to know its serializer.TypeDenotationType to be able to decode it. -func (ts TypeSettings) WithObjectType(t interface{}) TypeSettings { - ts.objectType = t - - return ts -} - -// ObjectType returns the object type as an uint8 or uint32 number. -func (ts TypeSettings) ObjectType() interface{} { - return ts.objectType -} - // WithLexicalOrdering specifies whether the type must be lexically ordered during serialization. func (ts TypeSettings) WithLexicalOrdering(val bool) TypeSettings { ts.lexicalOrdering = &val @@ -234,16 +245,6 @@ func (ts TypeSettings) MapRules() *MapRules { return ts.mapRules } -// WithMapMaxByteSize specifies max serialized byte size for the map. 0 means unbounded. -func (ts TypeSettings) WithMapMaxByteSize(l uint) TypeSettings { - if ts.mapRules == nil { - ts.mapRules = new(MapRules) - } - ts.mapRules.MaxByteSize = l - - return ts -} - // WithMapKeyLengthPrefixType specifies MapKeyLengthPrefixType. func (ts TypeSettings) WithMapKeyLengthPrefixType(lpt LengthPrefixType) TypeSettings { if ts.mapRules == nil { From 4c5cdc10cff853dc4a806bc5219eb9b9eee4b367 Mon Sep 17 00:00:00 2001 From: muXxer Date: Wed, 8 Nov 2023 16:54:37 +0100 Subject: [PATCH 16/17] Remove map rules in favor of type settings for key and value --- serializer/serix/decode.go | 12 +-- serializer/serix/encode.go | 18 ++--- serializer/serix/map_decode.go | 15 ++-- serializer/serix/map_encode.go | 12 +-- serializer/serix/serix.go | 105 ++++++++++---------------- serializer/serix/serix_test.go | 77 +++++++++---------- serializer/serix/type_settings.go | 120 ------------------------------ 7 files changed, 105 insertions(+), 254 deletions(-) diff --git a/serializer/serix/decode.go b/serializer/serix/decode.go index 61fd153ce..8b504ae41 100644 --- a/serializer/serix/decode.go +++ b/serializer/serix/decode.go @@ -426,13 +426,9 @@ func (api *API) decodeSlice(ctx context.Context, b []byte, value reflect.Value, return bytesRead, nil } -func (api *API) decodeMapKVPair(ctx context.Context, b []byte, key, val reflect.Value, ts TypeSettings, opts *options) (int, error) { - keyTypeSettings := TypeSettings{} - valueTypeSettings := TypeSettings{} - if ts.mapRules != nil { - keyTypeSettings = ts.mapRules.KeyRules.ToTypeSettings() - valueTypeSettings = ts.mapRules.ValueRules.ToTypeSettings() - } +func (api *API) decodeMapKVPair(ctx context.Context, b []byte, key, val reflect.Value, opts *options) (int, error) { + keyTypeSettings := api.getTypeSettingsByValue(key) + valueTypeSettings := api.getTypeSettingsByValue(val) keyBytesRead, err := api.decode(ctx, b, key, keyTypeSettings, opts) if err != nil { @@ -456,7 +452,7 @@ func (api *API) decodeMap(ctx context.Context, b []byte, value reflect.Value, deserializeItem := func(b []byte) (bytesRead int, err error) { keyValue := reflect.New(valueType.Key()).Elem() elemValue := reflect.New(valueType.Elem()).Elem() - bytesRead, err = api.decodeMapKVPair(ctx, b, keyValue, elemValue, ts, opts) + bytesRead, err = api.decodeMapKVPair(ctx, b, keyValue, elemValue, opts) if err != nil { return 0, ierrors.WithStack(err) } diff --git a/serializer/serix/encode.go b/serializer/serix/encode.go index 36ce22a07..406435f12 100644 --- a/serializer/serix/encode.go +++ b/serializer/serix/encode.go @@ -114,7 +114,11 @@ func (api *API) encodeBasedOnType( if !set { return nil, ierrors.New("can't serialize 'string' type: no LengthPrefixType was provided") } - minLen, maxLen := ts.MinMaxLen() + + var minLen, maxLen int + if opts.validation { + minLen, maxLen = ts.MinMaxLen() + } seri := serializer.NewSerializer() return seri.WriteString( @@ -321,13 +325,9 @@ func (api *API) encodeSlice(ctx context.Context, value reflect.Value, valueType return encodeSliceOfBytes(data, valueType, ts, opts) } -func (api *API) encodeMapKVPair(ctx context.Context, key, val reflect.Value, ts TypeSettings, opts *options) ([]byte, error) { - keyTypeSettings := TypeSettings{} - valueTypeSettings := TypeSettings{} - if ts.mapRules != nil { - keyTypeSettings = ts.mapRules.KeyRules.ToTypeSettings() - valueTypeSettings = ts.mapRules.ValueRules.ToTypeSettings() - } +func (api *API) encodeMapKVPair(ctx context.Context, key, val reflect.Value, opts *options) ([]byte, error) { + keyTypeSettings := api.getTypeSettingsByValue(key) + valueTypeSettings := api.getTypeSettingsByValue(val) keyBytes, err := api.encode(ctx, key, keyTypeSettings, opts) if err != nil { @@ -359,7 +359,7 @@ func (api *API) encodeMap(ctx context.Context, value reflect.Value, valueType re for i := 0; iter.Next(); i++ { key := iter.Key() elem := iter.Value() - b, err := api.encodeMapKVPair(ctx, key, elem, ts, opts) + b, err := api.encodeMapKVPair(ctx, key, elem, opts) if err != nil { return nil, ierrors.WithStack(err) } diff --git a/serializer/serix/map_decode.go b/serializer/serix/map_decode.go index bc1e12947..c79237d34 100644 --- a/serializer/serix/map_decode.go +++ b/serializer/serix/map_decode.go @@ -467,17 +467,18 @@ func (api *API) mapDecodeMap(ctx context.Context, mapVal any, value reflect.Valu value.Set(reflect.MakeMap(valueType)) } - keyTypeSettings := TypeSettings{} - valueTypeSettings := TypeSettings{} - if ts.mapRules != nil { - keyTypeSettings = ts.mapRules.KeyRules.ToTypeSettings() - valueTypeSettings = ts.mapRules.ValueRules.ToTypeSettings() - } - + var typeSettingsSet bool + var keyTypeSettings, valueTypeSettings TypeSettings for k, v := range m { keyValue := reflect.New(valueType.Key()).Elem() elemValue := reflect.New(valueType.Elem()).Elem() + if !typeSettingsSet { + keyTypeSettings = api.getTypeSettingsByValue(keyValue) + valueTypeSettings = api.getTypeSettingsByValue(elemValue) + typeSettingsSet = true + } + if err := api.mapDecode(ctx, k, keyValue, keyTypeSettings, opts); err != nil { return ierrors.Wrapf(err, "failed to map decode map key of type %s", keyValue.Type()) } diff --git a/serializer/serix/map_encode.go b/serializer/serix/map_encode.go index b86dfc8cb..e81508cbf 100644 --- a/serializer/serix/map_encode.go +++ b/serializer/serix/map_encode.go @@ -282,13 +282,9 @@ func (api *API) mapEncodeSlice(ctx context.Context, value reflect.Value, valueTy return data, nil } -func (api *API) mapEncodeMapKVPair(ctx context.Context, key, val reflect.Value, ts TypeSettings, opts *options) (string, any, error) { - keyTypeSettings := TypeSettings{} - valueTypeSettings := TypeSettings{} - if ts.mapRules != nil { - keyTypeSettings = ts.mapRules.KeyRules.ToTypeSettings() - valueTypeSettings = ts.mapRules.ValueRules.ToTypeSettings() - } +func (api *API) mapEncodeMapKVPair(ctx context.Context, key, val reflect.Value, opts *options) (string, any, error) { + keyTypeSettings := api.getTypeSettingsByValue(key) + valueTypeSettings := api.getTypeSettingsByValue(val) k, err := api.mapEncode(ctx, key, keyTypeSettings, opts) if err != nil { @@ -316,7 +312,7 @@ func (api *API) mapEncodeMap(ctx context.Context, value reflect.Value, ts TypeSe for i := 0; iter.Next(); i++ { key := iter.Key() elem := iter.Value() - k, v, err := api.mapEncodeMapKVPair(ctx, key, elem, ts, opts) + k, v, err := api.mapEncodeMapKVPair(ctx, key, elem, opts) if err != nil { return nil, ierrors.WithStack(err) } diff --git a/serializer/serix/serix.go b/serializer/serix/serix.go index 0e1ceaf62..3ce95108a 100644 --- a/serializer/serix/serix.go +++ b/serializer/serix/serix.go @@ -5,7 +5,12 @@ Structs serialization/deserialization In order for a field to be detected by serix it must have `serix:""` struct tag. The first part in the tag is the key used for json serialization. -If the name is empty, serix uses the field name in camel case (Exceptions: "ID" => "Id", "URL" => "Url"). +If the name is empty, serix uses the field name in camel case. + Exceptions: + - "ID" => "Id" + - "NFT" => "Nft" + - "URL" => "Url" + - "HRP" => "Hrp" Examples: - `serix:"" @@ -37,25 +42,6 @@ You can provide the following settings to serix via struct tags: - "maxLen": maximum length for that field (string, slice, map) `serix:"example,maxLen=5"` - - "mapKeyLenPrefix": provide serializer.SeriLengthPrefixType for the keys of that map - `serix:"example,mapKeyLenPrefix=uint32"` - - - "mapKeyMinLen": minimum length for the keys of that map - `serix:"example,mapKeyMinLen=2"` - - - "mapKeyMaxLen": maximum length for the keys of that map - `serix:"example,mapKeyMaxLen=5"` - - - "mapValueLenPrefix": provide serializer.SeriLengthPrefixType for the values of that map - `serix:"example,mapValueLenPrefix=uint32"` - - - "mapValueMinLen": minimum length for the values of that map - `serix:"example,mapValueMinLen=2"` - - - "mapValueMaxLen": maximum length for the values of that map - `serix:"example,mapValueMaxLen=5"` - - See serix_text.go for more detail. */ package serix @@ -556,6 +542,7 @@ func (api *API) RegisterTypeSettings(obj interface{}, ts TypeSettings) error { if objType == nil { return ierrors.New("'obj' is a nil interface, it's need to be a valid type") } + api.typeSettingsRegistryMutex.Lock() defer api.typeSettingsRegistryMutex.Unlock() api.typeSettingsRegistry[objType] = ts @@ -566,6 +553,7 @@ func (api *API) RegisterTypeSettings(obj interface{}, ts TypeSettings) error { func (api *API) getTypeSettings(objType reflect.Type) (TypeSettings, bool) { api.typeSettingsRegistryMutex.RLock() defer api.typeSettingsRegistryMutex.RUnlock() + ts, ok := api.typeSettingsRegistry[objType] if ok { return ts, true @@ -580,6 +568,35 @@ func (api *API) getTypeSettings(objType reflect.Type) (TypeSettings, bool) { return TypeSettings{}, false } +//nolint:unparam // false positive, we will use it later +func (api *API) getTypeSettingsByValue(objValue reflect.Value, optTS ...TypeSettings) TypeSettings { + api.typeSettingsRegistryMutex.RLock() + defer api.typeSettingsRegistryMutex.RUnlock() + + for { + if ts, ok := api.typeSettingsRegistry[objValue.Type()]; ok { + if len(optTS) > 0 { + return optTS[0].merge(ts) + } + + return ts + } + + // resolve indirections + switch objValue.Kind() { + case reflect.Ptr, reflect.Interface: + objValue = objValue.Elem() + + default: + if len(optTS) > 0 { + return optTS[0] + } + + return TypeSettings{} + } + } +} + // RegisterInterfaceObjects tells serix that when it encounters iType during serialization/deserialization // it actually might be one of the objs types. // Those objs type must provide their ObjectTypes beforehand via API.RegisterTypeSettings(). @@ -868,7 +885,9 @@ func parseSerixSettings(tag string, serixPosition int) (tagSettings, error) { return tagSettings{}, ierrors.Errorf("incorrect struct tag format: %s, must start with the field key or \",\"", tag) } - settings.ts = settings.ts.WithFieldKey(keyPart) + if keyPart != "" { + settings.ts = settings.ts.WithFieldKey(keyPart) + } parts = parts[1:] seenParts := map[string]struct{}{} @@ -917,48 +936,6 @@ func parseSerixSettings(tag string, serixPosition int) (tagSettings, error) { } settings.ts = settings.ts.WithMaxLen(value) - case "mapKeyLenPrefix": - value, err := parseStructTagValuePrefixType("mapKeyLenPrefix", keyValue, currentPart) - if err != nil { - return tagSettings{}, err - } - settings.ts = settings.ts.WithMapKeyLengthPrefixType(value) - - case "mapKeyMinLen": - value, err := parseStructTagValueUint("mapKeyMinLen", keyValue, currentPart) - if err != nil { - return tagSettings{}, err - } - settings.ts = settings.ts.WithMapKeyMinLen(value) - - case "mapKeyMaxLen": - value, err := parseStructTagValueUint("mapKeyMaxLen", keyValue, currentPart) - if err != nil { - return tagSettings{}, err - } - settings.ts = settings.ts.WithMapKeyMaxLen(value) - - case "mapValueLenPrefix": - value, err := parseStructTagValuePrefixType("mapValueLenPrefix", keyValue, currentPart) - if err != nil { - return tagSettings{}, err - } - settings.ts = settings.ts.WithMapValueLengthPrefixType(value) - - case "mapValueMinLen": - value, err := parseStructTagValueUint("mapValueMinLen", keyValue, currentPart) - if err != nil { - return tagSettings{}, err - } - settings.ts = settings.ts.WithMapValueMinLen(value) - - case "mapValueMaxLen": - value, err := parseStructTagValueUint("mapValueMaxLen", keyValue, currentPart) - if err != nil { - return tagSettings{}, err - } - settings.ts = settings.ts.WithMapValueMaxLen(value) - default: return tagSettings{}, ierrors.Errorf("unknown tag part: %s", currentPart) } @@ -1048,7 +1025,7 @@ func getNumberTypeToConvert(kind reflect.Kind) (int, reflect.Type, reflect.Type) // FieldKeyString converts the given string to camelCase. // Special keywords like ID or URL are converted to only first letter upper case. func FieldKeyString(str string) string { - for _, keyword := range []string{"ID", "URL"} { + for _, keyword := range []string{"ID", "NFT", "URL", "HRP"} { if !strings.Contains(str, keyword) { continue } diff --git a/serializer/serix/serix_test.go b/serializer/serix/serix_test.go index dd5ce3565..7bf59db9c 100644 --- a/serializer/serix/serix_test.go +++ b/serializer/serix/serix_test.go @@ -178,18 +178,22 @@ func (test *serializeTest) run(t *testing.T) { func TestSerixMapSerialize(t *testing.T) { - type MyMapType map[string]string - + type MyMapTypeKey string + type MyMapTypeValue string + type MyMapType map[MyMapTypeKey]MyMapTypeValue type MapStruct struct { - MyMap MyMapType `serix:",lenPrefix=uint8,minLen=2,maxLen=4,maxByteSize=50,mapKeyLenPrefix=uint16,mapKeyMinLen=2,mapKeyMaxLen=5,mapValueLenPrefix=uint32,mapValueMinLen=1,mapValueMaxLen=6"` + MyMap MyMapType `serix:",lenPrefix=uint8,minLen=2,maxLen=4,maxByteSize=50"` } + + testAPI.RegisterTypeSettings(MyMapTypeKey(""), serix.TypeSettings{}.WithLengthPrefixType(serix.LengthPrefixTypeAsUint16).WithMinLen(2).WithMaxLen(5)) + testAPI.RegisterTypeSettings(MyMapTypeValue(""), serix.TypeSettings{}.WithLengthPrefixType(serix.LengthPrefixTypeAsUint32).WithMinLen(1).WithMaxLen(6)) testAPI.RegisterTypeSettings(MapStruct{}, serix.TypeSettings{}) tests := []serializeTest{ { name: "ok", source: &MapStruct{ - MyMap: map[string]string{ + MyMap: MyMapType{ "k1": "v1", "k2": "v2", }, @@ -201,7 +205,7 @@ func TestSerixMapSerialize(t *testing.T) { { name: "fail - not enough entries", source: &MapStruct{ - MyMap: map[string]string{ + MyMap: MyMapType{ "k1": "v1", }, }, @@ -212,7 +216,7 @@ func TestSerixMapSerialize(t *testing.T) { { name: "fail - too many entries", source: &MapStruct{ - MyMap: map[string]string{ + MyMap: MyMapType{ "k1": "v1", "k2": "v2", "k3": "v3", @@ -227,7 +231,7 @@ func TestSerixMapSerialize(t *testing.T) { { name: "fail - too big", source: &MapStruct{ - MyMap: map[string]string{ + MyMap: MyMapType{ "k1": "v1000", "k2": "v2000", "k3": "v3000", @@ -241,7 +245,7 @@ func TestSerixMapSerialize(t *testing.T) { { name: "fail - key too short", source: &MapStruct{ - MyMap: map[string]string{ + MyMap: MyMapType{ "k1": "v1", "k": "v2", }, @@ -253,7 +257,7 @@ func TestSerixMapSerialize(t *testing.T) { { name: "fail - key too long", source: &MapStruct{ - MyMap: map[string]string{ + MyMap: MyMapType{ "k1": "v1", "k20000": "v2", }, @@ -265,7 +269,7 @@ func TestSerixMapSerialize(t *testing.T) { { name: "fail - value too short", source: &MapStruct{ - MyMap: map[string]string{ + MyMap: MyMapType{ "k1": "v1", "k2": "", }, @@ -277,7 +281,7 @@ func TestSerixMapSerialize(t *testing.T) { { name: "fail - value too long", source: &MapStruct{ - MyMap: map[string]string{ + MyMap: MyMapType{ "k1": "v1", "k2": "v200000", }, @@ -303,11 +307,11 @@ type deSerializeTest struct { func (test *deSerializeTest) run(t *testing.T) { // binary serialize test data - serixData, err := testAPI.Encode(context.Background(), test.source, serix.WithValidation()) + serixData, err := testAPI.Encode(context.Background(), test.source) require.NoError(t, err) // json serialize test data - sourceJSON, err := testAPI.JSONEncode(context.Background(), test.source, serix.WithValidation()) + sourceJSON, err := testAPI.JSONEncode(context.Background(), test.source) require.NoError(t, err) // binary deserialize @@ -337,24 +341,21 @@ func (test *deSerializeTest) run(t *testing.T) { func TestSerixMapDeserialize(t *testing.T) { - type MyMapType map[string]string - - // used to create test data - type TestVectorMapStruct struct { - MyMap MyMapType `serix:",lenPrefix=uint8,minLen=1,maxLen=5,maxByteSize=100,mapKeyLenPrefix=uint16,mapKeyMinLen=1,mapKeyMaxLen=7,mapValueLenPrefix=uint32,mapValueMinLen=0,mapValueMaxLen=10"` - } - testAPI.RegisterTypeSettings(TestVectorMapStruct{}, serix.TypeSettings{}) - + type MyMapTypeKey string + type MyMapTypeValue string type MapStruct struct { - MyMap MyMapType `serix:",lenPrefix=uint8,minLen=2,maxLen=4,maxByteSize=50,mapKeyLenPrefix=uint16,mapKeyMinLen=2,mapKeyMaxLen=5,mapValueLenPrefix=uint32,mapValueMinLen=1,mapValueMaxLen=6"` + MyMap map[MyMapTypeKey]MyMapTypeValue `serix:",lenPrefix=uint8,minLen=2,maxLen=4,maxByteSize=50"` } + + testAPI.RegisterTypeSettings(MyMapTypeKey(""), serix.TypeSettings{}.WithLengthPrefixType(serix.LengthPrefixTypeAsUint16).WithMinLen(2).WithMaxLen(5)) + testAPI.RegisterTypeSettings(MyMapTypeValue(""), serix.TypeSettings{}.WithLengthPrefixType(serix.LengthPrefixTypeAsUint32).WithMinLen(1).WithMaxLen(6)) testAPI.RegisterTypeSettings(MapStruct{}, serix.TypeSettings{}) tests := []deSerializeTest{ { name: "ok", - source: &TestVectorMapStruct{ - MyMap: map[string]string{ + source: &MapStruct{ + MyMap: map[MyMapTypeKey]MyMapTypeValue{ "k1": "v1", "k2": "v2", }, @@ -365,8 +366,8 @@ func TestSerixMapDeserialize(t *testing.T) { }, { name: "fail - not enough entries", - source: &TestVectorMapStruct{ - MyMap: map[string]string{ + source: &MapStruct{ + MyMap: map[MyMapTypeKey]MyMapTypeValue{ "k1": "v1", }, }, @@ -376,8 +377,8 @@ func TestSerixMapDeserialize(t *testing.T) { }, { name: "fail - too many entries", - source: &TestVectorMapStruct{ - MyMap: map[string]string{ + source: &MapStruct{ + MyMap: map[MyMapTypeKey]MyMapTypeValue{ "k1": "v1", "k2": "v2", "k3": "v3", @@ -391,8 +392,8 @@ func TestSerixMapDeserialize(t *testing.T) { }, { name: "fail - too big", - source: &TestVectorMapStruct{ - MyMap: map[string]string{ + source: &MapStruct{ + MyMap: map[MyMapTypeKey]MyMapTypeValue{ "k1": "v1000", "k2": "v2000", "k3": "v3000", @@ -405,8 +406,8 @@ func TestSerixMapDeserialize(t *testing.T) { }, { name: "fail - key too short", - source: &TestVectorMapStruct{ - MyMap: map[string]string{ + source: &MapStruct{ + MyMap: map[MyMapTypeKey]MyMapTypeValue{ "k1": "v1", "k": "v2", }, @@ -417,8 +418,8 @@ func TestSerixMapDeserialize(t *testing.T) { }, { name: "fail - key too long", - source: &TestVectorMapStruct{ - MyMap: map[string]string{ + source: &MapStruct{ + MyMap: map[MyMapTypeKey]MyMapTypeValue{ "k1": "v1", "k20000": "v2", }, @@ -429,8 +430,8 @@ func TestSerixMapDeserialize(t *testing.T) { }, { name: "fail - value too short", - source: &TestVectorMapStruct{ - MyMap: map[string]string{ + source: &MapStruct{ + MyMap: map[MyMapTypeKey]MyMapTypeValue{ "k1": "v1", "k2": "", }, @@ -441,8 +442,8 @@ func TestSerixMapDeserialize(t *testing.T) { }, { name: "fail - value too long", - source: &TestVectorMapStruct{ - MyMap: map[string]string{ + source: &MapStruct{ + MyMap: map[MyMapTypeKey]MyMapTypeValue{ "k1": "v1", "k2": "v200000", }, diff --git a/serializer/serix/type_settings.go b/serializer/serix/type_settings.go index d529d52ba..e0a2ee4f5 100644 --- a/serializer/serix/type_settings.go +++ b/serializer/serix/type_settings.go @@ -39,32 +39,6 @@ func LengthPrefixTypeSize(t LengthPrefixType) (int, error) { // Min and Max at 0 define an unbounded array. type ArrayRules serializer.ArrayRules -// MapElementRules defines rules around to be deserialized map elements (key or value). -// MinLength and MaxLength at 0 define an unbounded map element. -type MapElementRules struct { - LengthPrefixType *LengthPrefixType - MinLength uint - MaxLength uint -} - -func (m *MapElementRules) ToTypeSettings() TypeSettings { - return TypeSettings{ - lengthPrefixType: m.LengthPrefixType, - arrayRules: &ArrayRules{ - Min: m.MinLength, - Max: m.MaxLength, - }, - } -} - -// MapRules defines rules around a to be deserialized map. -type MapRules struct { - // KeyRules define the rules applied to the keys of the map. - KeyRules *MapElementRules - // ValueRules define the rules applied to the values of the map. - ValueRules *MapElementRules -} - // TypeSettings holds various settings for a particular type. // Those settings determine how the object should be serialized/deserialized. // There are three ways to provide TypeSettings @@ -87,7 +61,6 @@ type TypeSettings struct { lexicalOrdering *bool // arrayRules defines rules around a to be deserialized array. arrayRules *ArrayRules - mapRules *MapRules } // WithFieldKey specifies the key for the field. @@ -233,96 +206,6 @@ func (ts TypeSettings) MinMaxLen() (int, int) { return min, max } -// WithMapRules specifies the map rules. -func (ts TypeSettings) WithMapRules(rules *MapRules) TypeSettings { - ts.mapRules = rules - - return ts -} - -// MapRules returns the map rules. -func (ts TypeSettings) MapRules() *MapRules { - return ts.mapRules -} - -// WithMapKeyLengthPrefixType specifies MapKeyLengthPrefixType. -func (ts TypeSettings) WithMapKeyLengthPrefixType(lpt LengthPrefixType) TypeSettings { - if ts.mapRules == nil { - ts.mapRules = new(MapRules) - } - if ts.mapRules.KeyRules == nil { - ts.mapRules.KeyRules = new(MapElementRules) - } - ts.mapRules.KeyRules.LengthPrefixType = &lpt - - return ts -} - -// WithMapKeyMinLen specifies the min length for the object in the map key. -func (ts TypeSettings) WithMapKeyMinLen(l uint) TypeSettings { - if ts.mapRules == nil { - ts.mapRules = new(MapRules) - } - if ts.mapRules.KeyRules == nil { - ts.mapRules.KeyRules = new(MapElementRules) - } - ts.mapRules.KeyRules.MinLength = l - - return ts -} - -// WithMapKeyMaxLen specifies the max length for the object in the map key. -func (ts TypeSettings) WithMapKeyMaxLen(l uint) TypeSettings { - if ts.mapRules == nil { - ts.mapRules = new(MapRules) - } - if ts.mapRules.KeyRules == nil { - ts.mapRules.KeyRules = new(MapElementRules) - } - ts.mapRules.KeyRules.MaxLength = l - - return ts -} - -// MapValueLengthPrefixType specifies MapValueLengthPrefixType. -func (ts TypeSettings) WithMapValueLengthPrefixType(lpt LengthPrefixType) TypeSettings { - if ts.mapRules == nil { - ts.mapRules = new(MapRules) - } - if ts.mapRules.ValueRules == nil { - ts.mapRules.ValueRules = new(MapElementRules) - } - ts.mapRules.ValueRules.LengthPrefixType = &lpt - - return ts -} - -// WithMapValueMinLen specifies the min length for the object in the map value. -func (ts TypeSettings) WithMapValueMinLen(l uint) TypeSettings { - if ts.mapRules == nil { - ts.mapRules = new(MapRules) - } - if ts.mapRules.ValueRules == nil { - ts.mapRules.ValueRules = new(MapElementRules) - } - ts.mapRules.ValueRules.MinLength = l - - return ts -} - -// WithMapValueMaxLen specifies the max length for the object in the map value. -func (ts TypeSettings) WithMapValueMaxLen(l uint) TypeSettings { - if ts.mapRules == nil { - ts.mapRules = new(MapRules) - } - if ts.mapRules.ValueRules == nil { - ts.mapRules.ValueRules = new(MapElementRules) - } - ts.mapRules.ValueRules.MaxLength = l - - return ts -} - func (ts TypeSettings) ensureOrdering() TypeSettings { newTS := ts.WithLexicalOrdering(true) arrayRules := newTS.ArrayRules() @@ -351,9 +234,6 @@ func (ts TypeSettings) merge(other TypeSettings) TypeSettings { if ts.fieldKey == nil { ts.fieldKey = other.fieldKey } - if ts.mapRules == nil { - ts.mapRules = other.mapRules - } return ts } From 33a8a640d459d4fdd1d54dd992562d85e964c668 Mon Sep 17 00:00:00 2001 From: muXxer Date: Wed, 8 Nov 2023 17:04:37 +0100 Subject: [PATCH 17/17] Add LengthPrefixTypeAsUint64 --- serializer/serix/serix.go | 2 ++ serializer/serix/type_settings.go | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/serializer/serix/serix.go b/serializer/serix/serix.go index 3ce95108a..c3ebf318f 100644 --- a/serializer/serix/serix.go +++ b/serializer/serix/serix.go @@ -850,6 +850,8 @@ func parseLengthPrefixType(prefixTypeRaw string) (LengthPrefixType, error) { return LengthPrefixTypeAsUint16, nil case "uint32": return LengthPrefixTypeAsUint32, nil + case "uint64": + return LengthPrefixTypeAsUint64, nil default: return LengthPrefixTypeAsByte, ierrors.Wrapf(ErrUnknownLengthPrefixType, "%s", prefixTypeRaw) } diff --git a/serializer/serix/type_settings.go b/serializer/serix/type_settings.go index e0a2ee4f5..d391bd0da 100644 --- a/serializer/serix/type_settings.go +++ b/serializer/serix/type_settings.go @@ -20,6 +20,8 @@ const ( LengthPrefixTypeAsUint16 = LengthPrefixType(serializer.SeriLengthPrefixTypeAsUint16) // LengthPrefixTypeAsUint32 defines a collection length to be denoted by a uint32. LengthPrefixTypeAsUint32 = LengthPrefixType(serializer.SeriLengthPrefixTypeAsUint32) + // LengthPrefixTypeAsUint64 defines a collection length to be denoted by a uint64. + LengthPrefixTypeAsUint64 = LengthPrefixType(serializer.SeriLengthPrefixTypeAsUint64) ) func LengthPrefixTypeSize(t LengthPrefixType) (int, error) { @@ -30,6 +32,8 @@ func LengthPrefixTypeSize(t LengthPrefixType) (int, error) { return 2, nil case LengthPrefixTypeAsUint32: return 4, nil + case LengthPrefixTypeAsUint64: + return 8, nil default: return 0, ErrUnknownLengthPrefixType }