From 34acb612c505ae2251d9eafcb4e418f4c1627eb1 Mon Sep 17 00:00:00 2001 From: John Jannotti Date: Wed, 7 Oct 2026 13:43:11 -0400 Subject: [PATCH] Allow maxtotalbytes to be specified on top types --- parse/directives.go | 35 ++- tests/maxtotalbytes.go | 33 +++ tests/maxtotalbytes_gen.go | 375 ++++++++++++++++++++++++++++++++ tests/maxtotalbytes_gen_test.go | 194 +++++++++++++++++ tests/maxtotalbytes_test.go | 56 +++++ 5 files changed, 688 insertions(+), 5 deletions(-) create mode 100644 tests/maxtotalbytes.go create mode 100644 tests/maxtotalbytes_gen.go create mode 100644 tests/maxtotalbytes_gen_test.go create mode 100644 tests/maxtotalbytes_test.go diff --git a/parse/directives.go b/parse/directives.go index 75d0011..3d8cddc 100644 --- a/parse/directives.go +++ b/parse/directives.go @@ -22,11 +22,12 @@ type passDirective func(gen.Method, []string, *gen.Printer) error // to add a directive, define a func([]string, *FileSet) error // and then add it to this list. var directives = map[string]directive{ - "shim": applyShim, - "ignore": ignore, - "tuple": astuple, - "sort": sortintf, - "allocbound": allocbound, + "shim": applyShim, + "ignore": ignore, + "tuple": astuple, + "sort": sortintf, + "allocbound": allocbound, + "maxtotalbytes": maxtotalbytes, // _postunmarshalcheck is used to add callbacks to the end of un-marshalling that are tied to a specific Element. _postunmarshalcheck: postunmarshalcheck, } @@ -192,3 +193,27 @@ func allocbound(text []string, f *FileSet) error { } return nil } + +// maxtotalbytes is the type-level counterpart of the maxtotalbytes= field tag, +// for named slice types that have no field to carry the tag. As with the tag, +// Bound replaces the generated MaxSize for the slice's contents, so it must +// include each element's msgp prefix. Directives are split on spaces, so the +// rest of the line is rejoined to allow Bound to be an expression like +// "(N*msgp.BytesPrefixSize) + M". +// +//msgp:maxtotalbytes {Type} {Bound} +func maxtotalbytes(text []string, f *FileSet) error { + if len(text) < 3 { + return fmt.Errorf("maxtotalbytes directive should have 2 arguments; found %d", len(text)-1) + } + maxTotalBytesType := strings.TrimSpace(text[1]) + maxTotalBytes := strings.TrimSpace(strings.Join(text[2:], " ")) + t, ok := f.Identities[maxTotalBytesType] + if !ok { + warnf("maxtotalbytes: cannot find type %s\n", maxTotalBytesType) + } else { + t.SetMaxTotalBytes(maxTotalBytes) + infof("maxtotalbytes(%s): setting to %s\n", maxTotalBytesType, maxTotalBytes) + } + return nil +} diff --git a/tests/maxtotalbytes.go b/tests/maxtotalbytes.go new file mode 100644 index 0000000..87ecce6 --- /dev/null +++ b/tests/maxtotalbytes.go @@ -0,0 +1,33 @@ +// This file exercises the maxtotalbytes directive, which gives a named slice +// type the same tight MaxSize bound that the maxtotalbytes= field tag gives a +// struct field. +package tests + +//go:generate msgp + +const ( + argCount = 8 + argMaxSize = 100 + // argsTotal is much smaller than argCount*argMaxSize, so a MaxSize that + // uses it is visibly tighter than the product of the allocbounds. + argsTotal = 200 +) + +// BoundedArgs has only allocbounds, so its MaxSize is the product of them. +// +//msgp:allocbound BoundedArgs argCount,argMaxSize +type BoundedArgs [][]byte + +// TotalBoundedArgs adds a maxtotalbytes directive whose bound is an +// expression containing spaces, to check that the whole expression survives. +// +//msgp:allocbound TotalBoundedArgs argCount,argMaxSize +//msgp:maxtotalbytes TotalBoundedArgs (argCount*msgp.BytesPrefixSize) + argsTotal +type TotalBoundedArgs [][]byte + +// ArgsHolder holds TotalBoundedArgs, and should use its MaxSize. +type ArgsHolder struct { + _struct struct{} `codec:",omitempty"` + + Args TotalBoundedArgs `codec:"a"` +} diff --git a/tests/maxtotalbytes_gen.go b/tests/maxtotalbytes_gen.go new file mode 100644 index 0000000..ccdbeb7 --- /dev/null +++ b/tests/maxtotalbytes_gen.go @@ -0,0 +1,375 @@ +// Code generated by github.com/algorand/msgp DO NOT EDIT. + +package tests + +import ( + "github.com/algorand/msgp/msgp" +) + +// The following msgp objects are implemented in this file: +// ArgsHolder +// |-----> (*) MarshalMsg +// |-----> (*) CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) UnmarshalMsgWithState +// |-----> (*) CanUnmarshalMsg +// |-----> (*) Msgsize +// |-----> (*) MsgIsZero +// |-----> ArgsHolderMaxSize() +// +// BoundedArgs +// |-----> MarshalMsg +// |-----> CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) UnmarshalMsgWithState +// |-----> (*) CanUnmarshalMsg +// |-----> Msgsize +// |-----> MsgIsZero +// |-----> BoundedArgsMaxSize() +// +// TotalBoundedArgs +// |-----> MarshalMsg +// |-----> CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) UnmarshalMsgWithState +// |-----> (*) CanUnmarshalMsg +// |-----> Msgsize +// |-----> MsgIsZero +// |-----> TotalBoundedArgsMaxSize() +// + +// MarshalMsg implements msgp.Marshaler +func (z *ArgsHolder) MarshalMsg(b []byte) (o []byte) { + o = msgp.Require(b, z.Msgsize()) + // omitempty: check for empty values + zb0002Len := uint32(1) + var zb0002Mask uint8 /* 2 bits */ + if len((*z).Args) == 0 { + zb0002Len-- + zb0002Mask |= 0x2 + } + // variable map header, size zb0002Len + o = append(o, 0x80|uint8(zb0002Len)) + if zb0002Len != 0 { + if (zb0002Mask & 0x2) == 0 { // if not empty + // string "a" + o = append(o, 0xa1, 0x61) + if (*z).Args == nil { + o = msgp.AppendNil(o) + } else { + o = msgp.AppendArrayHeader(o, uint32(len((*z).Args))) + } + for zb0001 := range (*z).Args { + o = msgp.AppendBytes(o, (*z).Args[zb0001]) + } + } + } + return +} + +func (_ *ArgsHolder) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(*ArgsHolder) + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ArgsHolder) UnmarshalMsgWithState(bts []byte, st msgp.UnmarshalState) (o []byte, err error) { + if st.AllowableDepth == 0 { + err = msgp.ErrMaxDepthExceeded{} + return + } + st.AllowableDepth-- + var field []byte + _ = field + var zb0002 int + var zb0003 bool + zb0002, zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0003 { + (*z) = ArgsHolder{} + } + for zb0002 > 0 { + zb0002-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch string(field) { + case "a": + var zb0004 int + var zb0005 bool + zb0004, zb0005, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Args") + return + } + if zb0004 > argCount { + err = msgp.ErrOverflow(uint64(zb0004), uint64(argCount)) + err = msgp.WrapError(err, "Args") + return + } + if zb0005 { + (*z).Args = nil + } else if (*z).Args != nil && cap((*z).Args) >= zb0004 { + (*z).Args = ((*z).Args)[:zb0004] + } else { + (*z).Args = make(TotalBoundedArgs, zb0004) + } + for zb0001 := range (*z).Args { + var zb0006 int + zb0006, err = msgp.ReadBytesBytesHeader(bts) + if err != nil { + err = msgp.WrapError(err, "Args", zb0001) + return + } + if zb0006 > argMaxSize { + err = msgp.ErrOverflow(uint64(zb0006), uint64(argMaxSize)) + return + } + (*z).Args[zb0001], bts, err = msgp.ReadBytesBytes(bts, (*z).Args[zb0001]) + if err != nil { + err = msgp.WrapError(err, "Args", zb0001) + return + } + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +func (z *ArgsHolder) UnmarshalMsg(bts []byte) (o []byte, err error) { + return z.UnmarshalMsgWithState(bts, msgp.DefaultUnmarshalState) +} +func (_ *ArgsHolder) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*ArgsHolder) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ArgsHolder) Msgsize() (s int) { + s = 1 + 2 + msgp.ArrayHeaderSize + for zb0001 := range (*z).Args { + s += msgp.BytesPrefixSize + len((*z).Args[zb0001]) + } + return +} + +// MsgIsZero returns whether this is a zero value +func (z *ArgsHolder) MsgIsZero() bool { + return (len((*z).Args) == 0) +} + +// ArgsHolderMaxSize returns a maximum valid message size for this message type +func ArgsHolderMaxSize() (s int) { + s = 1 + 2 + // Calculating size of slice: z.Args + s += msgp.ArrayHeaderSize + (argCount * msgp.BytesPrefixSize) + argsTotal + return +} + +// MarshalMsg implements msgp.Marshaler +func (z BoundedArgs) MarshalMsg(b []byte) (o []byte) { + o = msgp.Require(b, z.Msgsize()) + if z == nil { + o = msgp.AppendNil(o) + } else { + o = msgp.AppendArrayHeader(o, uint32(len(z))) + } + for za0001 := range z { + o = msgp.AppendBytes(o, z[za0001]) + } + return +} + +func (_ BoundedArgs) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(BoundedArgs) + if !ok { + _, ok = (z).(*BoundedArgs) + } + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BoundedArgs) UnmarshalMsgWithState(bts []byte, st msgp.UnmarshalState) (o []byte, err error) { + if st.AllowableDepth == 0 { + err = msgp.ErrMaxDepthExceeded{} + return + } + st.AllowableDepth-- + var zb0002 int + var zb0003 bool + zb0002, zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0002 > argCount { + err = msgp.ErrOverflow(uint64(zb0002), uint64(argCount)) + err = msgp.WrapError(err) + return + } + if zb0003 { + (*z) = nil + } else if (*z) != nil && cap((*z)) >= zb0002 { + (*z) = (*z)[:zb0002] + } else { + (*z) = make(BoundedArgs, zb0002) + } + for zb0001 := range *z { + var zb0004 int + zb0004, err = msgp.ReadBytesBytesHeader(bts) + if err != nil { + err = msgp.WrapError(err, zb0001) + return + } + if zb0004 > argMaxSize { + err = msgp.ErrOverflow(uint64(zb0004), uint64(argMaxSize)) + return + } + (*z)[zb0001], bts, err = msgp.ReadBytesBytes(bts, (*z)[zb0001]) + if err != nil { + err = msgp.WrapError(err, zb0001) + return + } + } + o = bts + return +} + +func (z *BoundedArgs) UnmarshalMsg(bts []byte) (o []byte, err error) { + return z.UnmarshalMsgWithState(bts, msgp.DefaultUnmarshalState) +} +func (_ *BoundedArgs) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*BoundedArgs) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z BoundedArgs) Msgsize() (s int) { + s = msgp.ArrayHeaderSize + for za0001 := range z { + s += msgp.BytesPrefixSize + len(z[za0001]) + } + return +} + +// MsgIsZero returns whether this is a zero value +func (z BoundedArgs) MsgIsZero() bool { + return len(z) == 0 +} + +// BoundedArgsMaxSize returns a maximum valid message size for this message type +func BoundedArgsMaxSize() (s int) { + // Calculating size of slice: z + s += msgp.ArrayHeaderSize + ((argCount) * (msgp.BytesPrefixSize + argMaxSize)) + return +} + +// MarshalMsg implements msgp.Marshaler +func (z TotalBoundedArgs) MarshalMsg(b []byte) (o []byte) { + o = msgp.Require(b, z.Msgsize()) + if z == nil { + o = msgp.AppendNil(o) + } else { + o = msgp.AppendArrayHeader(o, uint32(len(z))) + } + for za0001 := range z { + o = msgp.AppendBytes(o, z[za0001]) + } + return +} + +func (_ TotalBoundedArgs) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(TotalBoundedArgs) + if !ok { + _, ok = (z).(*TotalBoundedArgs) + } + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *TotalBoundedArgs) UnmarshalMsgWithState(bts []byte, st msgp.UnmarshalState) (o []byte, err error) { + if st.AllowableDepth == 0 { + err = msgp.ErrMaxDepthExceeded{} + return + } + st.AllowableDepth-- + var zb0002 int + var zb0003 bool + zb0002, zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0002 > argCount { + err = msgp.ErrOverflow(uint64(zb0002), uint64(argCount)) + err = msgp.WrapError(err) + return + } + if zb0003 { + (*z) = nil + } else if (*z) != nil && cap((*z)) >= zb0002 { + (*z) = (*z)[:zb0002] + } else { + (*z) = make(TotalBoundedArgs, zb0002) + } + for zb0001 := range *z { + var zb0004 int + zb0004, err = msgp.ReadBytesBytesHeader(bts) + if err != nil { + err = msgp.WrapError(err, zb0001) + return + } + if zb0004 > argMaxSize { + err = msgp.ErrOverflow(uint64(zb0004), uint64(argMaxSize)) + return + } + (*z)[zb0001], bts, err = msgp.ReadBytesBytes(bts, (*z)[zb0001]) + if err != nil { + err = msgp.WrapError(err, zb0001) + return + } + } + o = bts + return +} + +func (z *TotalBoundedArgs) UnmarshalMsg(bts []byte) (o []byte, err error) { + return z.UnmarshalMsgWithState(bts, msgp.DefaultUnmarshalState) +} +func (_ *TotalBoundedArgs) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*TotalBoundedArgs) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z TotalBoundedArgs) Msgsize() (s int) { + s = msgp.ArrayHeaderSize + for za0001 := range z { + s += msgp.BytesPrefixSize + len(z[za0001]) + } + return +} + +// MsgIsZero returns whether this is a zero value +func (z TotalBoundedArgs) MsgIsZero() bool { + return len(z) == 0 +} + +// TotalBoundedArgsMaxSize returns a maximum valid message size for this message type +func TotalBoundedArgsMaxSize() (s int) { + // Calculating size of slice: z + s += msgp.ArrayHeaderSize + (argCount * msgp.BytesPrefixSize) + argsTotal + return +} diff --git a/tests/maxtotalbytes_gen_test.go b/tests/maxtotalbytes_gen_test.go new file mode 100644 index 0000000..2fcfcd9 --- /dev/null +++ b/tests/maxtotalbytes_gen_test.go @@ -0,0 +1,194 @@ +//go:build !skip_msgp_testing + +// Code generated by github.com/algorand/msgp DO NOT EDIT. + +package tests + +import ( + "testing" + + "github.com/algorand/msgp/msgp" + + "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/test/partitiontest" +) + +func TestMarshalUnmarshalArgsHolder(t *testing.T) { + partitiontest.PartitionTest(t) + v := ArgsHolder{} + bts := v.MarshalMsg(nil) + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func TestRandomizedEncodingArgsHolder(t *testing.T) { + protocol.RunEncodingTest(t, &ArgsHolder{}) +} + +func BenchmarkMarshalMsgArgsHolder(b *testing.B) { + v := ArgsHolder{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgArgsHolder(b *testing.B) { + v := ArgsHolder{} + bts := make([]byte, 0, v.Msgsize()) + bts = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalArgsHolder(b *testing.B) { + v := ArgsHolder{} + bts := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalBoundedArgs(t *testing.T) { + partitiontest.PartitionTest(t) + v := BoundedArgs{} + bts := v.MarshalMsg(nil) + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func TestRandomizedEncodingBoundedArgs(t *testing.T) { + protocol.RunEncodingTest(t, &BoundedArgs{}) +} + +func BenchmarkMarshalMsgBoundedArgs(b *testing.B) { + v := BoundedArgs{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBoundedArgs(b *testing.B) { + v := BoundedArgs{} + bts := make([]byte, 0, v.Msgsize()) + bts = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBoundedArgs(b *testing.B) { + v := BoundedArgs{} + bts := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalTotalBoundedArgs(t *testing.T) { + partitiontest.PartitionTest(t) + v := TotalBoundedArgs{} + bts := v.MarshalMsg(nil) + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func TestRandomizedEncodingTotalBoundedArgs(t *testing.T) { + protocol.RunEncodingTest(t, &TotalBoundedArgs{}) +} + +func BenchmarkMarshalMsgTotalBoundedArgs(b *testing.B) { + v := TotalBoundedArgs{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgTotalBoundedArgs(b *testing.B) { + v := TotalBoundedArgs{} + bts := make([]byte, 0, v.Msgsize()) + bts = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalTotalBoundedArgs(b *testing.B) { + v := TotalBoundedArgs{} + bts := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/tests/maxtotalbytes_test.go b/tests/maxtotalbytes_test.go new file mode 100644 index 0000000..0005095 --- /dev/null +++ b/tests/maxtotalbytes_test.go @@ -0,0 +1,56 @@ +package tests + +import ( + "testing" + + "github.com/algorand/msgp/msgp" +) + +func TestMaxTotalBytesDirective(t *testing.T) { + // Without the directive, MaxSize assumes every slot is full and maximal. + product := msgp.ArrayHeaderSize + argCount*(msgp.BytesPrefixSize+argMaxSize) + if got := BoundedArgsMaxSize(); got != product { + t.Fatalf("BoundedArgsMaxSize() = %d, want %d", got, product) + } + + // With it, MaxSize is exactly the directive's expression plus the header. + total := msgp.ArrayHeaderSize + argCount*msgp.BytesPrefixSize + argsTotal + if got := TotalBoundedArgsMaxSize(); got != total { + t.Fatalf("TotalBoundedArgsMaxSize() = %d, want %d", got, total) + } + if total >= product { + t.Fatalf("test constants should make the total bound (%d) tighter than the product (%d)", total, product) + } + + // A struct holding the type picks up the same tight bound, plus a + // one-byte map header and the two-byte key "a". + holder := 1 + 2 + total + if got := ArgsHolderMaxSize(); got != holder { + t.Fatalf("ArgsHolderMaxSize() = %d, want %d", got, holder) + } +} + +func TestMaxTotalBytesDirectiveIsBound(t *testing.T) { + // The most bytes a value obeying argCount and argsTotal can encode to: + // every slot used, with the total spread so each element needs a + // multi-byte bin header. + var args TotalBoundedArgs + for i := 0; i < argCount; i++ { + args = append(args, make([]byte, argsTotal/argCount)) + } + if n := len(args.MarshalMsg(nil)); n > TotalBoundedArgsMaxSize() { + t.Fatalf("encoded %d bytes, more than TotalBoundedArgsMaxSize() = %d", n, TotalBoundedArgsMaxSize()) + } + h := ArgsHolder{Args: args} + if n := len(h.MarshalMsg(nil)); n > ArgsHolderMaxSize() { + t.Fatalf("encoded %d bytes, more than ArgsHolderMaxSize() = %d", n, ArgsHolderMaxSize()) + } + + // The directive only affects MaxSize; decoding still enforces the + // per-element allocbound. + over := TotalBoundedArgs{make([]byte, argMaxSize+1)} + var out TotalBoundedArgs + if _, err := out.UnmarshalMsg(over.MarshalMsg(nil)); err == nil { + t.Fatalf("decoded an element longer than argMaxSize") + } +}