generic ssz list encoding

Author: kasey

beacon-chain/p2p/types/BUILD.bazel

@@ -25,6 +25,7 @@ go_library(
         "//consensus-types/primitives:go_default_library",
         "//consensus-types/wrapper:go_default_library",
         "//encoding/bytesutil:go_default_library",
+        "//encoding/ssz:go_default_library",
         "//proto/engine/v1:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//proto/prysm/v1alpha1/metadata:go_default_library",
@@ -45,10 +46,10 @@ go_test(
         "//config/params:go_default_library",
         "//consensus-types/primitives:go_default_library",
         "//encoding/bytesutil:go_default_library",
+        "//encoding/ssz:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//runtime/version:go_default_library",
         "//testing/assert:go_default_library",
         "//testing/require:go_default_library",
-        "@com_github_prysmaticlabs_fastssz//:go_default_library",
     ],
 )

beacon-chain/p2p/types/types.go

@@ -5,14 +5,14 @@ package types
 
 import (
 	"bytes"
-	"encoding/binary"
 	"sort"
 
 	fieldparams "github.com/OffchainLabs/prysm/v6/config/fieldparams"
 	"github.com/OffchainLabs/prysm/v6/config/params"
+	"github.com/OffchainLabs/prysm/v6/encoding/ssz"
 	eth "github.com/OffchainLabs/prysm/v6/proto/prysm/v1alpha1"
 	"github.com/pkg/errors"
-	ssz "github.com/prysmaticlabs/fastssz"
+	fastssz "github.com/prysmaticlabs/fastssz"
 )
 
 const (
@@ -25,11 +25,11 @@ type SSZBytes []byte
 
 // HashTreeRoot hashes the uint64 object following the SSZ standard.
 func (b *SSZBytes) HashTreeRoot() ([32]byte, error) {
-	return ssz.HashWithDefaultHasher(b)
+	return fastssz.HashWithDefaultHasher(b)
 }
 
 // HashTreeRootWith hashes the uint64 object with the given hasher.
-func (b *SSZBytes) HashTreeRootWith(hh *ssz.Hasher) error {
+func (b *SSZBytes) HashTreeRootWith(hh *fastssz.Hasher) error {
 	indx := hh.Index()
 	hh.PutBytes(*b)
 	hh.Merkleize(indx)
@@ -74,7 +74,7 @@ func (r *BeaconBlockByRootsReq) UnmarshalSSZ(buf []byte) error {
 		return errors.Errorf("expected buffer with length of up to %d but received length %d", maxLength, bufLen)
 	}
 	if bufLen%fieldparams.RootLength != 0 {
-		return ssz.ErrIncorrectByteSize
+		return fastssz.ErrIncorrectByteSize
 	}
 	numOfRoots := bufLen / fieldparams.RootLength
 	roots := make([][fieldparams.RootLength]byte, 0, numOfRoots)
@@ -131,14 +131,6 @@ func (m *ErrorMessage) UnmarshalSSZ(buf []byte) error {
 // BlobSidecarsByRootReq is used to specify a list of blob targets (root+index) in a BlobSidecarsByRoot RPC request.
 type BlobSidecarsByRootReq []*eth.BlobIdentifier
 
-// BlobIdentifier is a fixed size value, so we can compute its fixed size at start time (see init below)
-var blobIdSize int
-
-// SizeSSZ returns the size of the serialized representation.
-func (b *BlobSidecarsByRootReq) SizeSSZ() int {
-	return len(*b) * blobIdSize
-}
-
 // MarshalSSZTo appends the serialized BlobSidecarsByRootReq value to the provided byte slice.
 func (b *BlobSidecarsByRootReq) MarshalSSZTo(dst []byte) ([]byte, error) {
 	// A List without an enclosing container is marshaled exactly like a vector, no length offset required.
@@ -151,38 +143,22 @@ func (b *BlobSidecarsByRootReq) MarshalSSZTo(dst []byte) ([]byte, error) {
 
 // MarshalSSZ serializes the BlobSidecarsByRootReq value to a byte slice.
 func (b *BlobSidecarsByRootReq) MarshalSSZ() ([]byte, error) {
-	buf := make([]byte, len(*b)*blobIdSize)
-	for i, id := range *b {
-		by, err := id.MarshalSSZ()
-		if err != nil {
-			return nil, err
-		}
-		copy(buf[i*blobIdSize:(i+1)*blobIdSize], by)
-	}
-	return buf, nil
+	return ssz.MarshalListFixedElement[*eth.BlobIdentifier](*b)
 }
 
+func newBSBR() *eth.BlobIdentifier { return &eth.BlobIdentifier{} }
+
 // UnmarshalSSZ unmarshals the provided bytes buffer into the
 // BlobSidecarsByRootReq value.
 func (b *BlobSidecarsByRootReq) UnmarshalSSZ(buf []byte) error {
-	bufLen := len(buf)
-	maxLength := int(params.BeaconConfig().MaxRequestBlobSidecarsElectra) * blobIdSize
-	if bufLen > maxLength {
-		return errors.Wrapf(ssz.ErrIncorrectListSize, "expected buffer with length of up to %d but received length %d", maxLength, bufLen)
-	}
-	if bufLen%blobIdSize != 0 {
-		return errors.Wrapf(ssz.ErrIncorrectByteSize, "size=%d", bufLen)
+	v, err := ssz.UnmarshalListFixedElement[*eth.BlobIdentifier](buf, newBSBR)
+	if err != nil {
+		return errors.Wrapf(err, "failed to unmarshal BlobSidecarsByRootReq")
 	}
-	count := bufLen / blobIdSize
-	*b = make([]*eth.BlobIdentifier, count)
-	for i := 0; i < count; i++ {
-		id := &eth.BlobIdentifier{}
-		err := id.UnmarshalSSZ(buf[i*blobIdSize : (i+1)*blobIdSize])
-		if err != nil {
-			return err
-		}
-		(*b)[i] = id
+	if len(v) > int(params.BeaconConfig().MaxRequestBlobSidecarsElectra) {
+		return ErrMaxBlobReqExceeded
 	}
+	*b = v
 	return nil
 }
 
@@ -213,102 +189,25 @@ func (s BlobSidecarsByRootReq) Len() int {
 // ====================================
 // DataColumnsByRootIdentifiers section
 // ====================================
-var _ ssz.Marshaler = DataColumnsByRootIdentifiers{}
-var _ ssz.Unmarshaler = (*DataColumnsByRootIdentifiers)(nil)
+var _ fastssz.Marshaler = &DataColumnsByRootIdentifiers{}
+var _ fastssz.Unmarshaler = &DataColumnsByRootIdentifiers{}
 
 // DataColumnsByRootIdentifiers is used to specify a list of data column targets (root+index) in a DataColumnSidecarsByRoot RPC request.
 type DataColumnsByRootIdentifiers []*eth.DataColumnsByRootIdentifier
 
-// DataColumnIdentifier is a fixed size value, so we can compute its fixed size at start time (see init below)
-var dataColumnIdSize int
+func newDCRI() *eth.DataColumnsByRootIdentifier { return &eth.DataColumnsByRootIdentifier{} }
 
-// UnmarshalSSZ implements ssz.Unmarshaler. It unmarshals the provided bytes buffer into the DataColumnSidecarsByRootReq value.
 func (d *DataColumnsByRootIdentifiers) UnmarshalSSZ(buf []byte) error {
-	// Exit early if the buffer is too small.
-	if len(buf) < bytesPerLengthOffset {
-		return nil
-	}
-
-	// Get the size of the offsets.
-	offsetEnd := binary.LittleEndian.Uint32(buf[:bytesPerLengthOffset])
-	if offsetEnd%bytesPerLengthOffset != 0 {
-		return errors.Errorf("expected offsets size to be a multiple of %d but got %d", bytesPerLengthOffset, offsetEnd)
-	}
-
-	count := offsetEnd / bytesPerLengthOffset
-	if count < 1 {
-		return nil
-	}
-
-	maxSize := params.BeaconConfig().MaxRequestBlocksDeneb
-	if uint64(count) > maxSize {
-		return errors.Errorf("data column identifiers list exceeds max size: %d > %d", count, maxSize)
-	}
-
-	if offsetEnd > uint32(len(buf)) {
-		return errors.Errorf("offsets value %d larger than buffer %d", offsetEnd, len(buf))
-	}
-	valueStart := offsetEnd
-
-	// Decode the identifers.
-	*d = make([]*eth.DataColumnsByRootIdentifier, count)
-	var start uint32
-	end := uint32(len(buf))
-	for i := count; i > 0; i-- {
-		offsetEnd -= bytesPerLengthOffset
-		start = binary.LittleEndian.Uint32(buf[offsetEnd : offsetEnd+bytesPerLengthOffset])
-		if start > end {
-			return errors.Errorf("expected offset[%d] %d to be less than %d", i-1, start, end)
-		}
-		if start < valueStart {
-			return errors.Errorf("offset[%d] %d indexes before value section %d", i-1, start, valueStart)
-		}
-		// Decode the identifier.
-		ident := &eth.DataColumnsByRootIdentifier{}
-		if err := ident.UnmarshalSSZ(buf[start:end]); err != nil {
-			return err
-		}
-		(*d)[i-1] = ident
-		end = start
+	v, err := ssz.UnmarshalListVariableElement[*eth.DataColumnsByRootIdentifier](buf, newDCRI)
+	if err != nil {
+		return errors.Wrapf(err, "failed to unmarshal DataColumnsByRootIdentifiers")
 	}
-
+	*d = v
 	return nil
 }
 
-func (d DataColumnsByRootIdentifiers) MarshalSSZ() ([]byte, error) {
-	var err error
-	count := len(d)
-	maxSize := params.BeaconConfig().MaxRequestBlocksDeneb
-	if uint64(count) > maxSize {
-		return nil, errors.Errorf("data column identifiers list exceeds max size: %d > %d", count, maxSize)
-	}
-
-	if len(d) == 0 {
-		return []byte{}, nil
-	}
-	sizes := make([]uint32, count)
-	valTotal := uint32(0)
-	for i, elem := range d {
-		if elem == nil {
-			return nil, errors.New("nil item in DataColumnsByRootIdentifiers list")
-		}
-		sizes[i] = uint32(elem.SizeSSZ())
-		valTotal += sizes[i]
-	}
-	offSize := uint32(4 * len(d))
-	out := make([]byte, offSize, offSize+valTotal)
-	for i := range sizes {
-		binary.LittleEndian.PutUint32(out[i*4:i*4+4], offSize)
-		offSize += sizes[i]
-	}
-	for _, elem := range d {
-		out, err = elem.MarshalSSZTo(out)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return out, nil
+func (d *DataColumnsByRootIdentifiers) MarshalSSZ() ([]byte, error) {
+	return ssz.MarshalListVariableElement[*eth.DataColumnsByRootIdentifier](*d)
 }
 
 // MarshalSSZTo implements ssz.Marshaler. It appends the serialized DataColumnSidecarsByRootReq value to the provided byte slice.
@@ -329,11 +228,3 @@ func (d DataColumnsByRootIdentifiers) SizeSSZ() int {
 	}
 	return size
 }
-
-func init() {
-	blobSizer := &eth.BlobIdentifier{}
-	blobIdSize = blobSizer.SizeSSZ()
-
-	dataColumnSizer := &eth.DataColumnSidecarsByRangeRequest{}
-	dataColumnIdSize = dataColumnSizer.SizeSSZ()
-}

beacon-chain/p2p/types/types_test.go

@@ -8,10 +8,10 @@ import (
 	"github.com/OffchainLabs/prysm/v6/config/params"
 	"github.com/OffchainLabs/prysm/v6/consensus-types/primitives"
 	"github.com/OffchainLabs/prysm/v6/encoding/bytesutil"
+	"github.com/OffchainLabs/prysm/v6/encoding/ssz"
 	eth "github.com/OffchainLabs/prysm/v6/proto/prysm/v1alpha1"
 	"github.com/OffchainLabs/prysm/v6/testing/assert"
 	"github.com/OffchainLabs/prysm/v6/testing/require"
-	ssz "github.com/prysmaticlabs/fastssz"
 )
 
 func generateBlobIdentifiers(n int) []*eth.BlobIdentifier {
@@ -51,7 +51,7 @@ func TestBlobSidecarsByRootReq_MarshalSSZ(t *testing.T) {
 		{
 			name:         "beyond max list",
 			ids:          generateBlobIdentifiers(int(params.BeaconConfig().MaxRequestBlobSidecarsElectra) + 1),
-			unmarshalErr: ssz.ErrIncorrectListSize,
+			unmarshalErr: ErrMaxBlobReqExceeded,
 		},
 		{
 			name: "wonky unmarshal size",
@@ -60,7 +60,7 @@ func TestBlobSidecarsByRootReq_MarshalSSZ(t *testing.T) {
 				in = append(in, byte(0))
 				return in
 			},
-			unmarshalErr: ssz.ErrIncorrectByteSize,
+			unmarshalErr: ssz.ErrInvalidFixedEncodingLen,
 		},
 	}
 
@@ -305,7 +305,8 @@ func TestDataColumnSidecarsByRootReq_MarshalUnmarshal(t *testing.T) {
 			name: "size too big",
 			ids:  generateDataColumnIdentifiers(1),
 			unmarshalMod: func(in []byte) []byte {
-				maxLen := params.BeaconConfig().MaxRequestDataColumnSidecars * uint64(dataColumnIdSize)
+				sizer := &eth.DataColumnSidecarsByRangeRequest{}
+				maxLen := params.BeaconConfig().MaxRequestDataColumnSidecars * uint64(sizer.SizeSSZ())
 				add := make([]byte, maxLen)
 				in = append(in, add...)
 				return in

changelog/kasey_generic-list-serdes.md

@@ -0,0 +1,2 @@
+## Added
+- Methods to generically encode/decode independent lists of ssz values.

encoding/ssz/BUILD.bazel

@@ -3,9 +3,11 @@ load("@prysm//tools/go:def.bzl", "go_library", "go_test")
 go_library(
     name = "go_default_library",
     srcs = [
+        "errors.go",
         "hashers.go",
         "helpers.go",
         "htrutils.go",
+        "list.go",
         "merkleize.go",
         "slice_root.go",
     ],

encoding/ssz/errors.go

@@ -0,0 +1,17 @@
+package ssz
+
+import "github.com/pkg/errors"
+
+var (
+	ErrInvalidEncodingLength     = errors.New("invalid encoded length")
+	ErrInvalidFixedEncodingLen   = errors.Wrap(ErrInvalidEncodingLength, "not multiple of fixed size")
+	ErrEncodingSmallerThanOffset = errors.Wrap(ErrInvalidEncodingLength, "smaller than a single offset")
+	ErrInvalidOffset             = errors.New("invalid offset")
+	ErrOffsetIntoFixed           = errors.Wrap(ErrInvalidOffset, "does not point past fixed section of encoding")
+	ErrOffsetExceedsBuffer       = errors.Wrap(ErrInvalidOffset, "exceeds buffer length")
+	ErrNegativeRelativeOffset    = errors.Wrap(ErrInvalidOffset, "less than previous offset")
+	ErrOffsetInsufficient        = errors.Wrap(ErrInvalidOffset, "insufficient difference relative to previous")
+	ErrOffsetSectionMisaligned   = errors.Wrap(ErrInvalidOffset, "offset bytes are not a multiple of offset size")
+
+	ErrOffsetDecodedMismatch = errors.New("unmarshaled size does not relative offsets")
+)