Replace fastssz with dynamic-ssz

go.mod

@@ -32,6 +32,7 @@ require (
 	github.com/klauspost/compress v1.17.9
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
+	github.com/pk910/dynamic-ssz v1.3.2
 	github.com/princjef/gomarkdoc v0.4.1
 	github.com/prometheus/client_golang v1.20.0
 	github.com/prysmaticlabs/go-bitfield v0.0.0-20240328144219-a1caa50c3a1e
@@ -64,6 +65,7 @@ require (
 	github.com/bits-and-blooms/bitset v1.20.0 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect
 	github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 // indirect
+	github.com/casbin/govaluate v1.10.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cheggaaa/pb/v3 v3.0.8 // indirect
 	github.com/cloudflare/circl v1.6.3 // indirect
@@ -146,6 +148,7 @@ require (
 	github.com/opencontainers/image-spec v1.0.2 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pjbgf/sha1cd v0.6.0 // indirect
+	github.com/pk910/hashtree-bindings v0.2.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
@@ -188,9 +191,11 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/crypto v0.51.0 // indirect
+	golang.org/x/mod v0.35.0 // indirect
 	golang.org/x/net v0.55.0 // indirect
 	golang.org/x/oauth2 v0.35.0 // indirect
 	golang.org/x/sys v0.45.0 // indirect
+	golang.org/x/tools v0.44.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
 	google.golang.org/grpc v1.80.0 // indirect
@@ -202,3 +207,5 @@ require (
 )
 
 replace github.com/wealdtech/go-merkletree v1.0.1-0.20190605192610-2bb163c2ea2a => github.com/rocket-pool/go-merkletree v1.0.1-0.20220406020931-c262d9b976dd
+
+tool github.com/pk910/dynamic-ssz/dynssz-gen

go.sum

@@ -113,6 +113,8 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku
 github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
 github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
+github.com/casbin/govaluate v1.10.0 h1:ffGw51/hYH3w3rZcxO/KcaUIDOLP84w7nsidMVgaDG0=
+github.com/casbin/govaluate v1.10.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A=
 github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
 github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
 github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
@@ -783,6 +785,10 @@ github.com/pion/webrtc/v3 v3.3.0 h1:Rf4u6n6U5t5sUxhYPQk/samzU/oDv7jk6BA5hyO2F9I=
 github.com/pion/webrtc/v3 v3.3.0/go.mod h1:hVmrDJvwhEertRWObeb1xzulzHGeVUoPlWvxdGzcfU0=
 github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU=
 github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM=
+github.com/pk910/dynamic-ssz v1.3.2 h1:65UR/O+ss+U2Dn86Rdl7LwehHo3u2ElutduS/pcuUXE=
+github.com/pk910/dynamic-ssz v1.3.2/go.mod h1:lqmnou2bjr2UWQ3C/L3082TGW0SFl/SwT7ionwM0+FU=
+github.com/pk910/hashtree-bindings v0.2.2 h1:gkczxxekBW2NeMK9N3OLj7Jepe7zPmJGVwr8LyofGsA=
+github.com/pk910/hashtree-bindings v0.2.2/go.mod h1:zrWt88783JmhBfcgni6kkIMYRdXTZi/FL//OyI5T/l4=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

shared/services/beacon/client.go

@@ -156,15 +156,18 @@ const (
 
 // SSZ response go into these wrapper types.
 // You can stream Data into a deserializer.
+// Size is the total SSZ payload size from Content-Length; <= 0 if unknown.
 // Fork is the consensus version, e.g, "deneb" or "electra"
 
 type BeaconStateSSZ struct {
 	Data io.ReadCloser
+	Size int64
 	Fork string
 }
 
 type BeaconBlockSSZ struct {
 	Data io.ReadCloser
+	Size int64
 	Fork string
 }
 

shared/services/beacon/client/std-http-client.go

@@ -1028,6 +1028,7 @@ func (c *StandardHttpClient) GetBeaconStateSSZ(slot uint64) (*beacon.BeaconState
 
 	return &beacon.BeaconStateSSZ{
 		Data: response.Body,
+		Size: response.ContentLength,
 		Fork: response.Header.Get(ResponseConsensusVersionHeader),
 	}, nil
 }
@@ -1050,6 +1051,7 @@ func (c *StandardHttpClient) GetBeaconBlockSSZ(slot uint64) (*beacon.BeaconBlock
 
 	return &beacon.BeaconBlockSSZ{
 		Data: response.Body,
+		Size: response.ContentLength,
 		Fork: response.Header.Get(ResponseConsensusVersionHeader),
 	}, true, nil
 }
@@ -1213,6 +1215,10 @@ func (c *StandardHttpClient) sszRequest(requestPath string) (*http.Response, err
 		return nil, err
 	}
 	request.Header.Set("Accept", RequestSSZContentType)
+	// Disable transparent compression: a compressed response has no
+	// Content-Length, which would force the buffered (non-streaming) SSZ
+	// decode path
+	request.Header.Set("Accept-Encoding", "identity")
 	return http.DefaultClient.Do(request)
 }
 

shared/services/megapools.go

@@ -14,7 +14,7 @@ import (
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
-	ssz "github.com/ferranbt/fastssz"
+	"github.com/pk910/dynamic-ssz/treeproof"
 	"github.com/urfave/cli/v3"
 	"golang.org/x/sync/errgroup"
 
@@ -165,7 +165,7 @@ func GetWithdrawableEpochProof(c *cli.Command, wallet *wallet.Wallet, eth2Config
 		return api.ValidatorWithdrawableEpochProof{}, err
 	}
 
-	beaconState, err := eth2.NewBeaconState(beaconStateResponse.Data, beaconStateResponse.Fork)
+	beaconState, err := eth2.NewBeaconState(beaconStateResponse.Data, beaconStateResponse.Size, beaconStateResponse.Fork)
 	if err != nil {
 		return api.ValidatorWithdrawableEpochProof{}, err
 	}
@@ -722,7 +722,7 @@ func GetWithdrawalProofForSlot(c *cli.Command, slot uint64, validatorIndex uint6
 		return megapool.FinalBalanceProof{}, 0, nil, err
 	}
 
-	beaconState, err := eth2.NewBeaconState(stateResponse.Data, stateResponse.Fork)
+	beaconState, err := eth2.NewBeaconState(stateResponse.Data, stateResponse.Size, stateResponse.Fork)
 	if err != nil {
 		return megapool.FinalBalanceProof{}, 0, nil, err
 	}
@@ -768,7 +768,7 @@ func GetWithdrawalProofForSlot(c *cli.Command, slot uint64, validatorIndex uint6
 		if err != nil {
 			return megapool.FinalBalanceProof{}, 0, nil, err
 		}
-		blockRootsState, err := eth2.NewBeaconState(blockRootsStateResponse.Data, blockRootsStateResponse.Fork)
+		blockRootsState, err := eth2.NewBeaconState(blockRootsStateResponse.Data, blockRootsStateResponse.Size, blockRootsStateResponse.Fork)
 		if err != nil {
 			return megapool.FinalBalanceProof{}, 0, nil, err
 		}
@@ -778,8 +778,8 @@ func GetWithdrawalProofForSlot(c *cli.Command, slot uint64, validatorIndex uint6
 		}
 
 		// Get historical summary proof
-		var tree *ssz.Node
-		tree, err = fuluState.GetTree()
+		var tree *treeproof.Node
+		tree, err = generic.SSZ.GetTree(fuluState)
 		if err != nil {
 			return megapool.FinalBalanceProof{}, 0, nil, fmt.Errorf("could not get state tree: %w", err)
 		}
@@ -863,7 +863,7 @@ func FindWithdrawalBlockAndArrayPosition(slot uint64, validatorIndex uint64, bc
 		}
 		notFounds = 0
 
-		beaconBlock, err := eth2.NewSignedBeaconBlock(blockResponse.Data, blockResponse.Fork)
+		beaconBlock, err := eth2.NewSignedBeaconBlock(blockResponse.Data, blockResponse.Size, blockResponse.Fork)
 		if err != nil {
 			return 0, nil, 0, nil, nil, err
 		}

shared/services/services.go

@@ -231,7 +231,7 @@ func GetBeaconState(bc beacon.Client) (eth2.BeaconState, error) {
 		return nil, err
 	}
 
-	beaconState, err := eth2.NewBeaconState(beaconStateResponse.Data, beaconStateResponse.Fork)
+	beaconState, err := eth2.NewBeaconState(beaconStateResponse.Data, beaconStateResponse.Size, beaconStateResponse.Fork)
 	if err != nil {
 		return nil, err
 	}

shared/types/eth2/fork/deneb/block_deneb.go

@@ -6,7 +6,7 @@ import "github.com/rocket-pool/smartnode/shared/types/eth2/generic"
 const BeaconBlockBodyChunksCeil uint64 = 16
 
 func (b *SignedBeaconBlock) ProveWithdrawal(indexInWithdrawalsArray uint64) ([][]byte, error) {
-	tree, err := b.Block.GetTree()
+	tree, err := generic.SSZ.GetTree(b.Block)
 	if err != nil {
 		return nil, err
 	}