add zstd

common/go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/Masterminds/semver/v3 v3.2.1
 	github.com/bits-and-blooms/bitset v1.12.0
 	github.com/cloudflare/cfssl v0.0.0-20180223231731-4e2dcbde5004
+	github.com/colinlyguo/zstd v0.0.0-20240422161758-2c9809420237
 	github.com/docker/docker v25.0.3+incompatible
 	github.com/gin-contrib/pprof v1.4.0
 	github.com/gin-gonic/gin v1.9.1

common/go.sum

@@ -117,6 +117,10 @@ github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+g
 github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE=
 github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4=
+github.com/colinlyguo/zstd v0.0.0-20240422153245-68e20236939c h1:k/55pwq7Co77RfFHQpdI7cOghzS9rr1f4WWmYOeuDwU=
+github.com/colinlyguo/zstd v0.0.0-20240422153245-68e20236939c/go.mod h1:ok1MRsMAuuMr6NJAbRQDLSWWK2CvXgc7WZnOFTAkhhU=
+github.com/colinlyguo/zstd v0.0.0-20240422161758-2c9809420237 h1:CrzIvxsH1WIrWC48fkbDcIGTnz3zrCEIVNM7IjZWeTk=
+github.com/colinlyguo/zstd v0.0.0-20240422161758-2c9809420237/go.mod h1:ok1MRsMAuuMr6NJAbRQDLSWWK2CvXgc7WZnOFTAkhhU=
 github.com/compose-spec/compose-go/v2 v2.0.0-rc.2 h1:eJ01FpliL/02KvsaPyH1bSLbM1S70yWQUojHVRbyvy4=
 github.com/compose-spec/compose-go/v2 v2.0.0-rc.2/go.mod h1:IVsvFyGVhw4FASzUtlWNVaAOhYmakXAFY9IlZ7LAuD8=
 github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ=

common/types/encoding/codecv1/codecv1.go

@@ -10,6 +10,7 @@ import (
 	"math/big"
 	"strings"
 
+	"github.com/colinlyguo/zstd"
 	"github.com/scroll-tech/go-ethereum/accounts/abi"
 	"github.com/scroll-tech/go-ethereum/common"
 	"github.com/scroll-tech/go-ethereum/core/types"
@@ -204,7 +205,7 @@ func (c *DAChunk) Hash() (common.Hash, error) {
 }
 
 // NewDABatch creates a DABatch from the provided encoding.Batch.
-func NewDABatch(batch *encoding.Batch) (*DABatch, error) {
+func NewDABatch(batch *encoding.Batch, withCompression bool) (*DABatch, error) {
 	// this encoding can only support a fixed number of chunks per batch
 	if len(batch.Chunks) > MaxNumChunks {
 		return nil, fmt.Errorf("too many chunks in batch")
@@ -227,7 +228,7 @@ func NewDABatch(batch *encoding.Batch) (*DABatch, error) {
 	}
 
 	// blob payload
-	blob, blobVersionedHash, z, err := constructBlobPayload(batch.Chunks)
+	blob, blobVersionedHash, z, err := constructBlobPayload(batch.Chunks, withCompression)
 	if err != nil {
 		return nil, err
 	}
@@ -274,12 +275,12 @@ func computeBatchDataHash(chunks []*encoding.Chunk, totalL1MessagePoppedBefore u
 }
 
 // constructBlobPayload constructs the 4844 blob payload.
-func constructBlobPayload(chunks []*encoding.Chunk) (*kzg4844.Blob, common.Hash, *kzg4844.Point, error) {
+func constructBlobPayload(chunks []*encoding.Chunk, withCompression bool) (*kzg4844.Blob, common.Hash, *kzg4844.Point, error) {
 	// metadata consists of num_chunks (2 bytes) and chunki_size (4 bytes per chunk)
 	metadataLength := 2 + MaxNumChunks*4
 
 	// the raw (un-padded) blob payload
-	blobBytes := make([]byte, metadataLength)
+	batchBytes := make([]byte, metadataLength)
 
 	// challenge digest preimage
 	// 1 hash for metadata, 1 hash for each chunk, 1 hash for blob versioned hash
@@ -289,12 +290,12 @@ func constructBlobPayload(chunks []*encoding.Chunk) (*kzg4844.Blob, common.Hash,
 	var chunkDataHash common.Hash
 
 	// blob metadata: num_chunks
-	binary.BigEndian.PutUint16(blobBytes[0:], uint16(len(chunks)))
+	binary.BigEndian.PutUint16(batchBytes[0:], uint16(len(chunks)))
 
 	// encode blob metadata and L2 transactions,
 	// and simultaneously also build challenge preimage
 	for chunkID, chunk := range chunks {
-		currentChunkStartIndex := len(blobBytes)
+		currentChunkStartIndex := len(batchBytes)
 
 		for _, block := range chunk.Blocks {
 			for _, tx := range block.Transactions {
@@ -304,18 +305,18 @@ func constructBlobPayload(chunks []*encoding.Chunk) (*kzg4844.Blob, common.Hash,
 					if err != nil {
 						return nil, common.Hash{}, nil, err
 					}
-					blobBytes = append(blobBytes, rlpTxData...)
+					batchBytes = append(batchBytes, rlpTxData...)
 				}
 			}
 		}
 
 		// blob metadata: chunki_size
-		if chunkSize := len(blobBytes) - currentChunkStartIndex; chunkSize != 0 {
-			binary.BigEndian.PutUint32(blobBytes[2+4*chunkID:], uint32(chunkSize))
+		if chunkSize := len(batchBytes) - currentChunkStartIndex; chunkSize != 0 {
+			binary.BigEndian.PutUint32(batchBytes[2+4*chunkID:], uint32(chunkSize))
 		}
 
 		// challenge: compute chunk data hash
-		chunkDataHash = crypto.Keccak256Hash(blobBytes[currentChunkStartIndex:])
+		chunkDataHash = crypto.Keccak256Hash(batchBytes[currentChunkStartIndex:])
 		copy(challengePreimage[32+chunkID*32:], chunkDataHash[:])
 	}
 
@@ -328,9 +329,18 @@ func constructBlobPayload(chunks []*encoding.Chunk) (*kzg4844.Blob, common.Hash,
 	}
 
 	// challenge: compute metadata hash
-	hash := crypto.Keccak256Hash(blobBytes[0:metadataLength])
+	hash := crypto.Keccak256Hash(batchBytes[0:metadataLength])
 	copy(challengePreimage[0:], hash[:])
 
+	blobBytes := batchBytes
+	if withCompression {
+		var err error
+		blobBytes, err = zstd.CompressScrollBatchBytes(batchBytes)
+		if err != nil {
+			return nil, common.Hash{}, nil, err
+		}
+	}
+
 	// convert raw data to BLSFieldElements
 	blob, err := makeBlobCanonical(blobBytes)
 	if err != nil {
@@ -363,7 +373,7 @@ func constructBlobPayload(chunks []*encoding.Chunk) (*kzg4844.Blob, common.Hash,
 // makeBlobCanonical converts the raw blob data into the canonical blob representation of 4096 BLSFieldElements.
 func makeBlobCanonical(blobBytes []byte) (*kzg4844.Blob, error) {
 	// blob contains 131072 bytes but we can only utilize 31/32 of these
-	if len(blobBytes) > 126976 {
+	if len(blobBytes) > 124*1024 {
 		return nil, fmt.Errorf("oversized batch payload")
 	}
 

rollup/internal/controller/relayer/l2_relayer.go

@@ -766,7 +766,7 @@ func (r *Layer2Relayer) constructCommitBatchPayloadCodecV1(dbBatch *orm.Batch, d
 		Chunks:                     chunks,
 	}
 
-	daBatch, createErr := codecv1.NewDABatch(batch)
+	daBatch, createErr := codecv1.NewDABatch(batch, false)
 	if createErr != nil {
 		return nil, nil, fmt.Errorf("failed to create DA batch: %w", createErr)
 	}
@@ -825,7 +825,7 @@ func (r *Layer2Relayer) constructFinalizeBatchPayloadCodecV1(dbBatch *orm.Batch,
 		Chunks:                     chunks,
 	}
 
-	daBatch, createErr := codecv1.NewDABatch(batch)
+	daBatch, createErr := codecv1.NewDABatch(batch, false)
 	if createErr != nil {
 		return nil, fmt.Errorf("failed to create DA batch: %w", createErr)
 	}

rollup/internal/utils/utils.go

@@ -242,7 +242,7 @@ func GetBatchMetadata(batch *encoding.Batch, codecVersion encoding.CodecVersion)
 		}
 		return batchMeta, nil
 	case encoding.CodecV1:
-		daBatch, err := codecv1.NewDABatch(batch)
+		daBatch, err := codecv1.NewDABatch(batch, false)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create codecv1 DA batch: %w", err)
 		}