Add Header to Hotshot Payload

arbutil/espresso_utils.go

@@ -4,6 +4,8 @@ import (
 	"bytes"
 	"encoding/binary"
 	"errors"
+	"fmt"
+	"math"
 	"time"
 
 	espressoTypes "github.com/EspressoSystems/espresso-network/sdks/go/types"
@@ -16,6 +18,8 @@ import (
 const MAX_ATTESTATION_QUOTE_SIZE int = 4 * 1024
 const LEN_SIZE int = 8
 const INDEX_SIZE int = 8
+const HEADER_SIZE = 1
+const HEADER_LEN = 4
 
 type SubmittedEspressoTx struct {
 	Hash        string
@@ -24,6 +28,29 @@ type SubmittedEspressoTx struct {
 	SubmittedAt time.Time `rlp:"optional"`
 }
 
+type Header struct {
+	// Version for header in case any of these fields change we can parse correctly based on version
+	Version TransactionVersion
+	// Transaction how the header is formatted
+	TransactionType TransactionType
+	// Reserved unused bytes, also helps with header verification
+	Reserved uint16
+}
+
+type TransactionType uint8
+
+const (
+	Legacy       TransactionType = 0
+	EphemeralKey TransactionType = 1
+	Timeboost    TransactionType = 2
+)
+
+type TransactionVersion uint8
+
+const (
+	V0 TransactionVersion = 0
+)
+
 func BuildRawHotShotPayload(
 	msgPositions []MessageIndex,
 	msgFetcher func(MessageIndex) ([]byte, error),
@@ -67,10 +94,32 @@ func SignHotShotPayload(
 		return nil, err
 	}
 
+	header := Header{
+		Version:         V0,
+		TransactionType: EphemeralKey,
+		Reserved:        0,
+	}
+
+	encoded := []byte{
+		uint8(header.Version),
+		uint8(header.TransactionType),
+		byte(header.Reserved >> 8),
+		byte(header.Reserved),
+	}
+
+	headerBuf := make([]byte, HEADER_SIZE)
+	size := len(encoded)
+	if size > math.MaxUint8 {
+		return nil, fmt.Errorf("encoded data too large: %d bytes (max %d)", len(encoded), math.MaxUint8)
+	}
+	headerBuf[0] = uint8(size)
+	result := headerBuf
+	result = append(result, encoded...)
+
 	quoteSizeBuf := make([]byte, LEN_SIZE)
 	binary.BigEndian.PutUint64(quoteSizeBuf, uint64(len(quote)))
 	// Put the signature first. That would help easier parsing.
-	result := quoteSizeBuf
+	result = append(result, quoteSizeBuf...)
 	result = append(result, quote...)
 	result = append(result, unsigned...)
 
@@ -88,7 +137,44 @@ func ValidateIfPayloadIsInBlock(p []byte, payloads []espressoTypes.Bytes) bool {
 	return validated
 }
 
-func ParseHotShotPayload(payload []byte) (signature []byte, userDataHash []byte, indices []uint64, messages [][]byte, err error) {
+func ParseHotshotPayloadForHeader(tx []byte) *TransactionType {
+	if len(tx) < HEADER_SIZE+HEADER_LEN {
+		log.Warn("hotshot transaction is too small for a header")
+		return nil
+	}
+	// Try and see if there is a header
+	size := tx[0]
+
+	if size == HEADER_LEN {
+		encoded := tx[HEADER_SIZE : HEADER_SIZE+HEADER_LEN]
+		header := Header{
+			Version:         TransactionVersion(encoded[0]),
+			TransactionType: TransactionType(encoded[1]),
+			Reserved:        binary.BigEndian.Uint16(encoded[2:4]),
+		}
+
+		var transactionType TransactionType
+		if header.Version == V0 && header.Reserved == 0 {
+			switch header.TransactionType {
+			case Legacy:
+				transactionType = Legacy
+			case EphemeralKey:
+				transactionType = EphemeralKey
+			case Timeboost:
+				transactionType = Timeboost
+			default:
+				return nil
+			}
+			return &transactionType
+		}
+	}
+	return nil
+}
+
+func ParseHotShotPayload(payload []byte, txType *TransactionType) (signature []byte, userDataHash []byte, indices []uint64, messages [][]byte, err error) {
+	if txType != nil {
+		payload = payload[HEADER_SIZE+HEADER_LEN:]
+	}
 	if len(payload) < LEN_SIZE {
 		return nil, nil, nil, nil, errors.New("payload too short to parse signature size")
 	}

arbutil/espresso_utils_test.go

@@ -36,7 +36,8 @@ func TestParsePayload(t *testing.T) {
 	}
 
 	// Parse the signed payload
-	signature, userDataHash, indices, messages, err := ParseHotShotPayload(signedPayload)
+	txType := ParseHotshotPayloadForHeader(signedPayload)
+	signature, userDataHash, indices, messages, err := ParseHotShotPayload(signedPayload, txType)
 	if err != nil {
 		t.Fatalf("failed to parse payload: %v", err)
 	}
@@ -70,6 +71,91 @@ func TestParsePayload(t *testing.T) {
 	}
 }
 
+func TestParsePayloadWithAndWithoutHeader(t *testing.T) {
+	msgPositions := []MessageIndex{1, 2, 10, 24, 100}
+
+	rawPayload, cnt := BuildRawHotShotPayload(msgPositions, mockMsgFetcher, 200*1024)
+	if cnt != len(msgPositions) {
+		t.Fatal("exceed transactions")
+	}
+
+	mockSignature := []byte("fake_signature")
+	fakeSigner := func(payload []byte) ([]byte, error) {
+		return mockSignature, nil
+	}
+	signedPayload, err := SignHotShotPayload(rawPayload, fakeSigner)
+	if err != nil {
+		t.Fatalf("failed to sign payload: %v", err)
+	}
+
+	// Parse the signed payload
+	txType := ParseHotshotPayloadForHeader(signedPayload)
+	signature, userDataHash, indices, messages, err := ParseHotShotPayload(signedPayload, txType)
+	if err != nil {
+		t.Fatalf("failed to parse payload: %v", err)
+	}
+
+	if !slices.Equal(userDataHash, crypto.Keccak256(rawPayload)) {
+		t.Fatalf("User data hash is not for the correct payload")
+	}
+
+	// Validate parsed data
+	if !bytes.Equal(signature, mockSignature) {
+		t.Errorf("expected signature 'fake_signature', got %v", mockSignature)
+	}
+
+	for i, index := range indices {
+		if MessageIndex(index) != msgPositions[i] {
+			t.Errorf("expected index %d, got %d", msgPositions[i], index)
+		}
+	}
+
+	expectedMessages := [][]byte{
+		[]byte("message1"),
+		[]byte("message2"),
+		[]byte("message10"),
+		[]byte("message24"),
+		[]byte("message100"),
+	}
+	for i, message := range messages {
+		if !bytes.Equal(message, expectedMessages[i]) {
+			t.Errorf("expected message %s, got %s", expectedMessages[i], message)
+		}
+	}
+
+	// Remove header from payload
+	signedPayload = signedPayload[HEADER_LEN+HEADER_SIZE:]
+	txType = ParseHotshotPayloadForHeader(signedPayload)
+	if txType != nil {
+		t.Fatalf("no header should be parsed")
+	}
+	signature, userDataHash, indices, messages, err = ParseHotShotPayload(signedPayload, txType)
+	if err != nil {
+		t.Fatalf("failed to parse payload: %v", err)
+	}
+
+	if !slices.Equal(userDataHash, crypto.Keccak256(rawPayload)) {
+		t.Fatalf("User data hash is not for the correct payload")
+	}
+
+	// Validate parsed data
+	if !bytes.Equal(signature, mockSignature) {
+		t.Errorf("expected signature 'fake_signature', got %v", mockSignature)
+	}
+
+	for i, index := range indices {
+		if MessageIndex(index) != msgPositions[i] {
+			t.Errorf("expected index %d, got %d", msgPositions[i], index)
+		}
+	}
+
+	for i, message := range messages {
+		if !bytes.Equal(message, expectedMessages[i]) {
+			t.Errorf("expected message %s, got %s", expectedMessages[i], message)
+		}
+	}
+}
+
 func TestValidateIfPayloadIsInBlock(t *testing.T) {
 	msgPositions := []MessageIndex{1, 2}
 
@@ -123,7 +209,7 @@ func TestParsePayloadInvalidCases(t *testing.T) {
 
 	for _, tc := range invalidPayloads {
 		t.Run(tc.description, func(t *testing.T) {
-			_, _, _, _, err := ParseHotShotPayload(tc.payload)
+			_, _, _, _, err := ParseHotShotPayload(tc.payload, nil)
 			if err == nil {
 				t.Errorf("expected error for case '%s', but got none", tc.description)
 			}

espressostreamer/espresso_streamer.go

@@ -246,39 +246,88 @@ func (s *EspressoStreamer) verifyLegacy(attestation []byte, signature [32]byte)
 	return err
 }
 
-func (s *EspressoStreamer) parseEspressoTransaction(tx espressoTypes.Bytes, l1Height uint64) ([]*MessageWithMetadataAndPos, error) {
-	signature, userDataHash, indices, messages, err := arbutil.ParseHotShotPayload(tx)
-	if err != nil {
-		log.Warn("failed to parse hotshot payload", "err", err)
-		return nil, err
-	}
-	if len(messages) == 0 {
-		return nil, ErrPayloadHadNoMessages
+func (s *EspressoStreamer) fallbackLegacyVerification(data []byte, userDataHashArr [32]byte, l1Height uint64) error {
+	if s.espressoSGXVerifier == nil {
+		return fmt.Errorf("failed to verify attestation quote, legacy header found but sgx verifier is nil")
 	}
-	if len(userDataHash) != 32 {
-		log.Warn("user data hash is not 32 bytes")
-		return nil, ErrUserDataHashNot32Bytes
+	err := s.verifyLegacy(data, userDataHashArr)
+	if err != nil {
+		log.Warn("failed to verify attestation quote", "err", err)
+		return err
 	}
+	return nil
+}
 
-	userDataHashArr := [32]byte(userDataHash)
-
+func (s *EspressoStreamer) verifySignature(data []byte, userDataHashArr [32]byte, l1Height uint64, fallback bool) error {
+	err := s.verifyBatchPosterSignature(data, userDataHashArr, l1Height)
 	var success bool
-	err = s.verifyBatchPosterSignature(signature, userDataHashArr, l1Height)
 	if err == nil {
 		success = true
 	} else if strings.Contains(err.Error(), ErrRetryParsingHotShotPayload.Error()) {
 		log.Warn("retrying to verify batch poster signature", "err", err)
-		return nil, err
+		return err
 	} else {
 		log.Warn("failed to verify batch poster signature", "err", err)
+		if !fallback {
+			return err
+		}
 	}
 
-	if !success && s.espressoSGXVerifier != nil {
-		err = s.verifyLegacy(signature, userDataHashArr)
-		if err != nil {
-			log.Warn("failed to verify attestation quote", "err", err)
-			return nil, err
+	if !success {
+		if err := s.fallbackLegacyVerification(data, userDataHashArr, l1Height); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *EspressoStreamer) verify(data []byte, userDataHashArr [32]byte, l1Height uint64, transactionType *arbutil.TransactionType) error {
+	if transactionType != nil {
+		txType := *transactionType
+		switch txType {
+		case arbutil.Legacy:
+			if err := s.fallbackLegacyVerification(data, userDataHashArr, l1Height); err != nil {
+				return err
+			}
+		case arbutil.EphemeralKey:
+			if err := s.verifySignature(data, userDataHashArr, l1Height, true); err != nil {
+				return err
+			}
+		default:
+			return fmt.Errorf("failed to verify transaction, received unexpected transaction type: %d", txType)
 		}
+	} else if err := s.verifySignature(data, userDataHashArr, l1Height, true); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *EspressoStreamer) parseEspressoTransaction(tx espressoTypes.Bytes, l1Height uint64) ([]*MessageWithMetadataAndPos, error) {
+	transactionType := arbutil.ParseHotshotPayloadForHeader(tx)
+	signature, userDataHash, indices, messages, err := arbutil.ParseHotShotPayload(tx, transactionType)
+	if err != nil {
+		if transactionType != nil {
+			// in case somehow we parsed a header and there wasnt one, try again
+			transactionType = nil
+			signature, userDataHash, indices, messages, err = arbutil.ParseHotShotPayload(tx, transactionType)
+			if err != nil {
+				log.Warn("failed to parse hotshot payload", "err", err)
+				return nil, err
+			}
+		}
+
+	}
+	if len(messages) == 0 {
+		return nil, ErrPayloadHadNoMessages
+	}
+	if len(userDataHash) != 32 {
+		log.Warn("user data hash is not 32 bytes")
+		return nil, ErrUserDataHashNot32Bytes
+	}
+
+	err = s.verify(signature, [32]byte(userDataHash), l1Height, transactionType)
+	if err != nil {
+		return nil, err
 	}
 
 	result := []*MessageWithMetadataAndPos{}

system_tests/espresso/generate/messages.go

@@ -214,7 +214,8 @@ func ConvertEspressoTransactionsInBlockToMessages(
 		// We can parse the transactions to get the messages
 		// This is a mock function that simulates the parsing of the transaction
 		// In a real scenario, this would be replaced with the actual parsing logic
-		_, _, _, messages, err := arbutil.ParseHotShotPayload(tx)
+		transactionType := arbutil.ParseHotshotPayloadForHeader(tx)
+		_, _, _, messages, err := arbutil.ParseHotShotPayload(tx, transactionType)
 		if err != nil {
 			return nil, fmt.Errorf("encountered error while parsing transaction: %w", err)
 		}