diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4bc13aff92..39554550c4 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -10,6 +10,7 @@ beacon/merkle/ @zsfelfoldi beacon/types/ @zsfelfoldi @fjl beacon/params/ @zsfelfoldi @fjl cmd/evm/ @MariusVanDerWijden @lightclient +cmd/keeper/ @gballet core/state/ @rjl493456442 crypto/ @gballet @jwasinger @fjl core/ @rjl493456442 diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 342cd9da07..653b700622 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -70,8 +70,8 @@ jobs: - name: Install cross toolchain run: | - apt-get update - apt-get -yq --no-install-suggests --no-install-recommends install gcc-multilib + sudo apt-get update + sudo apt-get -yq --no-install-suggests --no-install-recommends install gcc-multilib - name: Build run: go run build/ci.go test -arch 386 -short -p 8 diff --git a/.github/workflows/validate_pr.yml b/.github/workflows/validate_pr.yml index d8148cfc97..3d5f978eba 100644 --- a/.github/workflows/validate_pr.yml +++ b/.github/workflows/validate_pr.yml @@ -10,10 +10,54 @@ jobs: validate-pr: runs-on: ubuntu-latest steps: + - name: Check for Spam PR + uses: actions/github-script@v7 + with: + script: | + const prTitle = context.payload.pull_request.title; + const spamRegex = /^(feat|chore|fix)(\(.*\))?\s*:/i; + + if (spamRegex.test(prTitle)) { + // Leave a comment explaining why + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + body: `## PR Closed as Spam + + This PR was automatically closed because the title format \`feat:\`, \`fix:\`, or \`chore:\` is commonly associated with spam contributions. + + If this is a legitimate contribution, please: + 1. Review our contribution guidelines + 2. Use the correct PR title format: \`directory, ...: description\` + 3. Open a new PR with the proper title format + + Thank you for your understanding.` + }); + + // Close the PR + await github.rest.pulls.update({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number, + state: 'closed' + }); + + core.setFailed('PR closed as spam due to suspicious title format'); + return; + } + + console.log('✅ PR passed spam check'); + + - name: Checkout repository + uses: actions/checkout@v4 + - name: Check PR Title Format uses: actions/github-script@v7 with: script: | + const fs = require('fs'); + const path = require('path'); const prTitle = context.payload.pull_request.title; const titleRegex = /^([\w\s,{}/.]+): .+/; @@ -21,6 +65,22 @@ jobs: core.setFailed(`PR title "${prTitle}" does not match required format: directory, ...: description`); return; } + + const match = prTitle.match(titleRegex); + const dirPart = match[1]; + const directories = dirPart.split(',').map(d => d.trim()); + const missingDirs = []; + for (const dir of directories) { + const fullPath = path.join(process.env.GITHUB_WORKSPACE, dir); + if (!fs.existsSync(fullPath)) { + missingDirs.push(dir); + } + } + + if (missingDirs.length > 0) { + core.setFailed(`The following directories in the PR title do not exist: ${missingDirs.join(', ')}`); + return; + } console.log('✅ PR title format is valid'); diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000..91032c8f17 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,102 @@ +# AGENTS + +## Guidelines + +- **Keep changes minimal and focused.** Only modify code directly related to the task at hand. Do not refactor unrelated code, rename existing variables or functions for style, or bundle unrelated fixes into the same commit or PR. +- **Do not add, remove, or update dependencies** unless the task explicitly requires it. + +## Pre-Commit Checklist + +Before every commit, run **all** of the following checks and ensure they pass: + +### 1. Formatting + +Before committing, always run `gofmt` and `goimports` on all modified files: + +```sh +gofmt -w +goimports -w +``` + +### 2. Build All Commands + +Verify that all tools compile successfully: + +```sh +make all +``` + +This builds all executables under `cmd/`, including `keeper` which has special build requirements. + +### 3. Tests + +While iterating during development, use `-short` for faster feedback: + +```sh +go run ./build/ci.go test -short +``` + +Before committing, run the full test suite **without** `-short` to ensure all tests pass, including the Ethereum execution-spec tests and all state/block test permutations: + +```sh +go run ./build/ci.go test +``` + +### 4. Linting + +```sh +go run ./build/ci.go lint +``` + +This runs additional style checks. Fix any issues before committing. + +### 5. Generated Code + +```sh +go run ./build/ci.go check_generate +``` + +Ensures that all generated files (e.g., `gen_*.go`) are up to date. If this fails, first install the required code generators by running `make devtools`, then run the appropriate `go generate` commands and include the updated files in your commit. + +### 6. Dependency Hygiene + +```sh +go run ./build/ci.go check_baddeps +``` + +Verifies that no forbidden dependencies have been introduced. + +## What to include in commits + +Do not commit binaries, whether they are produced by the main build or byproducts of investigations. + +## Commit Message Format + +Commit messages must be prefixed with the package(s) they modify, followed by a short lowercase description: + +``` +: description +``` + +Examples: +- `core/vm: fix stack overflow in PUSH instruction` +- `eth, rpc: make trace configs optional` +- `cmd/geth: add new flag for sync mode` + +Use comma-separated package names when multiple areas are affected. Keep the description concise. + +## Pull Request Title Format + +PR titles follow the same convention as commit messages: + +``` +: description +``` + +Examples: +- `core/vm: fix stack overflow in PUSH instruction` +- `core, eth: add arena allocator support` +- `cmd/geth, internal/ethapi: refactor transaction args` +- `trie/archiver: streaming subtree archival to fix OOM` + +Use the top-level package paths, comma-separated if multiple areas are affected. Only mention the directories with functional changes, interface changes that trickle all over the codebase should not generate an exhaustive list. The description should be a short, lowercase summary of the change. diff --git a/Dockerfile b/Dockerfile index 9b70e9e8a0..940e568cab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ ARG VERSION="" ARG BUILDNUM="" # Build Geth in a stock Go builder container -FROM golang:1.24-alpine AS builder +FROM golang:1.26-alpine AS builder RUN apk add --no-cache gcc musl-dev linux-headers git diff --git a/Dockerfile.alltools b/Dockerfile.alltools index ac9303c678..ae2ca931ae 100644 --- a/Dockerfile.alltools +++ b/Dockerfile.alltools @@ -4,7 +4,7 @@ ARG VERSION="" ARG BUILDNUM="" # Build Geth in a stock Go builder container -FROM golang:1.24-alpine AS builder +FROM golang:1.26-alpine AS builder RUN apk add --no-cache gcc musl-dev linux-headers git diff --git a/SECURITY.md b/SECURITY.md index 0b497b44ae..d497248de5 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -21,7 +21,6 @@ Audit reports are published in the `docs` folder: https://github.com/ethereum/go To find out how to disclose a vulnerability in Ethereum visit [https://bounty.ethereum.org](https://bounty.ethereum.org) or email bounty@ethereum.org. Please read the [disclosure page](https://github.com/ethereum/go-ethereum/security/advisories?state=published) for more information about publicly disclosed security vulnerabilities. -Use the built-in `geth version-check` feature to check whether the software is affected by any known vulnerability. This command will fetch the latest [`vulnerabilities.json`](https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities.json) file which contains known security vulnerabilities concerning `geth`, and cross-check the data against its own version number. The following key may be used to communicate sensitive information to developers. diff --git a/accounts/abi/bind/v2/dep_tree_test.go b/accounts/abi/bind/v2/dep_tree_test.go index e686e3fec4..b2470d8a16 100644 --- a/accounts/abi/bind/v2/dep_tree_test.go +++ b/accounts/abi/bind/v2/dep_tree_test.go @@ -158,10 +158,10 @@ func testLinkCase(tcInput linkTestCaseInput) error { overrideAddrs = make(map[rune]common.Address) ) // generate deterministic addresses for the override set. - rand.Seed(42) + rng := rand.New(rand.NewSource(42)) for contract := range tcInput.overrides { var addr common.Address - rand.Read(addr[:]) + rng.Read(addr[:]) overrideAddrs[contract] = addr overridesAddrs[addr] = struct{}{} } diff --git a/accounts/keystore/keystore.go b/accounts/keystore/keystore.go index 3e4266924f..29c4bdf2ca 100644 --- a/accounts/keystore/keystore.go +++ b/accounts/keystore/keystore.go @@ -418,6 +418,7 @@ func (ks *KeyStore) Export(a accounts.Account, passphrase, newPassphrase string) if err != nil { return nil, err } + defer zeroKey(key.PrivateKey) var N, P int if store, ok := ks.storage.(*keyStorePassphrase); ok { N, P = store.scryptN, store.scryptP @@ -477,6 +478,7 @@ func (ks *KeyStore) Update(a accounts.Account, passphrase, newPassphrase string) if err != nil { return err } + defer zeroKey(key.PrivateKey) return ks.storage.StoreKey(a.URL.Path, key, newPassphrase) } diff --git a/accounts/keystore/presale.go b/accounts/keystore/presale.go index 0664dc2cdd..6311e8d90a 100644 --- a/accounts/keystore/presale.go +++ b/accounts/keystore/presale.go @@ -81,6 +81,9 @@ func decryptPreSaleKey(fileContent []byte, password string) (key *Key, err error */ passBytes := []byte(password) derivedKey := pbkdf2.Key(passBytes, passBytes, 2000, 16, sha256.New) + if len(cipherText)%aes.BlockSize != 0 { + return nil, errors.New("ciphertext must be a multiple of block size") + } plainText, err := aesCBCDecrypt(derivedKey, cipherText, iv) if err != nil { return nil, err diff --git a/accounts/scwallet/securechannel.go b/accounts/scwallet/securechannel.go index b3a7be8df0..1e0230dc45 100644 --- a/accounts/scwallet/securechannel.go +++ b/accounts/scwallet/securechannel.go @@ -300,6 +300,10 @@ func (s *SecureChannelSession) decryptAPDU(data []byte) ([]byte, error) { return nil, err } + if len(data) == 0 || len(data)%aes.BlockSize != 0 { + return nil, fmt.Errorf("invalid ciphertext length: %d", len(data)) + } + ret := make([]byte, len(data)) crypter := cipher.NewCBCDecrypter(a, s.iv) diff --git a/accounts/usbwallet/hub.go b/accounts/usbwallet/hub.go index 81457b7da2..6f8ac0d8d9 100644 --- a/accounts/usbwallet/hub.go +++ b/accounts/usbwallet/hub.go @@ -43,6 +43,14 @@ const refreshCycle = time.Second // trashing. const refreshThrottling = 500 * time.Millisecond +const ( + // deviceUsagePage identifies Ledger devices by HID usage page (0xffa0) on Windows and macOS. + // See: https://github.com/LedgerHQ/ledger-live/blob/05a2980e838955a11a1418da638ef8ac3df4fb74/libs/ledgerjs/packages/hw-transport-node-hid-noevents/src/TransportNodeHid.ts + deviceUsagePage = 0xffa0 + // deviceInterface identifies Ledger devices by USB interface number (0) on Linux. + deviceInterface = 0 +) + // Hub is a accounts.Backend that can find and handle generic USB hardware wallets. type Hub struct { scheme string // Protocol scheme prefixing account and wallet URLs. @@ -82,6 +90,7 @@ func NewLedgerHub() (*Hub, error) { 0x0005, /* Ledger Nano S Plus */ 0x0006, /* Ledger Nano FTS */ 0x0007, /* Ledger Flex */ + 0x0008, /* Ledger Nano Gen5 */ 0x0000, /* WebUSB Ledger Blue */ 0x1000, /* WebUSB Ledger Nano S */ @@ -89,7 +98,8 @@ func NewLedgerHub() (*Hub, error) { 0x5000, /* WebUSB Ledger Nano S Plus */ 0x6000, /* WebUSB Ledger Nano FTS */ 0x7000, /* WebUSB Ledger Flex */ - }, 0xffa0, 0, newLedgerDriver) + 0x8000, /* WebUSB Ledger Nano Gen5 */ + }, deviceUsagePage, deviceInterface, newLedgerDriver) } // NewTrezorHubWithHID creates a new hardware wallet manager for Trezor devices. diff --git a/accounts/usbwallet/ledger.go b/accounts/usbwallet/ledger.go index 52595a1621..80e63f1864 100644 --- a/accounts/usbwallet/ledger.go +++ b/accounts/usbwallet/ledger.go @@ -184,7 +184,7 @@ func (w *ledgerDriver) SignTypedMessage(path accounts.DerivationPath, domainHash return nil, accounts.ErrWalletClosed } // Ensure the wallet is capable of signing the given transaction - if w.version[0] < 1 && w.version[1] < 5 { + if w.version[0] < 1 || (w.version[0] == 1 && w.version[1] < 5) { //lint:ignore ST1005 brand name displayed on the console return nil, fmt.Errorf("Ledger version >= 1.5.0 required for EIP-712 signing (found version v%d.%d.%d)", w.version[0], w.version[1], w.version[2]) } diff --git a/accounts/usbwallet/wallet.go b/accounts/usbwallet/wallet.go index 0fd0415a9e..f1597ca1a7 100644 --- a/accounts/usbwallet/wallet.go +++ b/accounts/usbwallet/wallet.go @@ -632,7 +632,7 @@ func (w *wallet) SignTx(account accounts.Account, tx *types.Transaction, chainID // data is not supported for Ledger wallets, so this method will always return // an error. func (w *wallet) SignTextWithPassphrase(account accounts.Account, passphrase string, text []byte) ([]byte, error) { - return w.SignText(account, accounts.TextHash(text)) + return w.SignText(account, text) } // SignTxWithPassphrase implements accounts.Wallet, attempting to sign the given diff --git a/arbcrypto/keccak.go b/arbcrypto/keccak.go index 61f0f33c3e..4218b8771f 100644 --- a/arbcrypto/keccak.go +++ b/arbcrypto/keccak.go @@ -5,9 +5,9 @@ package arbcrypto import ( "hash" - "golang.org/x/crypto/sha3" + "github.com/ethereum/go-ethereum/crypto/keccak" ) func NewLegacyKeccak256() hash.Hash { - return sha3.NewLegacyKeccak256() + return keccak.NewLegacyKeccak256() } diff --git a/arbitrum/apibackend.go b/arbitrum/apibackend.go index 36ed16c286..49de6aa38f 100644 --- a/arbitrum/apibackend.go +++ b/arbitrum/apibackend.go @@ -587,7 +587,7 @@ func StateAndHeaderFromHeader(ctx context.Context, chainDb ethdb.Database, bc *c return statedb, NoopStateRelease, nil } } - liveState, liveStateRelease, err := stateFor(bc.StateCache(), bc.Snapshots())(header) + liveState, liveStateRelease, err := stateFor(state.NewDatabase(bc.TrieDB(), bc.CodeDB()), bc.Snapshots())(header) if err == nil { liveStatesReferencedCounter.Inc(1) liveState.SetArbFinalizer(func(*state.ArbitrumExtraData) { diff --git a/arbitrum/recreatestate.go b/arbitrum/recreatestate.go index 996a08bfa3..a729565502 100644 --- a/arbitrum/recreatestate.go +++ b/arbitrum/recreatestate.go @@ -80,7 +80,7 @@ func AdvanceStateByBlock(ctx context.Context, bc *core.BlockChain, state *state. if logFunc != nil { logFunc(block.Header(), true) } - result, err := bc.Processor().Process(block, state, vmConfig) + result, err := bc.Processor().Process(ctx, block, state, vmConfig) if err != nil { return nil, nil, nil, fmt.Errorf("failed recreating state for block %d : %w", blockToRecreate, err) } diff --git a/arbitrum/retryables/scheduled_txes.go b/arbitrum/retryables/scheduled_txes.go index 029c4425a0..31de3c58ac 100644 --- a/arbitrum/retryables/scheduled_txes.go +++ b/arbitrum/retryables/scheduled_txes.go @@ -39,7 +39,7 @@ func RunScheduledTxes(ctx context.Context, b core.NodeInterfaceBackendAPI, state evm.Cancel() }() - scheduledTxResult, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(math.MaxUint64)) + scheduledTxResult, err := core.ApplyMessage(evm, msg, core.NewGasPool(math.MaxUint64)) if err != nil { return nil, err } diff --git a/beacon/blsync/engineclient.go b/beacon/blsync/engineclient.go index f9821fc6f3..6cac2bc5a8 100644 --- a/beacon/blsync/engineclient.go +++ b/beacon/blsync/engineclient.go @@ -87,6 +87,10 @@ func (ec *engineClient) updateLoop(headCh <-chan types.ChainHeadEvent) { if status, err := ec.callForkchoiceUpdated(forkName, event); err == nil { log.Info("Successful ForkchoiceUpdated", "head", event.Block.Hash(), "status", status) } else { + if err.Error() == "beacon syncer reorging" { + log.Debug("Failed ForkchoiceUpdated", "head", event.Block.Hash(), "error", err) + continue // ignore beacon syncer reorging errors, this error can occur if the blsync is skipping a block + } log.Error("Failed ForkchoiceUpdated", "head", event.Block.Hash(), "error", err) } } @@ -101,7 +105,16 @@ func (ec *engineClient) callNewPayload(fork string, event types.ChainHeadEvent) params = []any{execData} ) switch fork { - case "electra": + case "altair", "bellatrix": + method = "engine_newPayloadV1" + case "capella": + method = "engine_newPayloadV2" + case "deneb": + method = "engine_newPayloadV3" + parentBeaconRoot := event.BeaconHead.ParentRoot + blobHashes := collectBlobHashes(event.Block) + params = append(params, blobHashes, parentBeaconRoot) + default: // electra, fulu and above method = "engine_newPayloadV4" parentBeaconRoot := event.BeaconHead.ParentRoot blobHashes := collectBlobHashes(event.Block) @@ -110,15 +123,6 @@ func (ec *engineClient) callNewPayload(fork string, event types.ChainHeadEvent) hexRequests[i] = hexutil.Bytes(event.ExecRequests[i]) } params = append(params, blobHashes, parentBeaconRoot, hexRequests) - case "deneb": - method = "engine_newPayloadV3" - parentBeaconRoot := event.BeaconHead.ParentRoot - blobHashes := collectBlobHashes(event.Block) - params = append(params, blobHashes, parentBeaconRoot) - case "capella": - method = "engine_newPayloadV2" - default: - method = "engine_newPayloadV1" } ctx, cancel := context.WithTimeout(ec.rootCtx, time.Second*5) @@ -145,12 +149,12 @@ func (ec *engineClient) callForkchoiceUpdated(fork string, event types.ChainHead var method string switch fork { - case "deneb", "electra": - method = "engine_forkchoiceUpdatedV3" + case "altair", "bellatrix": + method = "engine_forkchoiceUpdatedV1" case "capella": method = "engine_forkchoiceUpdatedV2" - default: - method = "engine_forkchoiceUpdatedV1" + default: // deneb, electra, fulu and above + method = "engine_forkchoiceUpdatedV3" } ctx, cancel := context.WithTimeout(ec.rootCtx, time.Second*5) diff --git a/beacon/engine/gen_blockparams.go b/beacon/engine/gen_blockparams.go index b1f01b50ff..3a5d7ae593 100644 --- a/beacon/engine/gen_blockparams.go +++ b/beacon/engine/gen_blockparams.go @@ -21,6 +21,7 @@ func (p PayloadAttributes) MarshalJSON() ([]byte, error) { SuggestedFeeRecipient common.Address `json:"suggestedFeeRecipient" gencodec:"required"` Withdrawals []*types.Withdrawal `json:"withdrawals"` BeaconRoot *common.Hash `json:"parentBeaconBlockRoot"` + SlotNumber *hexutil.Uint64 `json:"slotNumber"` } var enc PayloadAttributes enc.Timestamp = hexutil.Uint64(p.Timestamp) @@ -28,6 +29,7 @@ func (p PayloadAttributes) MarshalJSON() ([]byte, error) { enc.SuggestedFeeRecipient = p.SuggestedFeeRecipient enc.Withdrawals = p.Withdrawals enc.BeaconRoot = p.BeaconRoot + enc.SlotNumber = (*hexutil.Uint64)(p.SlotNumber) return json.Marshal(&enc) } @@ -39,6 +41,7 @@ func (p *PayloadAttributes) UnmarshalJSON(input []byte) error { SuggestedFeeRecipient *common.Address `json:"suggestedFeeRecipient" gencodec:"required"` Withdrawals []*types.Withdrawal `json:"withdrawals"` BeaconRoot *common.Hash `json:"parentBeaconBlockRoot"` + SlotNumber *hexutil.Uint64 `json:"slotNumber"` } var dec PayloadAttributes if err := json.Unmarshal(input, &dec); err != nil { @@ -62,5 +65,8 @@ func (p *PayloadAttributes) UnmarshalJSON(input []byte) error { if dec.BeaconRoot != nil { p.BeaconRoot = dec.BeaconRoot } + if dec.SlotNumber != nil { + p.SlotNumber = (*uint64)(dec.SlotNumber) + } return nil } diff --git a/beacon/engine/gen_ed.go b/beacon/engine/gen_ed.go index 0ae5a3b8f1..b460368b84 100644 --- a/beacon/engine/gen_ed.go +++ b/beacon/engine/gen_ed.go @@ -17,24 +17,24 @@ var _ = (*executableDataMarshaling)(nil) // MarshalJSON marshals as JSON. func (e ExecutableData) MarshalJSON() ([]byte, error) { type ExecutableData struct { - ParentHash common.Hash `json:"parentHash" gencodec:"required"` - FeeRecipient common.Address `json:"feeRecipient" gencodec:"required"` - StateRoot common.Hash `json:"stateRoot" gencodec:"required"` - ReceiptsRoot common.Hash `json:"receiptsRoot" gencodec:"required"` - LogsBloom hexutil.Bytes `json:"logsBloom" gencodec:"required"` - Random common.Hash `json:"prevRandao" gencodec:"required"` - Number hexutil.Uint64 `json:"blockNumber" gencodec:"required"` - GasLimit hexutil.Uint64 `json:"gasLimit" gencodec:"required"` - GasUsed hexutil.Uint64 `json:"gasUsed" gencodec:"required"` - Timestamp hexutil.Uint64 `json:"timestamp" gencodec:"required"` - ExtraData hexutil.Bytes `json:"extraData" gencodec:"required"` - BaseFeePerGas *hexutil.Big `json:"baseFeePerGas" gencodec:"required"` - BlockHash common.Hash `json:"blockHash" gencodec:"required"` - Transactions []hexutil.Bytes `json:"transactions" gencodec:"required"` - Withdrawals []*types.Withdrawal `json:"withdrawals"` - BlobGasUsed *hexutil.Uint64 `json:"blobGasUsed"` - ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas"` - ExecutionWitness *types.ExecutionWitness `json:"executionWitness,omitempty"` + ParentHash common.Hash `json:"parentHash" gencodec:"required"` + FeeRecipient common.Address `json:"feeRecipient" gencodec:"required"` + StateRoot common.Hash `json:"stateRoot" gencodec:"required"` + ReceiptsRoot common.Hash `json:"receiptsRoot" gencodec:"required"` + LogsBloom hexutil.Bytes `json:"logsBloom" gencodec:"required"` + Random common.Hash `json:"prevRandao" gencodec:"required"` + Number hexutil.Uint64 `json:"blockNumber" gencodec:"required"` + GasLimit hexutil.Uint64 `json:"gasLimit" gencodec:"required"` + GasUsed hexutil.Uint64 `json:"gasUsed" gencodec:"required"` + Timestamp hexutil.Uint64 `json:"timestamp" gencodec:"required"` + ExtraData hexutil.Bytes `json:"extraData" gencodec:"required"` + BaseFeePerGas *hexutil.Big `json:"baseFeePerGas" gencodec:"required"` + BlockHash common.Hash `json:"blockHash" gencodec:"required"` + Transactions []hexutil.Bytes `json:"transactions" gencodec:"required"` + Withdrawals []*types.Withdrawal `json:"withdrawals"` + BlobGasUsed *hexutil.Uint64 `json:"blobGasUsed"` + ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas"` + SlotNumber *hexutil.Uint64 `json:"slotNumber"` } var enc ExecutableData enc.ParentHash = e.ParentHash @@ -59,31 +59,31 @@ func (e ExecutableData) MarshalJSON() ([]byte, error) { enc.Withdrawals = e.Withdrawals enc.BlobGasUsed = (*hexutil.Uint64)(e.BlobGasUsed) enc.ExcessBlobGas = (*hexutil.Uint64)(e.ExcessBlobGas) - enc.ExecutionWitness = e.ExecutionWitness + enc.SlotNumber = (*hexutil.Uint64)(e.SlotNumber) return json.Marshal(&enc) } // UnmarshalJSON unmarshals from JSON. func (e *ExecutableData) UnmarshalJSON(input []byte) error { type ExecutableData struct { - ParentHash *common.Hash `json:"parentHash" gencodec:"required"` - FeeRecipient *common.Address `json:"feeRecipient" gencodec:"required"` - StateRoot *common.Hash `json:"stateRoot" gencodec:"required"` - ReceiptsRoot *common.Hash `json:"receiptsRoot" gencodec:"required"` - LogsBloom *hexutil.Bytes `json:"logsBloom" gencodec:"required"` - Random *common.Hash `json:"prevRandao" gencodec:"required"` - Number *hexutil.Uint64 `json:"blockNumber" gencodec:"required"` - GasLimit *hexutil.Uint64 `json:"gasLimit" gencodec:"required"` - GasUsed *hexutil.Uint64 `json:"gasUsed" gencodec:"required"` - Timestamp *hexutil.Uint64 `json:"timestamp" gencodec:"required"` - ExtraData *hexutil.Bytes `json:"extraData" gencodec:"required"` - BaseFeePerGas *hexutil.Big `json:"baseFeePerGas" gencodec:"required"` - BlockHash *common.Hash `json:"blockHash" gencodec:"required"` - Transactions []hexutil.Bytes `json:"transactions" gencodec:"required"` - Withdrawals []*types.Withdrawal `json:"withdrawals"` - BlobGasUsed *hexutil.Uint64 `json:"blobGasUsed"` - ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas"` - ExecutionWitness *types.ExecutionWitness `json:"executionWitness,omitempty"` + ParentHash *common.Hash `json:"parentHash" gencodec:"required"` + FeeRecipient *common.Address `json:"feeRecipient" gencodec:"required"` + StateRoot *common.Hash `json:"stateRoot" gencodec:"required"` + ReceiptsRoot *common.Hash `json:"receiptsRoot" gencodec:"required"` + LogsBloom *hexutil.Bytes `json:"logsBloom" gencodec:"required"` + Random *common.Hash `json:"prevRandao" gencodec:"required"` + Number *hexutil.Uint64 `json:"blockNumber" gencodec:"required"` + GasLimit *hexutil.Uint64 `json:"gasLimit" gencodec:"required"` + GasUsed *hexutil.Uint64 `json:"gasUsed" gencodec:"required"` + Timestamp *hexutil.Uint64 `json:"timestamp" gencodec:"required"` + ExtraData *hexutil.Bytes `json:"extraData" gencodec:"required"` + BaseFeePerGas *hexutil.Big `json:"baseFeePerGas" gencodec:"required"` + BlockHash *common.Hash `json:"blockHash" gencodec:"required"` + Transactions []hexutil.Bytes `json:"transactions" gencodec:"required"` + Withdrawals []*types.Withdrawal `json:"withdrawals"` + BlobGasUsed *hexutil.Uint64 `json:"blobGasUsed"` + ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas"` + SlotNumber *hexutil.Uint64 `json:"slotNumber"` } var dec ExecutableData if err := json.Unmarshal(input, &dec); err != nil { @@ -157,8 +157,8 @@ func (e *ExecutableData) UnmarshalJSON(input []byte) error { if dec.ExcessBlobGas != nil { e.ExcessBlobGas = (*uint64)(dec.ExcessBlobGas) } - if dec.ExecutionWitness != nil { - e.ExecutionWitness = dec.ExecutionWitness + if dec.SlotNumber != nil { + e.SlotNumber = (*uint64)(dec.SlotNumber) } return nil } diff --git a/beacon/engine/types.go b/beacon/engine/types.go index ddb276ab09..5c94e67de1 100644 --- a/beacon/engine/types.go +++ b/beacon/engine/types.go @@ -50,6 +50,13 @@ var ( // ExecutionPayloadV3 has the syntax of ExecutionPayloadV2 and appends the new // fields: blobGasUsed and excessBlobGas. PayloadV3 PayloadVersion = 0x3 + + // PayloadV4 is the identifier of ExecutionPayloadV4 introduced in amsterdam fork. + // + // https://github.com/ethereum/execution-apis/blob/main/src/engine/amsterdam.md#executionpayloadv4 + // ExecutionPayloadV4 has the syntax of ExecutionPayloadV3 and appends the new + // field slotNumber. + PayloadV4 PayloadVersion = 0x4 ) //go:generate go run github.com/fjl/gencodec -type PayloadAttributes -field-override payloadAttributesMarshaling -out gen_blockparams.go @@ -62,35 +69,37 @@ type PayloadAttributes struct { SuggestedFeeRecipient common.Address `json:"suggestedFeeRecipient" gencodec:"required"` Withdrawals []*types.Withdrawal `json:"withdrawals"` BeaconRoot *common.Hash `json:"parentBeaconBlockRoot"` + SlotNumber *uint64 `json:"slotNumber"` } // JSON type overrides for PayloadAttributes. type payloadAttributesMarshaling struct { - Timestamp hexutil.Uint64 + Timestamp hexutil.Uint64 + SlotNumber *hexutil.Uint64 } //go:generate go run github.com/fjl/gencodec -type ExecutableData -field-override executableDataMarshaling -out gen_ed.go // ExecutableData is the data necessary to execute an EL payload. type ExecutableData struct { - ParentHash common.Hash `json:"parentHash" gencodec:"required"` - FeeRecipient common.Address `json:"feeRecipient" gencodec:"required"` - StateRoot common.Hash `json:"stateRoot" gencodec:"required"` - ReceiptsRoot common.Hash `json:"receiptsRoot" gencodec:"required"` - LogsBloom []byte `json:"logsBloom" gencodec:"required"` - Random common.Hash `json:"prevRandao" gencodec:"required"` - Number uint64 `json:"blockNumber" gencodec:"required"` - GasLimit uint64 `json:"gasLimit" gencodec:"required"` - GasUsed uint64 `json:"gasUsed" gencodec:"required"` - Timestamp uint64 `json:"timestamp" gencodec:"required"` - ExtraData []byte `json:"extraData" gencodec:"required"` - BaseFeePerGas *big.Int `json:"baseFeePerGas" gencodec:"required"` - BlockHash common.Hash `json:"blockHash" gencodec:"required"` - Transactions [][]byte `json:"transactions" gencodec:"required"` - Withdrawals []*types.Withdrawal `json:"withdrawals"` - BlobGasUsed *uint64 `json:"blobGasUsed"` - ExcessBlobGas *uint64 `json:"excessBlobGas"` - ExecutionWitness *types.ExecutionWitness `json:"executionWitness,omitempty"` + ParentHash common.Hash `json:"parentHash" gencodec:"required"` + FeeRecipient common.Address `json:"feeRecipient" gencodec:"required"` + StateRoot common.Hash `json:"stateRoot" gencodec:"required"` + ReceiptsRoot common.Hash `json:"receiptsRoot" gencodec:"required"` + LogsBloom []byte `json:"logsBloom" gencodec:"required"` + Random common.Hash `json:"prevRandao" gencodec:"required"` + Number uint64 `json:"blockNumber" gencodec:"required"` + GasLimit uint64 `json:"gasLimit" gencodec:"required"` + GasUsed uint64 `json:"gasUsed" gencodec:"required"` + Timestamp uint64 `json:"timestamp" gencodec:"required"` + ExtraData []byte `json:"extraData" gencodec:"required"` + BaseFeePerGas *big.Int `json:"baseFeePerGas" gencodec:"required"` + BlockHash common.Hash `json:"blockHash" gencodec:"required"` + Transactions [][]byte `json:"transactions" gencodec:"required"` + Withdrawals []*types.Withdrawal `json:"withdrawals"` + BlobGasUsed *uint64 `json:"blobGasUsed"` + ExcessBlobGas *uint64 `json:"excessBlobGas"` + SlotNumber *uint64 `json:"slotNumber"` } // JSON type overrides for executableData. @@ -105,6 +114,7 @@ type executableDataMarshaling struct { Transactions []hexutil.Bytes BlobGasUsed *hexutil.Uint64 ExcessBlobGas *hexutil.Uint64 + SlotNumber *hexutil.Uint64 } // StatelessPayloadStatusV1 is the result of a stateless payload execution. @@ -214,7 +224,7 @@ func encodeTransactions(txs []*types.Transaction) [][]byte { return enc } -func decodeTransactions(enc [][]byte) ([]*types.Transaction, error) { +func DecodeTransactions(enc [][]byte) ([]*types.Transaction, error) { var txs = make([]*types.Transaction, len(enc)) for i, encTx := range enc { var tx types.Transaction @@ -252,7 +262,7 @@ func ExecutableDataToBlock(data ExecutableData, versionedHashes []common.Hash, b // for stateless execution, so it skips checking if the executable data hashes to // the requested hash (stateless has to *compute* the root hash, it's not given). func ExecutableDataToBlockNoHash(data ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash, requests [][]byte) (*types.Block, error) { - txs, err := decodeTransactions(data.Transactions) + txs, err := DecodeTransactions(data.Transactions) if err != nil { return nil, err } @@ -314,10 +324,10 @@ func ExecutableDataToBlockNoHash(data ExecutableData, versionedHashes []common.H BlobGasUsed: data.BlobGasUsed, ParentBeaconRoot: beaconRoot, RequestsHash: requestsHash, + SlotNumber: data.SlotNumber, } return types.NewBlockWithHeader(header). - WithBody(types.Body{Transactions: txs, Uncles: nil, Withdrawals: data.Withdrawals}). - WithWitness(data.ExecutionWitness), + WithBody(types.Body{Transactions: txs, Uncles: nil, Withdrawals: data.Withdrawals}), nil } @@ -325,24 +335,24 @@ func ExecutableDataToBlockNoHash(data ExecutableData, versionedHashes []common.H // fields from the given block. It assumes the given block is post-merge block. func BlockToExecutableData(block *types.Block, fees *big.Int, sidecars []*types.BlobTxSidecar, requests [][]byte) *ExecutionPayloadEnvelope { data := &ExecutableData{ - BlockHash: block.Hash(), - ParentHash: block.ParentHash(), - FeeRecipient: block.Coinbase(), - StateRoot: block.Root(), - Number: block.NumberU64(), - GasLimit: block.GasLimit(), - GasUsed: block.GasUsed(), - BaseFeePerGas: block.BaseFee(), - Timestamp: block.Time(), - ReceiptsRoot: block.ReceiptHash(), - LogsBloom: block.Bloom().Bytes(), - Transactions: encodeTransactions(block.Transactions()), - Random: block.MixDigest(), - ExtraData: block.Extra(), - Withdrawals: block.Withdrawals(), - BlobGasUsed: block.BlobGasUsed(), - ExcessBlobGas: block.ExcessBlobGas(), - ExecutionWitness: block.ExecutionWitness(), + BlockHash: block.Hash(), + ParentHash: block.ParentHash(), + FeeRecipient: block.Coinbase(), + StateRoot: block.Root(), + Number: block.NumberU64(), + GasLimit: block.GasLimit(), + GasUsed: block.GasUsed(), + BaseFeePerGas: block.BaseFee(), + Timestamp: block.Time(), + ReceiptsRoot: block.ReceiptHash(), + LogsBloom: block.Bloom().Bytes(), + Transactions: encodeTransactions(block.Transactions()), + Random: block.MixDigest(), + ExtraData: block.Extra(), + Withdrawals: block.Withdrawals(), + BlobGasUsed: block.BlobGasUsed(), + ExcessBlobGas: block.ExcessBlobGas(), + SlotNumber: block.SlotNumber(), } // Add blobs. diff --git a/beacon/light/canonical.go b/beacon/light/canonical.go index b5371493b4..56622425b2 100644 --- a/beacon/light/canonical.go +++ b/beacon/light/canonical.go @@ -69,7 +69,10 @@ func newCanonicalStore[T any](db ethdb.Iteratee, keyPrefix []byte) (*canonicalSt // databaseKey returns the database key belonging to the given period. func (cs *canonicalStore[T]) databaseKey(period uint64) []byte { - return binary.BigEndian.AppendUint64(append([]byte{}, cs.keyPrefix...), period) + key := make([]byte, len(cs.keyPrefix)+8) + copy(key, cs.keyPrefix) + binary.BigEndian.PutUint64(key[len(cs.keyPrefix):], period) + return key } // add adds the given item to the database. It also ensures that the range remains diff --git a/beacon/light/sync/head_sync.go b/beacon/light/sync/head_sync.go index 5e41258053..7189767d9c 100644 --- a/beacon/light/sync/head_sync.go +++ b/beacon/light/sync/head_sync.go @@ -105,6 +105,7 @@ func (s *HeadSync) Process(requester request.Requester, events []request.Event) delete(s.serverHeads, event.Server) delete(s.unvalidatedOptimistic, event.Server) delete(s.unvalidatedFinality, event.Server) + delete(s.reqFinalityEpoch, event.Server) } } } diff --git a/beacon/params/checkpoint_hoodi.hex b/beacon/params/checkpoint_hoodi.hex index 2885d7c996..7bac591f96 100644 --- a/beacon/params/checkpoint_hoodi.hex +++ b/beacon/params/checkpoint_hoodi.hex @@ -1 +1 @@ -0x1bbf958008172591b6cbdb3d8d52e26998258e83d4bdb9eec10969d84519a6bd \ No newline at end of file +0xbb7a7f3c40d8ea0b450f91587db65d0f1c079669277e01a0426c8911702a863a \ No newline at end of file diff --git a/beacon/params/checkpoint_mainnet.hex b/beacon/params/checkpoint_mainnet.hex index 417e69a24b..c2886cc564 100644 --- a/beacon/params/checkpoint_mainnet.hex +++ b/beacon/params/checkpoint_mainnet.hex @@ -1 +1 @@ -0x2fe39a39b6f7cbd549e0f74d259de6db486005a65bd3bd92840dd6ce21d6f4c8 \ No newline at end of file +0x2af778d703186526a1b6304b423f338f11556206f618643c3f7fa0d7b1ef5c9b \ No newline at end of file diff --git a/beacon/params/checkpoint_sepolia.hex b/beacon/params/checkpoint_sepolia.hex index 02faf72187..55842f8ac0 100644 --- a/beacon/params/checkpoint_sepolia.hex +++ b/beacon/params/checkpoint_sepolia.hex @@ -1 +1 @@ -0x86686b2b366e24134e0e3969a9c5f3759f92e5d2b04785b42e22cc7d468c2107 \ No newline at end of file +0x48a89c9ea7ba19de2931797974cf8722344ab231c0edada278b108ef74125478 \ No newline at end of file diff --git a/beacon/params/config.go b/beacon/params/config.go index b01b739e07..437aa53788 100644 --- a/beacon/params/config.go +++ b/beacon/params/config.go @@ -38,7 +38,7 @@ import ( // across signing different data structures. const syncCommitteeDomain = 7 -var knownForks = []string{"GENESIS", "ALTAIR", "BELLATRIX", "CAPELLA", "DENEB"} +var knownForks = []string{"GENESIS", "ALTAIR", "BELLATRIX", "CAPELLA", "DENEB", "ELECTRA", "FULU"} // ClientConfig contains beacon light client configuration. type ClientConfig struct { @@ -103,6 +103,9 @@ func (c *ChainConfig) LoadForks(file []byte) error { epochs["GENESIS"] = 0 for key, value := range config { + if value == nil { + continue + } if strings.HasSuffix(key, "_FORK_VERSION") { name := key[:len(key)-len("_FORK_VERSION")] switch version := value.(type) { diff --git a/beacon/params/config_test.go b/beacon/params/config_test.go index 41e120469b..0b569b604c 100644 --- a/beacon/params/config_test.go +++ b/beacon/params/config_test.go @@ -15,6 +15,9 @@ ALTAIR_FORK_EPOCH: 1 EIP7928_FORK_VERSION: 0xb0000038 EIP7928_FORK_EPOCH: 18446744073709551615 +EIP7XXX_FORK_VERSION: +EIP7XXX_FORK_EPOCH: + BLOB_SCHEDULE: [] ` c := &ChainConfig{} diff --git a/beacon/params/networks.go b/beacon/params/networks.go index b35db34fd6..5dcf08cc5d 100644 --- a/beacon/params/networks.go +++ b/beacon/params/networks.go @@ -40,36 +40,39 @@ var ( GenesisTime: 1606824023, Checkpoint: common.HexToHash(checkpointMainnet), }). - AddFork("GENESIS", 0, []byte{0, 0, 0, 0}). - AddFork("ALTAIR", 74240, []byte{1, 0, 0, 0}). - AddFork("BELLATRIX", 144896, []byte{2, 0, 0, 0}). - AddFork("CAPELLA", 194048, []byte{3, 0, 0, 0}). - AddFork("DENEB", 269568, []byte{4, 0, 0, 0}). - AddFork("ELECTRA", 364032, []byte{5, 0, 0, 0}) + AddFork("GENESIS", 0, common.FromHex("0x00000000")). + AddFork("ALTAIR", 74240, common.FromHex("0x01000000")). + AddFork("BELLATRIX", 144896, common.FromHex("0x02000000")). + AddFork("CAPELLA", 194048, common.FromHex("0x03000000")). + AddFork("DENEB", 269568, common.FromHex("0x04000000")). + AddFork("ELECTRA", 364032, common.FromHex("0x05000000")). + AddFork("FULU", 411392, common.FromHex("0x06000000")) SepoliaLightConfig = (&ChainConfig{ GenesisValidatorsRoot: common.HexToHash("0xd8ea171f3c94aea21ebc42a1ed61052acf3f9209c00e4efbaaddac09ed9b8078"), GenesisTime: 1655733600, Checkpoint: common.HexToHash(checkpointSepolia), }). - AddFork("GENESIS", 0, []byte{144, 0, 0, 105}). - AddFork("ALTAIR", 50, []byte{144, 0, 0, 112}). - AddFork("BELLATRIX", 100, []byte{144, 0, 0, 113}). - AddFork("CAPELLA", 56832, []byte{144, 0, 0, 114}). - AddFork("DENEB", 132608, []byte{144, 0, 0, 115}). - AddFork("ELECTRA", 222464, []byte{144, 0, 0, 116}) + AddFork("GENESIS", 0, common.FromHex("0x90000069")). + AddFork("ALTAIR", 50, common.FromHex("0x90000070")). + AddFork("BELLATRIX", 100, common.FromHex("0x90000071")). + AddFork("CAPELLA", 56832, common.FromHex("0x90000072")). + AddFork("DENEB", 132608, common.FromHex("0x90000073")). + AddFork("ELECTRA", 222464, common.FromHex("0x90000074")). + AddFork("FULU", 272640, common.FromHex("0x90000075")) HoleskyLightConfig = (&ChainConfig{ GenesisValidatorsRoot: common.HexToHash("0x9143aa7c615a7f7115e2b6aac319c03529df8242ae705fba9df39b79c59fa8b1"), GenesisTime: 1695902400, Checkpoint: common.HexToHash(checkpointHolesky), }). - AddFork("GENESIS", 0, []byte{1, 1, 112, 0}). - AddFork("ALTAIR", 0, []byte{2, 1, 112, 0}). - AddFork("BELLATRIX", 0, []byte{3, 1, 112, 0}). - AddFork("CAPELLA", 256, []byte{4, 1, 112, 0}). - AddFork("DENEB", 29696, []byte{5, 1, 112, 0}). - AddFork("ELECTRA", 115968, []byte{6, 1, 112, 0}) + AddFork("GENESIS", 0, common.FromHex("0x01017000")). + AddFork("ALTAIR", 0, common.FromHex("0x02017000")). + AddFork("BELLATRIX", 0, common.FromHex("0x03017000")). + AddFork("CAPELLA", 256, common.FromHex("0x04017000")). + AddFork("DENEB", 29696, common.FromHex("0x05017000")). + AddFork("ELECTRA", 115968, common.FromHex("0x06017000")). + AddFork("FULU", 165120, common.FromHex("0x07017000")) HoodiLightConfig = (&ChainConfig{ GenesisValidatorsRoot: common.HexToHash("0x212f13fc4df078b6cb7db228f1c8307566dcecf900867401a92023d7ba99cb5f"), @@ -82,5 +85,5 @@ var ( AddFork("CAPELLA", 0, common.FromHex("0x40000910")). AddFork("DENEB", 0, common.FromHex("0x50000910")). AddFork("ELECTRA", 2048, common.FromHex("0x60000910")). - AddFork("FULU", 18446744073709551615, common.FromHex("0x70000910")) + AddFork("FULU", 50688, common.FromHex("0x70000910")) ) diff --git a/beacon/types/beacon_block.go b/beacon/types/beacon_block.go index a2e31d5abf..82a0814a9f 100644 --- a/beacon/types/beacon_block.go +++ b/beacon/types/beacon_block.go @@ -52,7 +52,7 @@ func BlockFromJSON(forkName string, data []byte) (*BeaconBlock, error) { obj = new(capella.BeaconBlock) case "deneb": obj = new(deneb.BeaconBlock) - case "electra": + case "electra", "fulu": obj = new(electra.BeaconBlock) default: return nil, fmt.Errorf("unsupported fork: %s", forkName) diff --git a/beacon/types/exec_header.go b/beacon/types/exec_header.go index ae79b00841..dbf5c54b36 100644 --- a/beacon/types/exec_header.go +++ b/beacon/types/exec_header.go @@ -45,7 +45,7 @@ func ExecutionHeaderFromJSON(forkName string, data []byte) (*ExecutionHeader, er switch forkName { case "capella": obj = new(capella.ExecutionPayloadHeader) - case "deneb", "electra": // note: the payload type was not changed in electra + case "deneb", "electra", "fulu": // note: the payload type was not changed in electra/fulu obj = new(deneb.ExecutionPayloadHeader) default: return nil, fmt.Errorf("unsupported fork: %s", forkName) diff --git a/build/checksums.txt b/build/checksums.txt index 98ee3a91ef..ba80a3e201 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -5,81 +5,102 @@ # https://github.com/ethereum/execution-spec-tests/releases/download/v5.1.0 a3192784375acec7eaec492799d5c5d0c47a2909a3cc40178898e4ecd20cc416 fixtures_develop.tar.gz -# version:golang 1.25.1 +# version:golang 1.25.7 # https://go.dev/dl/ -d010c109cee94d80efe681eab46bdea491ac906bf46583c32e9f0dbb0bd1a594 go1.25.1.src.tar.gz -1d622468f767a1b9fe1e1e67bd6ce6744d04e0c68712adc689748bbeccb126bb go1.25.1.darwin-amd64.tar.gz -68deebb214f39d542e518ebb0598a406ab1b5a22bba8ec9ade9f55fb4dd94a6c go1.25.1.darwin-arm64.tar.gz -d03cdcbc9bd8baf5cf028de390478e9e2b3e4d0afe5a6582dedc19bfe6a263b2 go1.25.1.linux-386.tar.gz -7716a0d940a0f6ae8e1f3b3f4f36299dc53e31b16840dbd171254312c41ca12e go1.25.1.linux-amd64.tar.gz -65a3e34fb2126f55b34e1edfc709121660e1be2dee6bdf405fc399a63a95a87d go1.25.1.linux-arm64.tar.gz -eb949be683e82a99e9861dafd7057e31ea40b161eae6c4cd18fdc0e8c4ae6225 go1.25.1.linux-armv6l.tar.gz -be13d5479b8c75438f2efcaa8c191fba3af684b3228abc9c99c7aa8502f34424 go1.25.1.windows-386.zip -4a974de310e7ee1d523d2fcedb114ba5fa75408c98eb3652023e55ccf3fa7cab go1.25.1.windows-amd64.zip -45ab4290adbd6ee9e7f18f0d57eaa9008fdbef590882778ed93eac3c8cca06c5 go1.25.1.aix-ppc64.tar.gz -2e3c1549bed3124763774d648f291ac42611232f48320ebbd23517c909c09b81 go1.25.1.dragonfly-amd64.tar.gz -dc0198dd4ec520e13f26798def8750544edf6448d8e9c43fd2a814e4885932af go1.25.1.freebsd-386.tar.gz -c4f1a7e7b258406e6f3b677ecdbd97bbb23ff9c0d44be4eb238a07d360f69ac8 go1.25.1.freebsd-amd64.tar.gz -7772fc5ff71ed39297ec0c1599fc54e399642c9b848eac989601040923b0de9c go1.25.1.freebsd-arm.tar.gz -5bb011d5d5b6218b12189f07aa0be618ab2002662fff1ca40afba7389735c207 go1.25.1.freebsd-arm64.tar.gz -ccac716240cb049bebfafcb7eebc3758512178a4c51fc26da9cc032035d850c8 go1.25.1.freebsd-riscv64.tar.gz -cc53910ffb9fcfdd988a9fa25b5423bae1cfa01b19616be646700e1f5453b466 go1.25.1.illumos-amd64.tar.gz -efe809f923bcedab44bf7be2b3af8d182b512b1bf9c07d302e0c45d26c8f56f3 go1.25.1.linux-loong64.tar.gz -c0de33679f6ed68991dc42dc4a602e74a666e3e166c1748ee1b5d1a7ea2ffbb2 go1.25.1.linux-mips.tar.gz -c270f7b0c0bdfbcd54fef4481227c40d41bb518f9ae38ee930870f04a0a6a589 go1.25.1.linux-mips64.tar.gz -80be871ba9c944f34d1868cdf5047e1cf2e1289fe08cdb90e2453d2f0d6965ae go1.25.1.linux-mips64le.tar.gz -9f09defa9bb22ebf2cde76162f40958564e57ce5c2b3649bc063bebcbc9294c1 go1.25.1.linux-mipsle.tar.gz -2c76b7d278c1d43ad19d478ad3f0f05e7b782b64b90870701b314fa48b5f43c6 go1.25.1.linux-ppc64.tar.gz -8b0c8d3ee5b1b5c28b6bd63dc4438792012e01d03b4bf7a61d985c87edab7d1f go1.25.1.linux-ppc64le.tar.gz -22fe934a9d0c9c57275716c55b92d46ebd887cec3177c9140705efa9f84ba1e2 go1.25.1.linux-riscv64.tar.gz -9cfe517ba423f59f3738ca5c3d907c103253cffbbcc2987142f79c5de8c1bf93 go1.25.1.linux-s390x.tar.gz -6af8a08353e76205d5b743dd7a3f0126684f96f62be0a31b75daf9837e512c46 go1.25.1.netbsd-386.tar.gz -e5d534ff362edb1bd8c8e10892b6a027c4c1482454245d1529167676498684c7 go1.25.1.netbsd-amd64.tar.gz -88bcf39254fdcea6a199c1c27d787831b652427ce60851ae9e41a3d7eb477f45 go1.25.1.netbsd-arm.tar.gz -d7c2eabe1d04ee47bcaea2816fdd90dbd25d90d4dfa756faa9786c788e4f3a4e go1.25.1.netbsd-arm64.tar.gz -14a2845977eb4dde11d929858c437a043467c427db87899935e90cee04a38d72 go1.25.1.openbsd-386.tar.gz -d27ac54b38a13a09c81e67c82ac70d387037341c85c3399291c73e13e83fdd8c go1.25.1.openbsd-amd64.tar.gz -0f4ab5f02500afa4befd51fed1e8b45e4d07ca050f641cc3acc76eaa4027b2c3 go1.25.1.openbsd-arm.tar.gz -d46c3bd156843656f7f3cb0dec27ea51cd926ec3f7b80744bf8156e67c1c812f go1.25.1.openbsd-arm64.tar.gz -c550514c67f22e409be10e40eace761e2e43069f4ef086ae6e60aac736c2b679 go1.25.1.openbsd-ppc64.tar.gz -8a09a8714a2556eb13fc1f10b7ce2553fcea4971e3330fc3be0efd24aab45734 go1.25.1.openbsd-riscv64.tar.gz -b0e1fefaf0c7abd71f139a54eee9767944aff5f0bc9d69c968234804884e552f go1.25.1.plan9-386.tar.gz -e94732c94f149690aa0ab11c26090577211b4a988137cb2c03ec0b54e750402e go1.25.1.plan9-amd64.tar.gz -7eb80e9de1e817d9089a54e8c7c5c8d8ed9e5fb4d4a012fc0f18fc422a484f0c go1.25.1.plan9-arm.tar.gz -1261dfad7c4953c0ab90381bc1242dc54e394db7485c59349428d532b2273343 go1.25.1.solaris-amd64.tar.gz -04bc3c078e9e904c4d58d6ac2532a5bdd402bd36a9ff0b5949b3c5e6006a05ee go1.25.1.windows-arm64.zip +178f2832820274b43e177d32f06a3ebb0129e427dd20a5e4c88df2c1763cf10a go1.25.7.src.tar.gz +81bf2a1f20633f62d55d826d82dde3b0570cf1408a91e15781b266037299285b go1.25.7.aix-ppc64.tar.gz +bf5050a2152f4053837b886e8d9640c829dbacbc3370f913351eb0904cb706f5 go1.25.7.darwin-amd64.tar.gz +ff18369ffad05c57d5bed888b660b31385f3c913670a83ef557cdfd98ea9ae1b go1.25.7.darwin-arm64.tar.gz +c5dccd7f192dd7b305dc209fb316ac1917776d74bd8e4d532ef2772f305bf42a go1.25.7.dragonfly-amd64.tar.gz +a2de97c8ac74bf64b0ae73fe9d379e61af530e061bc7f8f825044172ffe61a8b go1.25.7.freebsd-386.tar.gz +055f9e138787dcafa81eb0314c8ff70c6dd0f6dba1e8a6957fef5d5efd1ab8fd go1.25.7.freebsd-amd64.tar.gz +60e7f7a7c990f0b9539ac8ed668155746997d404643a4eecd47b3dee1b7e710b go1.25.7.freebsd-arm.tar.gz +631e03d5fd4c526e2f499154d8c6bf4cb081afb2fff171c428722afc9539d53a go1.25.7.freebsd-arm64.tar.gz +8a264fd685823808140672812e3ad9c43f6ad59444c0dc14cdd3a1351839ddd5 go1.25.7.freebsd-riscv64.tar.gz +57c672447d906a1bcab98f2b11492d54521a791aacbb4994a25169e59cbe289a go1.25.7.illumos-amd64.tar.gz +2866517e9ca81e6a2e85a930e9b11bc8a05cfeb2fc6dc6cb2765e7fb3c14b715 go1.25.7.linux-386.tar.gz +12e6d6a191091ae27dc31f6efc630e3a3b8ba409baf3573d955b196fdf086005 go1.25.7.linux-amd64.tar.gz +ba611a53534135a81067240eff9508cd7e256c560edd5d8c2fef54f083c07129 go1.25.7.linux-arm64.tar.gz +1ba07e0eb86b839e72467f4b5c7a5597d07f30bcf5563c951410454f7cda5266 go1.25.7.linux-armv6l.tar.gz +775753fc5952a334c415f08768df2f0b73a3228a16e8f5f63d545daacb4e3357 go1.25.7.linux-loong64.tar.gz +1a023bb367c5fbb4c637a2f6dc23ff17c6591ad929ce16ea88c74d857153b307 go1.25.7.linux-mips.tar.gz +a8e97223d8aa6fdfd45f132a4784d2f536bbac5f3d63a24b63d33b6bfe1549af go1.25.7.linux-mips64.tar.gz +eb9edb6223330d5e20275667c65dea076b064c08e595fe4eba5d7d6055cfaccf go1.25.7.linux-mips64le.tar.gz +9c1e693552a5f9bb9e0012d1c5e01456ecefbc59bef53a77305222ce10aba368 go1.25.7.linux-mipsle.tar.gz +28a788798e7329acbbc0ac2caa5e4368b1e5ede646cc24429c991214cfb45c63 go1.25.7.linux-ppc64.tar.gz +42124c0edc92464e2b37b2d7fcd3658f0c47ebd6a098732415a522be8cb88e3f go1.25.7.linux-ppc64le.tar.gz +88d59c6893c8425875d6eef8e3434bc2fa2552e5ad4c058c6cd8cd710a0301c8 go1.25.7.linux-riscv64.tar.gz +c6b77facf666dc68195ecab05dbf0ebb4e755b2a8b7734c759880557f1c29b0c go1.25.7.linux-s390x.tar.gz +f14c184d9ade0ee04c7735d4071257b90896ecbde1b32adae84135f055e6399b go1.25.7.netbsd-386.tar.gz +7e7389e404dca1088c31f0fc07f1dd60891d7182bcd621469c14f7e79eceb3ff go1.25.7.netbsd-amd64.tar.gz +70388bb3ef2f03dbf1357e9056bd09034a67e018262557354f8cf549766b3f9d go1.25.7.netbsd-arm.tar.gz +8c1cda9d25bfc9b18d24d5f95fc23949dd3ff99fa408a6cfa40e2cf12b07e362 go1.25.7.netbsd-arm64.tar.gz +42f0d1bfbe39b8401cccb84dd66b30795b97bfc9620dfdc17c5cd4fcf6495cb0 go1.25.7.openbsd-386.tar.gz +e514879c0a28bc32123cd52c4c093de912477fe83f36a6d07517d066ef55391a go1.25.7.openbsd-amd64.tar.gz +8cd22530695a0218232bf7efea8f162df1697a3106942ac4129b8c3de39ce4ef go1.25.7.openbsd-arm.tar.gz +938720f6ebc0d1c53d7840321d3a31f29fd02496e84a6538f442a9311dc1cc9a go1.25.7.openbsd-arm64.tar.gz +a4c378b73b98f89a3596c2ef51aabbb28783d9ca29f7e317d8ca07939660ce6f go1.25.7.openbsd-ppc64.tar.gz +937b58734fbeaa8c7941a0e4285e7e84b7885396e8d11c23f9ab1a8ff10ff20e go1.25.7.openbsd-riscv64.tar.gz +61a093c8c5244916f25740316386bb9f141545dcf01b06a79d1c78ece488403e go1.25.7.plan9-386.tar.gz +7fc8f6689c9de8ccb7689d2278035fa83c2d601409101840df6ddfe09ba58699 go1.25.7.plan9-amd64.tar.gz +9661dff8eaeeb62f1c3aadbc5ff189a2e6744e1ec885e32dbcb438f58a34def5 go1.25.7.plan9-arm.tar.gz +28ecba0e1d7950c8b29a4a04962dd49c3bf5221f55a44f17d98f369f82859cf4 go1.25.7.solaris-amd64.tar.gz +baa6b488291801642fa620026169e38bec2da2ac187cd3ae2145721cf826bbc3 go1.25.7.windows-386.zip +c75e5f4ff62d085cc0017be3ad19d5536f46825fa05db06ec468941f847e3228 go1.25.7.windows-amd64.zip +807033f85931bc4a589ca8497535dcbeb1f30d506e47fa200f5f04c4a71c3d9f go1.25.7.windows-arm64.zip -# version:golangci 2.4.0 +# version:golangci 2.10.1 # https://github.com/golangci/golangci-lint/releases/ -# https://github.com/golangci/golangci-lint/releases/download/v2.4.0/ -7904ce63f79db44934939cf7a063086ea0ea98e9b19eba0a9d52ccdd0d21951c golangci-lint-2.4.0-darwin-amd64.tar.gz -cd4dd53fa09b6646baff5fd22b8c64d91db02c21c7496df27992d75d34feec59 golangci-lint-2.4.0-darwin-arm64.tar.gz -d58f426ebe14cc257e81562b4bf37a488ffb4ffbbb3ec73041eb3b38bb25c0e1 golangci-lint-2.4.0-freebsd-386.tar.gz -6ec4a6177fc6c0dd541fbcb3a7612845266d020d35cc6fa92959220cdf64ca39 golangci-lint-2.4.0-freebsd-amd64.tar.gz -4d473e3e71c01feaa915a0604fb35758b41284fb976cdeac3f842118d9ee7e17 golangci-lint-2.4.0-freebsd-armv6.tar.gz -58727746c6530801a3f9a702a5945556a5eb7e88809222536dd9f9d54cafaeff golangci-lint-2.4.0-freebsd-armv7.tar.gz -fbf28c662760e24c32f82f8d16dffdb4a82de7726a52ba1fad94f890c22997ea golangci-lint-2.4.0-illumos-amd64.tar.gz -a15a000a8981ef665e971e0f67e2acda9066a9e37a59344393b7351d8fb49c81 golangci-lint-2.4.0-linux-386.tar.gz -fae792524c04424c0ac369f5b8076f04b45cf29fc945a370e55d369a8dc11840 golangci-lint-2.4.0-linux-amd64.tar.gz -70ac11f55b80ec78fd3a879249cc9255121b8dfd7f7ed4fc46ed137f4abf17e7 golangci-lint-2.4.0-linux-arm64.tar.gz -4acdc40e5cebe99e4e7ced358a05b2e71789f409b41cb4f39bbb86ccfa14b1dc golangci-lint-2.4.0-linux-armv6.tar.gz -2a68749568fa22b4a97cb88dbea655595563c795076536aa6c087f7968784bf3 golangci-lint-2.4.0-linux-armv7.tar.gz -9e3369afb023711036dcb0b4f45c9fe2792af962fa1df050c9f6ac101a6c5d73 golangci-lint-2.4.0-linux-loong64.tar.gz -bb9143d6329be2c4dbfffef9564078e7da7d88e7dde6c829b6263d98e072229e golangci-lint-2.4.0-linux-mips64.tar.gz -5ad1765b40d56cd04d4afd805b3ba6f4bfd9b36181da93c31e9b17e483d8608d golangci-lint-2.4.0-linux-mips64le.tar.gz -918936fb9c0d5ba96bef03cf4348b03938634cfcced49be1e9bb29cb5094fa73 golangci-lint-2.4.0-linux-ppc64le.tar.gz -f7474c638e1fb67ebbdc654b55ca0125377ea0bc88e8fee8d964a4f24eacf828 golangci-lint-2.4.0-linux-riscv64.tar.gz -b617a9543997c8bfceaffa88a75d4e595030c6add69fba800c1e4d8f5fe253dd golangci-lint-2.4.0-linux-s390x.tar.gz -7db027b03a9ba328f795215b04f594036837bc7dd0dd7cd16776b02a6167981c golangci-lint-2.4.0-netbsd-386.tar.gz -52d8f9393f4313df0a62b752c37775e3af0b818e43e8dd28954351542d7c60bc golangci-lint-2.4.0-netbsd-amd64.tar.gz -5c0086027fb5a4af3829e530c8115db4b35d11afe1914322eef528eb8cd38c69 golangci-lint-2.4.0-netbsd-arm64.tar.gz -6b779d6ed1aed87cefe195cc11759902b97a76551b593312c6833f2635a3488f golangci-lint-2.4.0-netbsd-armv6.tar.gz -f00d1f4b7ec3468a0f9fffd0d9ea036248b029b7621cbc9a59c449ef94356d09 golangci-lint-2.4.0-netbsd-armv7.tar.gz -3ce671b0b42b58e35066493aab75a7e2826c9e079988f1ba5d814a4029faaf87 golangci-lint-2.4.0-windows-386.zip -003112f7a56746feaabf20b744054bf9acdf900c9e77176383623c4b1d76aaa9 golangci-lint-2.4.0-windows-amd64.zip -dc0c2092af5d47fc2cd31a1dfe7b4c7e765fab22de98bd21ef2ffcc53ad9f54f golangci-lint-2.4.0-windows-arm64.zip -0263d23e20a260cb1592d35e12a388f99efe2c51b3611fdc66fbd9db1fce664d golangci-lint-2.4.0-windows-armv6.zip -9403c03bf648e6313036e0273149d44bad1b9ad53889b6d00e4ccb842ba3c058 golangci-lint-2.4.0-windows-armv7.zip +# https://github.com/golangci/golangci-lint/releases/download/v2.10.1 +66fb0da81b8033b477f97eea420d4b46b230ca172b8bb87c6610109f3772b6b6 golangci-lint-2.10.1-darwin-amd64.tar.gz +03bfadf67e52b441b7ec21305e501c717df93c959836d66c7f97312654acb297 golangci-lint-2.10.1-darwin-arm64.tar.gz +c9a44658ccc8f7b8dbbd4ae6020ba91c1a5d3987f4d91ced0f7d2bea013e57ca golangci-lint-2.10.1-freebsd-386.tar.gz +a513c5cb4e0f5bd5767001af9d5e97e7868cfc2d9c46739a4df93e713cfb24af golangci-lint-2.10.1-freebsd-amd64.tar.gz +2ef38eefc4b5cee2febacb75a30579526e5656c16338a921d80e59a8e87d4425 golangci-lint-2.10.1-freebsd-arm64.tar.gz +8fea6766318b4829e766bbe325f10191d75297dcc44ae35bf374816037878e38 golangci-lint-2.10.1-freebsd-armv6.tar.gz +30b629870574d6254f3e8804e5a74b34f98e1263c9d55465830d739c88b862ed golangci-lint-2.10.1-freebsd-armv7.tar.gz +c0db839f866ce80b1b6c96167aa101cfe50d9c936f42d942a3c1cbdc1801af68 golangci-lint-2.10.1-illumos-amd64.tar.gz +280eb56636e9175f671cd7b755d7d67f628ae2ed00a164d1e443c43c112034e5 golangci-lint-2.10.1-linux-386.deb +065a7d99da61dc7dfbfef2e2d7053dd3fa6672598f2747117aa4bb5f45e7df7f golangci-lint-2.10.1-linux-386.rpm +a55918c03bb413b2662287653ab2ae2fef4e37428b247dad6348724adde9d770 golangci-lint-2.10.1-linux-386.tar.gz +8aa9b3aa14f39745eeb7fc7ff50bcac683e785397d1e4bc9afd2184b12c4ce86 golangci-lint-2.10.1-linux-amd64.deb +62a111688e9e305032334a2cbc84f4d971b64bb3bffc99d3f80081d57fb25e32 golangci-lint-2.10.1-linux-amd64.rpm +dfa775874cf0561b404a02a8f4481fc69b28091da95aa697259820d429b09c99 golangci-lint-2.10.1-linux-amd64.tar.gz +b3f36937e8ea1660739dc0f5c892ea59c9c21ed4e75a91a25957c561f7f79a55 golangci-lint-2.10.1-linux-arm64.deb +36d50314d53683b1f1a2a6cedfb5a9468451b481c64ab9e97a8e843ea088074d golangci-lint-2.10.1-linux-arm64.rpm +6652b42ae02915eb2f9cb2a2e0cac99514c8eded8388d88ae3e06e1a52c00de8 golangci-lint-2.10.1-linux-arm64.tar.gz +a32d8d318e803496812dd3461f250e52ccc7f53c47b95ce404a9cf55778ceb6a golangci-lint-2.10.1-linux-armv6.deb +41d065f4c8ea165a1531abea644988ee2e973e4f0b49f9725ed3b979dac45112 golangci-lint-2.10.1-linux-armv6.rpm +59159a4df03aabbde69d15c7b7b3df143363cbb41f4bd4b200caffb8e34fb734 golangci-lint-2.10.1-linux-armv6.tar.gz +b2e8ec0e050a1e2251dfe1561434999d202f5a3f9fa47ce94378b0fd1662ea5a golangci-lint-2.10.1-linux-armv7.deb +28c9331429a497da27e9c77846063bd0e8275e878ffedb4eb9e9f21d24771cc0 golangci-lint-2.10.1-linux-armv7.rpm +818f33e95b273e3769284b25563b51ef6a294e9e25acf140fda5830c075a1a59 golangci-lint-2.10.1-linux-armv7.tar.gz +6b6b85ed4b7c27f51097dd681523000409dde835e86e6e314e87be4bb013e2ab golangci-lint-2.10.1-linux-loong64.deb +94050a0cf06169e2ae44afb307dcaafa7d7c3b38c0c23b5652cf9cb60f0c337f golangci-lint-2.10.1-linux-loong64.rpm +25820300fccb8c961c1cdcb1f77928040c079e04c43a3a5ceb34b1cb4a1c5c8d golangci-lint-2.10.1-linux-loong64.tar.gz +98bf39d10139fdcaa37f94950e9bbb8888660ae468847ae0bf1cb5bf67c1f68b golangci-lint-2.10.1-linux-mips64.deb +df3ce5f03808dcceaa8b683d1d06e95c885f09b59dc8e15deb840fbe2b3e3299 golangci-lint-2.10.1-linux-mips64.rpm +972508dda523067e6e6a1c8e6609d63bc7c4153819c11b947d439235cf17bac2 golangci-lint-2.10.1-linux-mips64.tar.gz +1d37f2919e183b5bf8b1777ed8c4b163d3b491d0158355a7999d647655cbbeb6 golangci-lint-2.10.1-linux-mips64le.deb +e341d031002cd09a416329ed40f674231051a38544b8f94deb2d1708ce1f4a6f golangci-lint-2.10.1-linux-mips64le.rpm +393560122b9cb5538df0c357d30eb27b6ee563533fbb9b138c8db4fd264002af golangci-lint-2.10.1-linux-mips64le.tar.gz +21ca46b6a96442e8957677a3ca059c6b93674a68a01b1c71f4e5df0ea2e96d19 golangci-lint-2.10.1-linux-ppc64le.deb +57fe0cbca0a9bbdf1547c5e8aa7d278e6896b438d72a541bae6bc62c38b43d1e golangci-lint-2.10.1-linux-ppc64le.rpm +e2883db9fa51584e5e203c64456f29993550a7faadc84e3faccdb48f0669992e golangci-lint-2.10.1-linux-ppc64le.tar.gz +aa6da0e98ab0ba3bb7582e112174c349907d5edfeff90a551dca3c6eecf92fc0 golangci-lint-2.10.1-linux-riscv64.deb +3c68d76cd884a7aad206223a980b9c20bb9ea74b560fa27ed02baf2389189234 golangci-lint-2.10.1-linux-riscv64.rpm +3bca11bfac4197205639cbd4676a5415054e629ac6c12ea10fcbe33ef852d9c3 golangci-lint-2.10.1-linux-riscv64.tar.gz +0c6aed2ce49db2586adbac72c80d871f06feb1caf4c0763a5ca98fec809a8f0b golangci-lint-2.10.1-linux-s390x.deb +16c285adfe1061d69dd8e503be69f87c7202857c6f4add74ac02e3571158fbec golangci-lint-2.10.1-linux-s390x.rpm +21011ad368eb04f024201b832095c6b5f96d0888de194cca5bfe4d9307d6364b golangci-lint-2.10.1-linux-s390x.tar.gz +7b5191e77a70485918712e31ed55159956323e4911bab1b67569c9d86e1b75eb golangci-lint-2.10.1-netbsd-386.tar.gz +07801fd38d293ebad10826f8285525a39ea91ce5ddad77d05bfa90bda9c884a9 golangci-lint-2.10.1-netbsd-amd64.tar.gz +7e7219d71c1bf33b98c328c93dc0560706dd896a1c43c44696e5222fc9d7446e golangci-lint-2.10.1-netbsd-arm64.tar.gz +92fbc90b9eec0e572269b0f5492a2895c426b086a68372fde49b7e4d4020863e golangci-lint-2.10.1-netbsd-armv6.tar.gz +f67b3ae1f47caeefa507a4ebb0c8336958a19011fe48766443212030f75d004b golangci-lint-2.10.1-netbsd-armv7.tar.gz +a40bc091c10cea84eaee1a90b84b65f5e8652113b0a600bb099e4e4d9d7caddb golangci-lint-2.10.1-windows-386.zip +c60c87695e79db8e320f0e5be885059859de52bb5ee5f11be5577828570bc2a3 golangci-lint-2.10.1-windows-amd64.zip +636ab790c8dcea8034aa34aba6031ca3893d68f7eda000460ab534341fadbab1 golangci-lint-2.10.1-windows-arm64.zip # This is the builder on PPA that will build Go itself (inception-y), don't modify! # diff --git a/build/ci.go b/build/ci.go index 59c948acb3..173288bcdc 100644 --- a/build/ci.go +++ b/build/ci.go @@ -107,6 +107,22 @@ var ( Tags: "ziren", Env: map[string]string{"GOMIPS": "softfloat", "CGO_ENABLED": "0"}, }, + { + Name: "womir", + GOOS: "wasip1", + GOARCH: "wasm", + Tags: "womir", + }, + { + Name: "wasm-js", + GOOS: "js", + GOARCH: "wasm", + }, + { + Name: "wasm-wasi", + GOOS: "wasip1", + GOARCH: "wasm", + }, { Name: "example", Tags: "example", @@ -151,13 +167,11 @@ var ( // Distros for which packages are created debDistros = []string{ - "xenial", // 16.04, EOL: 04/2026 - "bionic", // 18.04, EOL: 04/2028 - "focal", // 20.04, EOL: 04/2030 - "jammy", // 22.04, EOL: 04/2032 - "noble", // 24.04, EOL: 04/2034 - "oracular", // 24.10, EOL: 07/2025 - "plucky", // 25.04, EOL: 01/2026 + "xenial", // 16.04, EOL: 04/2026 + "bionic", // 18.04, EOL: 04/2028 + "focal", // 20.04, EOL: 04/2030 + "jammy", // 22.04, EOL: 04/2032 + "noble", // 24.04, EOL: 04/2034 } // This is where the tests should be unpacked. @@ -295,7 +309,7 @@ func doInstallKeeper(cmdline []string) { args := slices.Clone(gobuild.Args) args = append(args, "-o", executablePath(outputName)) args = append(args, ".") - build.MustRun(&exec.Cmd{Path: gobuild.Path, Args: args, Env: gobuild.Env}) + build.MustRun(&exec.Cmd{Path: gobuild.Path, Args: args, Env: gobuild.Env, Dir: gobuild.Dir}) } } @@ -331,6 +345,10 @@ func buildFlags(env build.Environment, staticLinking bool, buildTags []string) ( } ld = append(ld, "-extldflags", "'"+strings.Join(extld, " ")+"'") } + // TODO(gballet): revisit after the input api has been defined + if runtime.GOARCH == "wasm" { + ld = append(ld, "-gcflags=all=-d=softfloat") + } if len(ld) > 0 { flags = append(flags, "-ldflags", strings.Join(ld, " ")) } @@ -446,9 +464,14 @@ func doCheckGenerate() { ) pathList := []string{filepath.Join(protocPath, "bin"), protocGenGoPath, os.Getenv("PATH")} + excludes := []string{"tests/testdata", "build/cache", ".git"} + for i := range excludes { + excludes[i] = filepath.FromSlash(excludes[i]) + } + for _, mod := range goModules { // Compute the origin hashes of all the files - hashes, err := build.HashFolder(mod, []string{"tests/testdata", "build/cache", ".git"}) + hashes, err := build.HashFolder(mod, excludes) if err != nil { log.Fatal("Error computing hashes", "err", err) } @@ -458,7 +481,7 @@ func doCheckGenerate() { c.Dir = mod build.MustRun(c) // Check if generate file hashes have changed - generated, err := build.HashFolder(mod, []string{"tests/testdata", "build/cache", ".git"}) + generated, err := build.HashFolder(mod, excludes) if err != nil { log.Fatalf("Error re-computing hashes: %v", err) } @@ -1177,7 +1200,7 @@ func doWindowsInstaller(cmdline []string) { var ( arch = flag.String("arch", runtime.GOARCH, "Architecture for cross build packaging") signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. WINDOWS_SIGNING_KEY)`) - signify = flag.String("signify key", "", `Environment variable holding the signify signing key (e.g. WINDOWS_SIGNIFY_KEY)`) + signify = flag.String("signify", "", `Environment variable holding the signify signing key (e.g. WINDOWS_SIGNIFY_KEY)`) upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) ) diff --git a/circle.yml b/circle.yml deleted file mode 100644 index 39ff5d83c6..0000000000 --- a/circle.yml +++ /dev/null @@ -1,32 +0,0 @@ -machine: - services: - - docker - -dependencies: - cache_directories: - - "~/.ethash" # Cache the ethash DAG generated by hive for consecutive builds - - "~/.docker" # Cache all docker images manually to avoid lengthy rebuilds - override: - # Restore all previously cached docker images - - mkdir -p ~/.docker - - for img in `ls ~/.docker`; do docker load -i ~/.docker/$img; done - - # Pull in and hive, restore cached ethash DAGs and do a dry run - - go get -u github.com/karalabe/hive - - (cd ~/.go_workspace/src/github.com/karalabe/hive && mkdir -p workspace/ethash/ ~/.ethash) - - (cd ~/.go_workspace/src/github.com/karalabe/hive && cp -r ~/.ethash/. workspace/ethash/) - - (cd ~/.go_workspace/src/github.com/karalabe/hive && hive --docker-noshell --client=NONE --test=. --sim=. --loglevel=6) - - # Cache all the docker images and the ethash DAGs - - for img in `docker images | grep -v "^" | tail -n +2 | awk '{print $1}'`; do docker save $img > ~/.docker/`echo $img | tr '/' ':'`.tar; done - - cp -r ~/.go_workspace/src/github.com/karalabe/hive/workspace/ethash/. ~/.ethash - -test: - override: - # Build Geth and move into a known folder - - make geth - - cp ./build/bin/geth $HOME/geth - - # Run hive and move all generated logs into the public artifacts folder - - (cd ~/.go_workspace/src/github.com/karalabe/hive && hive --docker-noshell --client=go-ethereum:local --override=$HOME/geth --test=. --sim=.) - - cp -r ~/.go_workspace/src/github.com/karalabe/hive/workspace/logs/* $CIRCLE_ARTIFACTS diff --git a/cmd/devp2p/internal/ethtest/conn.go b/cmd/devp2p/internal/ethtest/conn.go index 5182d71ce1..98baba81a4 100644 --- a/cmd/devp2p/internal/ethtest/conn.go +++ b/cmd/devp2p/internal/ethtest/conn.go @@ -155,7 +155,7 @@ func (c *Conn) ReadEth() (any, error) { var msg any switch int(code) { case eth.StatusMsg: - msg = new(eth.StatusPacket69) + msg = new(eth.StatusPacket) case eth.GetBlockHeadersMsg: msg = new(eth.GetBlockHeadersPacket) case eth.BlockHeadersMsg: @@ -164,10 +164,6 @@ func (c *Conn) ReadEth() (any, error) { msg = new(eth.GetBlockBodiesPacket) case eth.BlockBodiesMsg: msg = new(eth.BlockBodiesPacket) - case eth.NewBlockMsg: - msg = new(eth.NewBlockPacket) - case eth.NewBlockHashesMsg: - msg = new(eth.NewBlockHashesPacket) case eth.TransactionsMsg: msg = new(eth.TransactionsPacket) case eth.NewPooledTransactionHashesMsg: @@ -229,7 +225,7 @@ func (c *Conn) ReadSnap() (any, error) { } // dialAndPeer creates a peer connection and runs the handshake. -func (s *Suite) dialAndPeer(status *eth.StatusPacket69) (*Conn, error) { +func (s *Suite) dialAndPeer(status *eth.StatusPacket) (*Conn, error) { c, err := s.dial() if err != nil { return nil, err @@ -242,7 +238,7 @@ func (s *Suite) dialAndPeer(status *eth.StatusPacket69) (*Conn, error) { // peer performs both the protocol handshake and the status message // exchange with the node in order to peer with it. -func (c *Conn) peer(chain *Chain, status *eth.StatusPacket69) error { +func (c *Conn) peer(chain *Chain, status *eth.StatusPacket) error { if err := c.handshake(); err != nil { return fmt.Errorf("handshake failed: %v", err) } @@ -315,7 +311,7 @@ func (c *Conn) negotiateEthProtocol(caps []p2p.Cap) { } // statusExchange performs a `Status` message exchange with the given node. -func (c *Conn) statusExchange(chain *Chain, status *eth.StatusPacket69) error { +func (c *Conn) statusExchange(chain *Chain, status *eth.StatusPacket) error { loop: for { code, data, err := c.Read() @@ -324,7 +320,7 @@ loop: } switch code { case eth.StatusMsg + protoOffset(ethProto): - msg := new(eth.StatusPacket69) + msg := new(eth.StatusPacket) if err := rlp.DecodeBytes(data, &msg); err != nil { return fmt.Errorf("error decoding status packet: %w", err) } @@ -363,7 +359,7 @@ loop: } if status == nil { // default status message - status = ð.StatusPacket69{ + status = ð.StatusPacket{ ProtocolVersion: uint32(c.negotiatedProtoVersion), NetworkID: chain.config.ChainID.Uint64(), Genesis: chain.blocks[0].Hash(), diff --git a/cmd/devp2p/internal/ethtest/protocol.go b/cmd/devp2p/internal/ethtest/protocol.go index af76082318..a21d1ca7a1 100644 --- a/cmd/devp2p/internal/ethtest/protocol.go +++ b/cmd/devp2p/internal/ethtest/protocol.go @@ -86,9 +86,3 @@ func protoOffset(proto Proto) uint64 { panic("unhandled protocol") } } - -// msgTypePtr is the constraint for protocol message types. -type msgTypePtr[U any] interface { - *U - Kind() byte -} diff --git a/cmd/devp2p/internal/ethtest/snap.go b/cmd/devp2p/internal/ethtest/snap.go index f4fce0931f..7c1ca70cc0 100644 --- a/cmd/devp2p/internal/ethtest/snap.go +++ b/cmd/devp2p/internal/ethtest/snap.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/protocols/snap" "github.com/ethereum/go-ethereum/internal/utesting" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" ) @@ -937,10 +938,14 @@ func (s *Suite) snapGetTrieNodes(t *utesting.T, tc *trieNodesTest) error { } // write0 request + paths, err := rlp.EncodeToRawList(tc.paths) + if err != nil { + panic(err) + } req := &snap.GetTrieNodesPacket{ ID: uint64(rand.Int63()), Root: tc.root, - Paths: tc.paths, + Paths: paths, Bytes: tc.nBytes, } msg, err := conn.snapRequest(snap.GetTrieNodesMsg, req) diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go index c23360bf82..7560c13137 100644 --- a/cmd/devp2p/internal/ethtest/suite.go +++ b/cmd/devp2p/internal/ethtest/suite.go @@ -34,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/internal/utesting" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/rlp" "github.com/holiman/uint256" ) @@ -151,7 +152,11 @@ func (s *Suite) TestGetBlockHeaders(t *utesting.T) { if err != nil { t.Fatalf("failed to get headers for given request: %v", err) } - if !headersMatch(expected, headers.BlockHeadersRequest) { + received, err := headers.List.Items() + if err != nil { + t.Fatalf("invalid headers received: %v", err) + } + if !headersMatch(expected, received) { t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected, headers) } } @@ -237,7 +242,7 @@ concurrently, with different request IDs.`) // Wait for responses. // Note they can arrive in either order. - resp, err := collectResponses(conn, 2, func(msg *eth.BlockHeadersPacket) uint64 { + resp, err := collectHeaderResponses(conn, 2, func(msg *eth.BlockHeadersPacket) uint64 { if msg.RequestId != 111 && msg.RequestId != 222 { t.Fatalf("response with unknown request ID: %v", msg.RequestId) } @@ -248,17 +253,11 @@ concurrently, with different request IDs.`) } // Check if headers match. - resp1 := resp[111] - if expected, err := s.chain.GetHeaders(req1); err != nil { - t.Fatalf("failed to get expected headers for request 1: %v", err) - } else if !headersMatch(expected, resp1.BlockHeadersRequest) { - t.Fatalf("header mismatch for request ID %v: \nexpected %v \ngot %v", 111, expected, resp1) - } - resp2 := resp[222] - if expected, err := s.chain.GetHeaders(req2); err != nil { - t.Fatalf("failed to get expected headers for request 2: %v", err) - } else if !headersMatch(expected, resp2.BlockHeadersRequest) { - t.Fatalf("header mismatch for request ID %v: \nexpected %v \ngot %v", 222, expected, resp2) + if err := s.checkHeadersAgainstChain(req1, resp[111]); err != nil { + t.Fatal(err) + } + if err := s.checkHeadersAgainstChain(req2, resp[222]); err != nil { + t.Fatal(err) } } @@ -303,8 +302,8 @@ same request ID. The node should handle the request by responding to both reques // Wait for the responses. They can arrive in either order, and we can't tell them // apart by their request ID, so use the number of headers instead. - resp, err := collectResponses(conn, 2, func(msg *eth.BlockHeadersPacket) uint64 { - id := uint64(len(msg.BlockHeadersRequest)) + resp, err := collectHeaderResponses(conn, 2, func(msg *eth.BlockHeadersPacket) uint64 { + id := uint64(msg.List.Len()) if id != 2 && id != 3 { t.Fatalf("invalid number of headers in response: %d", id) } @@ -315,26 +314,35 @@ same request ID. The node should handle the request by responding to both reques } // Check if headers match. - resp1 := resp[2] - if expected, err := s.chain.GetHeaders(request1); err != nil { - t.Fatalf("failed to get expected headers for request 1: %v", err) - } else if !headersMatch(expected, resp1.BlockHeadersRequest) { - t.Fatalf("headers mismatch: \nexpected %v \ngot %v", expected, resp1) - } - resp2 := resp[3] - if expected, err := s.chain.GetHeaders(request2); err != nil { - t.Fatalf("failed to get expected headers for request 2: %v", err) - } else if !headersMatch(expected, resp2.BlockHeadersRequest) { - t.Fatalf("headers mismatch: \nexpected %v \ngot %v", expected, resp2) + if err := s.checkHeadersAgainstChain(request1, resp[2]); err != nil { + t.Fatal(err) + } + if err := s.checkHeadersAgainstChain(request2, resp[3]); err != nil { + t.Fatal(err) + } +} + +func (s *Suite) checkHeadersAgainstChain(req *eth.GetBlockHeadersPacket, resp *eth.BlockHeadersPacket) error { + received2, err := resp.List.Items() + if err != nil { + return fmt.Errorf("invalid headers in response with request ID %v (%d items): %v", resp.RequestId, resp.List.Len(), err) + } + if expected, err := s.chain.GetHeaders(req); err != nil { + return fmt.Errorf("test chain failed to get expected headers for request: %v", err) + } else if !headersMatch(expected, received2) { + return fmt.Errorf("header mismatch for request ID %v (%d items): \nexpected %v \ngot %v", resp.RequestId, resp.List.Len(), expected, resp) } + return nil } // collectResponses waits for n messages of type T on the given connection. // The messsages are collected according to the 'identity' function. -func collectResponses[T any, P msgTypePtr[T]](conn *Conn, n int, identity func(P) uint64) (map[uint64]P, error) { - resp := make(map[uint64]P, n) +// +// This function is written in a generic way to handle +func collectHeaderResponses(conn *Conn, n int, identity func(*eth.BlockHeadersPacket) uint64) (map[uint64]*eth.BlockHeadersPacket, error) { + resp := make(map[uint64]*eth.BlockHeadersPacket, n) for range n { - r := new(T) + r := new(eth.BlockHeadersPacket) if err := conn.ReadMsg(ethProto, eth.BlockHeadersMsg, r); err != nil { return resp, fmt.Errorf("read error: %v", err) } @@ -373,10 +381,8 @@ and expects a response.`) if got, want := headers.RequestId, req.RequestId; got != want { t.Fatalf("unexpected request id") } - if expected, err := s.chain.GetHeaders(req); err != nil { - t.Fatalf("failed to get expected block headers: %v", err) - } else if !headersMatch(expected, headers.BlockHeadersRequest) { - t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected, headers) + if err := s.checkHeadersAgainstChain(req, headers); err != nil { + t.Fatal(err) } } @@ -407,9 +413,8 @@ func (s *Suite) TestGetBlockBodies(t *utesting.T) { if got, want := resp.RequestId, req.RequestId; got != want { t.Fatalf("unexpected request id in respond", got, want) } - bodies := resp.BlockBodiesResponse - if len(bodies) != len(req.GetBlockBodiesRequest) { - t.Fatalf("wrong bodies in response: expected %d bodies, got %d", len(req.GetBlockBodiesRequest), len(bodies)) + if resp.List.Len() != len(req.GetBlockBodiesRequest) { + t.Fatalf("wrong bodies in response: expected %d bodies, got %d", len(req.GetBlockBodiesRequest), resp.List.Len()) } } @@ -433,7 +438,7 @@ func (s *Suite) TestGetReceipts(t *utesting.T) { } } - // Create block bodies request. + // Create receipts request. req := ð.GetReceiptsPacket{ RequestId: 66, GetReceiptsRequest: (eth.GetReceiptsRequest)(hashes), @@ -442,15 +447,15 @@ func (s *Suite) TestGetReceipts(t *utesting.T) { t.Fatalf("could not write to connection: %v", err) } // Wait for response. - resp := new(eth.ReceiptsPacket[*eth.ReceiptList69]) + resp := new(eth.ReceiptsPacket) if err := conn.ReadMsg(ethProto, eth.ReceiptsMsg, &resp); err != nil { t.Fatalf("error reading block bodies msg: %v", err) } if got, want := resp.RequestId, req.RequestId; got != want { t.Fatalf("unexpected request id in respond", got, want) } - if len(resp.List) != len(req.GetReceiptsRequest) { - t.Fatalf("wrong bodies in response: expected %d bodies, got %d", len(req.GetReceiptsRequest), len(resp.List)) + if resp.List.Len() != len(req.GetReceiptsRequest) { + t.Fatalf("wrong receipts in response: expected %d receipts, got %d", len(req.GetReceiptsRequest), resp.List.Len()) } } @@ -804,7 +809,11 @@ on another peer connection using GetPooledTransactions.`) if got, want := msg.RequestId, req.RequestId; got != want { t.Fatalf("unexpected request id in response: got %d, want %d", got, want) } - for _, got := range msg.PooledTransactionsResponse { + responseTxs, err := msg.List.Items() + if err != nil { + t.Fatalf("invalid transactions in response: %v", err) + } + for _, got := range responseTxs { if _, exists := set[got.Hash()]; !exists { t.Fatalf("unexpected tx received: %v", got.Hash()) } @@ -976,7 +985,9 @@ func (s *Suite) TestBlobViolations(t *utesting.T) { if err := conn.ReadMsg(ethProto, eth.GetPooledTransactionsMsg, req); err != nil { t.Fatalf("reading pooled tx request failed: %v", err) } - resp := eth.PooledTransactionsPacket{RequestId: req.RequestId, PooledTransactionsResponse: test.resp} + + encTxs, _ := rlp.EncodeToRawList(test.resp) + resp := eth.PooledTransactionsPacket{RequestId: req.RequestId, List: encTxs} if err := conn.Write(ethProto, eth.PooledTransactionsMsg, resp); err != nil { t.Fatalf("writing pooled tx response failed: %v", err) } @@ -1104,7 +1115,8 @@ func (s *Suite) testBadBlobTx(t *utesting.T, tx *types.Transaction, badTx *types // the good peer is connected, and has announced the tx. // proceed to send the incorrect one from the bad peer. - resp := eth.PooledTransactionsPacket{RequestId: req.RequestId, PooledTransactionsResponse: eth.PooledTransactionsResponse(types.Transactions{badTx})} + encTxs, _ := rlp.EncodeToRawList([]*types.Transaction{badTx}) + resp := eth.PooledTransactionsPacket{RequestId: req.RequestId, List: encTxs} if err := conn.Write(ethProto, eth.PooledTransactionsMsg, resp); err != nil { errc <- fmt.Errorf("writing pooled tx response failed: %v", err) return @@ -1164,7 +1176,8 @@ func (s *Suite) testBadBlobTx(t *utesting.T, tx *types.Transaction, badTx *types return } - resp := eth.PooledTransactionsPacket{RequestId: req.RequestId, PooledTransactionsResponse: eth.PooledTransactionsResponse(types.Transactions{tx})} + encTxs, _ := rlp.EncodeToRawList([]*types.Transaction{tx}) + resp := eth.PooledTransactionsPacket{RequestId: req.RequestId, List: encTxs} if err := conn.Write(ethProto, eth.PooledTransactionsMsg, resp); err != nil { errc <- fmt.Errorf("writing pooled tx response failed: %v", err) return diff --git a/cmd/devp2p/internal/ethtest/transaction.go b/cmd/devp2p/internal/ethtest/transaction.go index cbbbbce8d9..8ce26f3e1a 100644 --- a/cmd/devp2p/internal/ethtest/transaction.go +++ b/cmd/devp2p/internal/ethtest/transaction.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/internal/utesting" + "github.com/ethereum/go-ethereum/rlp" ) // sendTxs sends the given transactions to the node and @@ -51,7 +52,8 @@ func (s *Suite) sendTxs(t *utesting.T, txs []*types.Transaction) error { return fmt.Errorf("peering failed: %v", err) } - if err = sendConn.Write(ethProto, eth.TransactionsMsg, eth.TransactionsPacket(txs)); err != nil { + encTxs, _ := rlp.EncodeToRawList(txs) + if err = sendConn.Write(ethProto, eth.TransactionsMsg, eth.TransactionsPacket{RawList: encTxs}); err != nil { return fmt.Errorf("failed to write message to connection: %v", err) } @@ -68,7 +70,8 @@ func (s *Suite) sendTxs(t *utesting.T, txs []*types.Transaction) error { } switch msg := msg.(type) { case *eth.TransactionsPacket: - for _, tx := range *msg { + txs, _ := msg.Items() + for _, tx := range txs { got[tx.Hash()] = true } case *eth.NewPooledTransactionHashesPacket: @@ -80,9 +83,10 @@ func (s *Suite) sendTxs(t *utesting.T, txs []*types.Transaction) error { if err != nil { t.Logf("invalid GetBlockHeaders request: %v", err) } + encHeaders, _ := rlp.EncodeToRawList(headers) recvConn.Write(ethProto, eth.BlockHeadersMsg, ð.BlockHeadersPacket{ - RequestId: msg.RequestId, - BlockHeadersRequest: headers, + RequestId: msg.RequestId, + List: encHeaders, }) default: return fmt.Errorf("unexpected eth wire msg: %s", pretty.Sdump(msg)) @@ -167,9 +171,10 @@ func (s *Suite) sendInvalidTxs(t *utesting.T, txs []*types.Transaction) error { if err != nil { t.Logf("invalid GetBlockHeaders request: %v", err) } + encHeaders, _ := rlp.EncodeToRawList(headers) recvConn.Write(ethProto, eth.BlockHeadersMsg, ð.BlockHeadersPacket{ - RequestId: msg.RequestId, - BlockHeadersRequest: headers, + RequestId: msg.RequestId, + List: encHeaders, }) default: return fmt.Errorf("unexpected eth message: %v", pretty.Sdump(msg)) diff --git a/cmd/devp2p/internal/v5test/discv5tests.go b/cmd/devp2p/internal/v5test/discv5tests.go index 2139cd8ca6..efe9144069 100644 --- a/cmd/devp2p/internal/v5test/discv5tests.go +++ b/cmd/devp2p/internal/v5test/discv5tests.go @@ -52,7 +52,7 @@ func (s *Suite) AllTests() []utesting.Test { {Name: "Ping", Fn: s.TestPing}, {Name: "PingLargeRequestID", Fn: s.TestPingLargeRequestID}, {Name: "PingMultiIP", Fn: s.TestPingMultiIP}, - {Name: "PingHandshakeInterrupted", Fn: s.TestPingHandshakeInterrupted}, + {Name: "HandshakeResend", Fn: s.TestHandshakeResend}, {Name: "TalkRequest", Fn: s.TestTalkRequest}, {Name: "FindnodeZeroDistance", Fn: s.TestFindnodeZeroDistance}, {Name: "FindnodeResults", Fn: s.TestFindnodeResults}, @@ -158,22 +158,20 @@ the attempt from a different IP.`) } } -// TestPingHandshakeInterrupted starts a handshake, but doesn't finish it and sends a second ordinary message -// packet instead of a handshake message packet. The remote node should respond with -// another WHOAREYOU challenge for the second packet. -func (s *Suite) TestPingHandshakeInterrupted(t *utesting.T) { - t.Log(`TestPingHandshakeInterrupted starts a handshake, but doesn't finish it and sends a second ordinary message -packet instead of a handshake message packet. The remote node should respond with -another WHOAREYOU challenge for the second packet.`) - +// TestHandshakeResend starts a handshake, but doesn't finish it and sends a second ordinary message +// packet instead of a handshake message packet. The remote node should repeat the previous WHOAREYOU +// challenge for the first PING. +func (s *Suite) TestHandshakeResend(t *utesting.T) { conn, l1 := s.listen1(t) defer conn.close() // First PING triggers challenge. ping := &v5wire.Ping{ReqID: conn.nextReqID()} conn.write(l1, ping, nil) + var challenge1 *v5wire.Whoareyou switch resp := conn.read(l1).(type) { case *v5wire.Whoareyou: + challenge1 = resp t.Logf("got WHOAREYOU for PING") default: t.Fatal("expected WHOAREYOU, got", resp) @@ -181,9 +179,16 @@ another WHOAREYOU challenge for the second packet.`) // Send second PING. ping2 := &v5wire.Ping{ReqID: conn.nextReqID()} - switch resp := conn.reqresp(l1, ping2).(type) { - case *v5wire.Pong: - checkPong(t, resp, ping2, l1) + conn.write(l1, ping2, nil) + switch resp := conn.read(l1).(type) { + case *v5wire.Whoareyou: + if resp.Nonce != challenge1.Nonce { + t.Fatalf("wrong nonce %x in WHOAREYOU (want %x)", resp.Nonce[:], challenge1.Nonce[:]) + } + if !bytes.Equal(resp.ChallengeData, challenge1.ChallengeData) { + t.Fatalf("wrong ChallengeData in resent WHOAREYOU (want %x)", resp.ChallengeData, challenge1.ChallengeData) + } + resp.Node = conn.remote default: t.Fatal("expected WHOAREYOU, got", resp) } diff --git a/cmd/devp2p/internal/v5test/framework.go b/cmd/devp2p/internal/v5test/framework.go index 92a5048150..acab1eef79 100644 --- a/cmd/devp2p/internal/v5test/framework.go +++ b/cmd/devp2p/internal/v5test/framework.go @@ -218,11 +218,15 @@ func (tc *conn) read(c net.PacketConn) v5wire.Packet { if err := c.SetReadDeadline(time.Now().Add(waitTime)); err != nil { return &readError{err} } - n, fromAddr, err := c.ReadFrom(buf) + n, _, err := c.ReadFrom(buf) if err != nil { return &readError{err} } - _, _, p, err := tc.codec.Decode(buf[:n], fromAddr.String()) + // Always use tc.remoteAddr for session lookup. The actual source address of + // the packet may differ from tc.remoteAddr when the remote node is reachable + // via multiple networks (e.g. Docker bridge vs. overlay), but the codec's + // session cache is keyed by the address used during Encode. + _, _, p, err := tc.codec.Decode(buf[:n], tc.remoteAddr.String()) if err != nil { return &readError{err} } diff --git a/cmd/era/main.go b/cmd/era/main.go index 35a889d4dc..1c26f44ad4 100644 --- a/cmd/era/main.go +++ b/cmd/era/main.go @@ -30,6 +30,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/era/execdb" + "github.com/ethereum/go-ethereum/internal/era/onedb" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/params" @@ -53,7 +55,7 @@ var ( eraSizeFlag = &cli.IntFlag{ Name: "size", Usage: "number of blocks per era", - Value: era.MaxEra1Size, + Value: era.MaxSize, } txsFlag = &cli.BoolFlag{ Name: "txs", @@ -131,7 +133,7 @@ func block(ctx *cli.Context) error { return nil } -// info prints some high-level information about the era1 file. +// info prints some high-level information about the era file. func info(ctx *cli.Context) error { epoch, err := strconv.ParseUint(ctx.Args().First(), 10, 64) if err != nil { @@ -142,33 +144,34 @@ func info(ctx *cli.Context) error { return err } defer e.Close() - acc, err := e.Accumulator() - if err != nil { - return fmt.Errorf("error reading accumulator: %w", err) + var ( + accHex string + tdStr string + ) + if acc, err := e.Accumulator(); err == nil { + accHex = acc.Hex() } - td, err := e.InitialTD() - if err != nil { - return fmt.Errorf("error reading total difficulty: %w", err) + if td, err := e.InitialTD(); err == nil { + tdStr = td.String() } info := struct { - Accumulator common.Hash `json:"accumulator"` - TotalDifficulty *big.Int `json:"totalDifficulty"` - StartBlock uint64 `json:"startBlock"` - Count uint64 `json:"count"` + Accumulator string `json:"accumulator,omitempty"` + TotalDifficulty string `json:"totalDifficulty,omitempty"` + StartBlock uint64 `json:"startBlock"` + Count uint64 `json:"count"` }{ - acc, td, e.Start(), e.Count(), + accHex, tdStr, e.Start(), e.Count(), } b, _ := json.MarshalIndent(info, "", " ") fmt.Println(string(b)) return nil } -// open opens an era1 file at a certain epoch. -func open(ctx *cli.Context, epoch uint64) (*era.Era, error) { - var ( - dir = ctx.String(dirFlag.Name) - network = ctx.String(networkFlag.Name) - ) +// open opens an era file at a certain epoch. +func open(ctx *cli.Context, epoch uint64) (era.Era, error) { + dir := ctx.String(dirFlag.Name) + network := ctx.String(networkFlag.Name) + entries, err := era.ReadDir(dir, network) if err != nil { return nil, fmt.Errorf("error reading era dir: %w", err) @@ -176,7 +179,28 @@ func open(ctx *cli.Context, epoch uint64) (*era.Era, error) { if epoch >= uint64(len(entries)) { return nil, fmt.Errorf("epoch out-of-bounds: last %d, want %d", len(entries)-1, epoch) } - return era.Open(filepath.Join(dir, entries[epoch])) + path := filepath.Join(dir, entries[epoch]) + return openByPath(path) +} + +// openByPath tries to open a single file as either eraE or era1 based on extension, +// falling back to the other reader if needed. +func openByPath(path string) (era.Era, error) { + switch strings.ToLower(filepath.Ext(path)) { + case ".erae": + if e, err := execdb.Open(path); err != nil { + return nil, err + } else { + return e, nil + } + case ".era1": + if e, err := onedb.Open(path); err != nil { + return nil, err + } else { + return e, nil + } + } + return nil, fmt.Errorf("unsupported or unreadable era file: %s", path) } // verify checks each era1 file in a directory to ensure it is well-formed and @@ -203,18 +227,58 @@ func verify(ctx *cli.Context) error { return fmt.Errorf("error reading %s: %w", dir, err) } - if len(entries) != len(roots) { - return errors.New("number of era1 files should match the number of accumulator hashes") + // Build the verification list respecting the rule: + // era1: must have accumulator, always verify + // erae: verify only if accumulator exists (pre-merge) + + // Build list of files to verify. + verify := make([]string, 0, len(entries)) + + for _, name := range entries { + path := filepath.Join(dir, name) + ext := strings.ToLower(filepath.Ext(name)) + + switch ext { + case ".era1": + e, err := onedb.Open(path) + if err != nil { + return fmt.Errorf("error opening era1 file %s: %w", name, err) + } + _, accErr := e.Accumulator() + e.Close() + if accErr != nil { + return fmt.Errorf("era1 file %s missing accumulator: %w", name, accErr) + } + verify = append(verify, path) + + case ".erae": + e, err := execdb.Open(path) + if err != nil { + return fmt.Errorf("error opening erae file %s: %w", name, err) + } + _, accErr := e.Accumulator() + e.Close() + if accErr == nil { + verify = append(verify, path) // pre-merge only + } + default: + return fmt.Errorf("unsupported era file: %s", name) + } + } + + if len(verify) != len(roots) { + return fmt.Errorf("mismatch between eras to verify (%d) and provided roots (%d)", len(verify), len(roots)) } // Verify each epoch matches the expected root. for i, want := range roots { // Wrap in function so defers don't stack. err := func() error { - name := entries[i] - e, err := era.Open(filepath.Join(dir, name)) + path := verify[i] + name := filepath.Base(path) + e, err := openByPath(path) if err != nil { - return fmt.Errorf("error opening era1 file %s: %w", name, err) + return fmt.Errorf("error opening era file %s: %w", name, err) } defer e.Close() // Read accumulator and check against expected. @@ -243,7 +307,7 @@ func verify(ctx *cli.Context) error { } // checkAccumulator verifies the accumulator matches the data in the Era. -func checkAccumulator(e *era.Era) error { +func checkAccumulator(e era.Era) error { var ( err error want common.Hash @@ -257,7 +321,7 @@ func checkAccumulator(e *era.Era) error { if td, err = e.InitialTD(); err != nil { return fmt.Errorf("error reading total difficulty: %w", err) } - it, err := era.NewIterator(e) + it, err := e.Iterator() if err != nil { return fmt.Errorf("error making era iterator: %w", err) } @@ -290,9 +354,13 @@ func checkAccumulator(e *era.Era) error { if rr != block.ReceiptHash() { return fmt.Errorf("receipt root in block %d mismatch: want %s, got %s", block.NumberU64(), block.ReceiptHash(), rr) } - hashes = append(hashes, block.Hash()) - td.Add(td, block.Difficulty()) - tds = append(tds, new(big.Int).Set(td)) + // Only include pre-merge blocks in accumulator calculation. + // Post-merge blocks have difficulty == 0. + if block.Difficulty().Sign() > 0 { + hashes = append(hashes, block.Hash()) + td.Add(td, block.Difficulty()) + tds = append(tds, new(big.Int).Set(td)) + } } if it.Error() != nil { return fmt.Errorf("error reading block %d: %w", it.Number(), it.Error()) diff --git a/cmd/evm/blockrunner.go b/cmd/evm/blockrunner.go index f6538b1356..c6fac5396e 100644 --- a/cmd/evm/blockrunner.go +++ b/cmd/evm/blockrunner.go @@ -17,16 +17,18 @@ package main import ( + "bufio" "encoding/json" - "errors" "fmt" "maps" "os" "regexp" "slices" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/tests" "github.com/urfave/cli/v2" ) @@ -34,33 +36,52 @@ import ( var blockTestCommand = &cli.Command{ Action: blockTestCmd, Name: "blocktest", - Usage: "Executes the given blockchain tests", + Usage: "Executes the given blockchain tests. Filenames can be fed via standard input (batch mode) or as an argument (one-off execution).", ArgsUsage: "", Flags: slices.Concat([]cli.Flag{ DumpFlag, HumanReadableFlag, RunFlag, WitnessCrossCheckFlag, + FuzzFlag, }, traceFlags), } func blockTestCmd(ctx *cli.Context) error { path := ctx.Args().First() - if len(path) == 0 { - return errors.New("path argument required") + + // If path is provided, run the tests at that path. + if len(path) != 0 { + var ( + collected = collectFiles(path) + results []testResult + ) + for _, fname := range collected { + r, err := runBlockTest(ctx, fname) + if err != nil { + return err + } + results = append(results, r...) + } + report(ctx, results) + return nil } - var ( - collected = collectFiles(path) - results []testResult - ) - for _, fname := range collected { - r, err := runBlockTest(ctx, fname) + // Otherwise, read filenames from stdin and execute back-to-back. + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + fname := scanner.Text() + if len(fname) == 0 { + return nil + } + results, err := runBlockTest(ctx, fname) if err != nil { return err } - results = append(results, r...) + // During fuzzing, we report the result after every block + if !ctx.IsSet(FuzzFlag.Name) { + report(ctx, results) + } } - report(ctx, results) return nil } @@ -79,6 +100,11 @@ func runBlockTest(ctx *cli.Context, fname string) ([]testResult, error) { } tracer := tracerFromFlags(ctx) + // Suppress INFO logs during fuzzing + if ctx.IsSet(FuzzFlag.Name) { + log.SetDefault(log.NewLogger(log.DiscardHandler())) + } + // Pull out keys to sort and ensure tests are run in order. keys := slices.Sorted(maps.Keys(tests)) @@ -88,16 +114,35 @@ func runBlockTest(ctx *cli.Context, fname string) ([]testResult, error) { if !re.MatchString(name) { continue } + test := tests[name] result := &testResult{Name: name, Pass: true} - if err := tests[name].Run(false, rawdb.PathScheme, ctx.Bool(WitnessCrossCheckFlag.Name), tracer, func(res error, chain *core.BlockChain) { + var finalRoot *common.Hash + if err := test.Run(false, rawdb.PathScheme, ctx.Bool(WitnessCrossCheckFlag.Name), tracer, func(res error, chain *core.BlockChain) { if ctx.Bool(DumpFlag.Name) { if s, _ := chain.State(); s != nil { result.State = dump(s) } } + // Capture final state root for end marker + if chain != nil { + root := chain.CurrentBlock().Root + finalRoot = &root + } }); err != nil { result.Pass, result.Error = false, err.Error() } + + // Always assign fork (regardless of pass/fail or tracer) + result.Fork = test.Network() + // Assign root if test succeeded + if result.Pass && finalRoot != nil { + result.Root = finalRoot + } + + // When fuzzing, write results after every block + if ctx.IsSet(FuzzFlag.Name) { + report(ctx, []testResult{*result}) + } results = append(results, *result) } return results, nil diff --git a/cmd/evm/internal/t8ntool/block.go b/cmd/evm/internal/t8ntool/block.go index 37a6db9ffc..6148e9e248 100644 --- a/cmd/evm/internal/t8ntool/block.go +++ b/cmd/evm/internal/t8ntool/block.go @@ -56,6 +56,7 @@ type header struct { BlobGasUsed *uint64 `json:"blobGasUsed" rlp:"optional"` ExcessBlobGas *uint64 `json:"excessBlobGas" rlp:"optional"` ParentBeaconBlockRoot *common.Hash `json:"parentBeaconBlockRoot" rlp:"optional"` + SlotNumber *uint64 `json:"slotNumber" rlp:"optional"` } type headerMarshaling struct { @@ -68,6 +69,7 @@ type headerMarshaling struct { BaseFee *math.HexOrDecimal256 BlobGasUsed *math.HexOrDecimal64 ExcessBlobGas *math.HexOrDecimal64 + SlotNumber *math.HexOrDecimal64 } type bbInput struct { @@ -136,6 +138,7 @@ func (i *bbInput) ToBlock() *types.Block { BlobGasUsed: i.Header.BlobGasUsed, ExcessBlobGas: i.Header.ExcessBlobGas, ParentBeaconRoot: i.Header.ParentBeaconBlockRoot, + SlotNumber: i.Header.SlotNumber, } // Fill optional values. diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 17ca65a381..58f8535f3f 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -18,6 +18,7 @@ package t8ntool import ( "fmt" + stdmath "math" "math/big" "github.com/ethereum/go-ethereum/arbcrypto" @@ -43,8 +44,9 @@ import ( ) type Prestate struct { - Env stEnv `json:"env"` - Pre types.GenesisAlloc `json:"pre"` + Env stEnv `json:"env"` + Pre types.GenesisAlloc `json:"pre"` + TreeLeaves map[common.Hash]hexutil.Bytes `json:"vkt,omitempty"` } //go:generate go run github.com/fjl/gencodec -type ExecutionResult -field-override executionResultMarshaling -out gen_execresult.go @@ -100,6 +102,7 @@ type stEnv struct { ParentExcessBlobGas *uint64 `json:"parentExcessBlobGas,omitempty"` ParentBlobGasUsed *uint64 `json:"parentBlobGasUsed,omitempty"` ParentBeaconBlockRoot *common.Hash `json:"parentBeaconBlockRoot"` + SlotNumber *uint64 `json:"slotNumber"` } type stEnvMarshaling struct { @@ -118,6 +121,7 @@ type stEnvMarshaling struct { ExcessBlobGas *math.HexOrDecimal64 ParentExcessBlobGas *math.HexOrDecimal64 ParentBlobGasUsed *math.HexOrDecimal64 + SlotNumber *math.HexOrDecimal64 } type rejectedTx struct { @@ -145,17 +149,16 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, return h } var ( - statedb = MakePreState(rawdb.NewMemoryDatabase(), pre.Pre) + isEIP4762 = chainConfig.IsVerkle(big.NewInt(int64(pre.Env.Number)), pre.Env.Timestamp) + statedb = MakePreState(rawdb.NewMemoryDatabase(), pre.Pre, isEIP4762) signer = types.MakeSigner(chainConfig, new(big.Int).SetUint64(pre.Env.Number), pre.Env.Timestamp, params.MaxArbosVersionSupported) - gaspool = new(core.GasPool) + gaspool = core.NewGasPool(pre.Env.GasLimit) blockHash = common.Hash{0x13, 0x37} rejectedTxs []*rejectedTx includedTxs types.Transactions - gasUsed = uint64(0) blobGasUsed = uint64(0) receipts = make(types.Receipts, 0) ) - gaspool.AddGas(pre.Env.GasLimit) vmContext := vm.BlockContext{ CanTransfer: core.CanTransfer, Transfer: core.Transfer, @@ -195,6 +198,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, ExcessBlobGas: pre.Env.ParentExcessBlobGas, BlobGasUsed: pre.Env.ParentBlobGasUsed, BaseFee: pre.Env.ParentBaseFee, + SlotNumber: pre.Env.SlotNumber, } header := &types.Header{ Time: pre.Env.Timestamp, @@ -255,16 +259,19 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, statedb.SetTxContext(tx.Hash(), len(receipts)) var ( snapshot = statedb.Snapshot() - prevGas = gaspool.Gas() + gp = gaspool.Snapshot() ) - receipt, _, err := core.ApplyTransactionWithEVM(msg, gaspool, statedb, vmContext.BlockNumber, blockHash, pre.Env.Timestamp, tx, &gasUsed, evm, nil) + receipt, _, err := core.ApplyTransactionWithEVM(msg, gaspool, statedb, vmContext.BlockNumber, blockHash, pre.Env.Timestamp, tx, evm, nil) if err != nil { statedb.RevertToSnapshot(snapshot) log.Info("rejected tx", "index", i, "hash", tx.Hash(), "from", msg.From, "error", err) rejectedTxs = append(rejectedTxs, &rejectedTx{i, err.Error()}) - gaspool.SetGas(prevGas) + gaspool.Set(gp) continue } + if receipt.Logs == nil { + receipt.Logs = []*types.Log{} + } includedTxs = append(includedTxs, tx) if hashError != nil { return nil, nil, nil, NewError(ErrorMissingBlockhash, hashError) @@ -304,6 +311,10 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, // Amount is in gwei, turn into wei amount := new(big.Int).Mul(new(big.Int).SetUint64(w.Amount), big.NewInt(params.GWei)) statedb.AddBalance(w.Address, uint256.MustFromBig(amount), tracing.BalanceIncreaseWithdrawal) + + if isEIP4762 { + statedb.AccessEvents().AddAccount(w.Address, true, stdmath.MaxUint64) + } } // Gather the execution-layer triggered requests. @@ -343,7 +354,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, Receipts: receipts, Rejected: rejectedTxs, Difficulty: (*math.HexOrDecimal256)(vmContext.Difficulty), - GasUsed: (math.HexOrDecimal64)(gasUsed), + GasUsed: (math.HexOrDecimal64)(gaspool.Used()), BaseFee: (*math.HexOrDecimal256)(vmContext.BaseFee), } if pre.Env.Withdrawals != nil { @@ -358,15 +369,10 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, // Set requestsHash on block. h := types.CalcRequestsHash(requests) execRs.RequestsHash = &h - for i := range requests { - // remove prefix - requests[i] = requests[i][1:] - } execRs.Requests = requests } - // Re-create statedb instance with new root upon the updated database - // for accessing latest states. + // Re-create statedb instance with new root for MPT mode statedb, err = state.New(root, statedb.Database()) if err != nil { return nil, nil, nil, NewError(ErrorEVM, fmt.Errorf("could not reopen state: %v", err)) @@ -375,12 +381,17 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, return statedb, execRs, body, nil } -func MakePreState(db ethdb.Database, accounts types.GenesisAlloc) *state.StateDB { - tdb := triedb.NewDatabase(db, &triedb.Config{Preimages: true}) +func MakePreState(db ethdb.Database, accounts types.GenesisAlloc, isBintrie bool) *state.StateDB { + tdb := triedb.NewDatabase(db, &triedb.Config{Preimages: true, IsVerkle: isBintrie}) sdb := state.NewDatabase(tdb, nil) - statedb, err := state.New(types.EmptyRootHash, sdb) + + root := types.EmptyRootHash + if isBintrie { + root = types.EmptyBinaryHash + } + statedb, err := state.New(root, sdb) if err != nil { - panic(fmt.Errorf("failed to create initial state: %v", err)) + panic(fmt.Errorf("failed to create initial statedb: %v", err)) } for addr, a := range accounts { statedb.SetCode(addr, a.Code, tracing.CodeChangeUnspecified) @@ -391,10 +402,15 @@ func MakePreState(db ethdb.Database, accounts types.GenesisAlloc) *state.StateDB } } // Commit and re-open to start with a clean state. - root, err := statedb.Commit(0, false, false) + root, err = statedb.Commit(0, false, false) if err != nil { panic(fmt.Errorf("failed to commit initial state: %v", err)) } + // If bintrie mode started, check if conversion happened + if isBintrie { + return statedb + } + // For MPT mode, reopen the state with the committed root statedb, err = state.New(root, sdb) if err != nil { panic(fmt.Errorf("failed to reopen state after commit: %v", err)) @@ -402,7 +418,7 @@ func MakePreState(db ethdb.Database, accounts types.GenesisAlloc) *state.StateDB return statedb } -func rlpHash(x interface{}) (h common.Hash) { +func rlpHash(x any) (h common.Hash) { hw := arbcrypto.NewLegacyKeccak256() rlp.Encode(hw, x) hw.Sum(h[:0]) diff --git a/cmd/evm/internal/t8ntool/file_tracer.go b/cmd/evm/internal/t8ntool/file_tracer.go index 38fc35bd32..d032252cba 100644 --- a/cmd/evm/internal/t8ntool/file_tracer.go +++ b/cmd/evm/internal/t8ntool/file_tracer.go @@ -56,27 +56,35 @@ func (l *fileWritingTracer) Write(p []byte) (n int, err error) { return n, nil } -// newFileWriter creates a set of hooks which wraps inner hooks (typically a logger), +// newFileWriter creates a tracer which wraps inner hooks (typically a logger), // and writes the output to a file, one file per transaction. -func newFileWriter(baseDir string, innerFn func(out io.Writer) *tracing.Hooks) *tracing.Hooks { +func newFileWriter(baseDir string, innerFn func(out io.Writer) *tracing.Hooks) *tracers.Tracer { t := &fileWritingTracer{ baseDir: baseDir, suffix: "jsonl", } t.inner = innerFn(t) // instantiate the inner tracer - return t.hooks() + return &tracers.Tracer{ + Hooks: t.hooks(), + GetResult: func() (json.RawMessage, error) { return json.RawMessage("{}"), nil }, + Stop: func(err error) {}, + } } -// newResultWriter creates a set of hooks wraps and invokes an underlying tracer, +// newResultWriter creates a tracer that wraps and invokes an underlying tracer, // and writes the result (getResult-output) to file, one per transaction. -func newResultWriter(baseDir string, tracer *tracers.Tracer) *tracing.Hooks { +func newResultWriter(baseDir string, tracer *tracers.Tracer) *tracers.Tracer { t := &fileWritingTracer{ baseDir: baseDir, getResult: tracer.GetResult, inner: tracer.Hooks, suffix: "json", } - return t.hooks() + return &tracers.Tracer{ + Hooks: t.hooks(), + GetResult: func() (json.RawMessage, error) { return json.RawMessage("{}"), nil }, + Stop: func(err error) {}, + } } // OnTxStart creates a new output-file specific for this transaction, and invokes diff --git a/cmd/evm/internal/t8ntool/flags.go b/cmd/evm/internal/t8ntool/flags.go index f2606c86d1..078d304927 100644 --- a/cmd/evm/internal/t8ntool/flags.go +++ b/cmd/evm/internal/t8ntool/flags.go @@ -88,6 +88,14 @@ var ( "\t - into the file ", Value: "block.json", } + OutputBTFlag = &cli.StringFlag{ + Name: "output.vkt", + Usage: "Determines where to put the `BT` of the post-state.\n" + + "\t`stdout` - into the stdout output\n" + + "\t`stderr` - into the stderr output\n" + + "\t - into the file ", + Value: "vkt.json", + } InputAllocFlag = &cli.StringFlag{ Name: "input.alloc", Usage: "`stdin` or file name of where to find the prestate alloc to use.", @@ -123,6 +131,11 @@ var ( Usage: "`stdin` or file name of where to find the transactions list in RLP form.", Value: "txs.rlp", } + // TODO(@CPerezz): rename `Name` of the file in a follow-up PR (relays on EEST -> https://github.com/ethereum/execution-spec-tests/tree/verkle/main) + InputBTFlag = &cli.StringFlag{ + Name: "input.vkt", + Usage: "`stdin` or file name of where to find the prestate BT.", + } SealCliqueFlag = &cli.StringFlag{ Name: "seal.clique", Usage: "Seal block with Clique. `stdin` or file name of where to find the Clique sealing data.", @@ -149,6 +162,11 @@ var ( strings.Join(vm.ActivateableEips(), ", ")), Value: "GrayGlacier", } + OpcodeCountFlag = &cli.StringFlag{ + Name: "opcode.count", + Usage: "If set, opcode execution counts will be written to this file (relative to output.basedir).", + Value: "", + } VerbosityFlag = &cli.IntFlag{ Name: "verbosity", Usage: "sets the verbosity level", diff --git a/cmd/evm/internal/t8ntool/gen_header.go b/cmd/evm/internal/t8ntool/gen_header.go index a8c8668978..f430feb6d2 100644 --- a/cmd/evm/internal/t8ntool/gen_header.go +++ b/cmd/evm/internal/t8ntool/gen_header.go @@ -38,6 +38,7 @@ func (h header) MarshalJSON() ([]byte, error) { BlobGasUsed *math.HexOrDecimal64 `json:"blobGasUsed" rlp:"optional"` ExcessBlobGas *math.HexOrDecimal64 `json:"excessBlobGas" rlp:"optional"` ParentBeaconBlockRoot *common.Hash `json:"parentBeaconBlockRoot" rlp:"optional"` + SlotNumber *math.HexOrDecimal64 `json:"slotNumber" rlp:"optional"` } var enc header enc.ParentHash = h.ParentHash @@ -60,6 +61,7 @@ func (h header) MarshalJSON() ([]byte, error) { enc.BlobGasUsed = (*math.HexOrDecimal64)(h.BlobGasUsed) enc.ExcessBlobGas = (*math.HexOrDecimal64)(h.ExcessBlobGas) enc.ParentBeaconBlockRoot = h.ParentBeaconBlockRoot + enc.SlotNumber = (*math.HexOrDecimal64)(h.SlotNumber) return json.Marshal(&enc) } @@ -86,6 +88,7 @@ func (h *header) UnmarshalJSON(input []byte) error { BlobGasUsed *math.HexOrDecimal64 `json:"blobGasUsed" rlp:"optional"` ExcessBlobGas *math.HexOrDecimal64 `json:"excessBlobGas" rlp:"optional"` ParentBeaconBlockRoot *common.Hash `json:"parentBeaconBlockRoot" rlp:"optional"` + SlotNumber *math.HexOrDecimal64 `json:"slotNumber" rlp:"optional"` } var dec header if err := json.Unmarshal(input, &dec); err != nil { @@ -155,5 +158,8 @@ func (h *header) UnmarshalJSON(input []byte) error { if dec.ParentBeaconBlockRoot != nil { h.ParentBeaconBlockRoot = dec.ParentBeaconBlockRoot } + if dec.SlotNumber != nil { + h.SlotNumber = (*uint64)(dec.SlotNumber) + } return nil } diff --git a/cmd/evm/internal/t8ntool/gen_stenv.go b/cmd/evm/internal/t8ntool/gen_stenv.go index d47db4a876..1b8e14abc6 100644 --- a/cmd/evm/internal/t8ntool/gen_stenv.go +++ b/cmd/evm/internal/t8ntool/gen_stenv.go @@ -37,6 +37,7 @@ func (s stEnv) MarshalJSON() ([]byte, error) { ParentExcessBlobGas *math.HexOrDecimal64 `json:"parentExcessBlobGas,omitempty"` ParentBlobGasUsed *math.HexOrDecimal64 `json:"parentBlobGasUsed,omitempty"` ParentBeaconBlockRoot *common.Hash `json:"parentBeaconBlockRoot"` + SlotNumber *math.HexOrDecimal64 `json:"slotNumber"` } var enc stEnv enc.Coinbase = common.UnprefixedAddress(s.Coinbase) @@ -59,6 +60,7 @@ func (s stEnv) MarshalJSON() ([]byte, error) { enc.ParentExcessBlobGas = (*math.HexOrDecimal64)(s.ParentExcessBlobGas) enc.ParentBlobGasUsed = (*math.HexOrDecimal64)(s.ParentBlobGasUsed) enc.ParentBeaconBlockRoot = s.ParentBeaconBlockRoot + enc.SlotNumber = (*math.HexOrDecimal64)(s.SlotNumber) return json.Marshal(&enc) } @@ -85,6 +87,7 @@ func (s *stEnv) UnmarshalJSON(input []byte) error { ParentExcessBlobGas *math.HexOrDecimal64 `json:"parentExcessBlobGas,omitempty"` ParentBlobGasUsed *math.HexOrDecimal64 `json:"parentBlobGasUsed,omitempty"` ParentBeaconBlockRoot *common.Hash `json:"parentBeaconBlockRoot"` + SlotNumber *math.HexOrDecimal64 `json:"slotNumber"` } var dec stEnv if err := json.Unmarshal(input, &dec); err != nil { @@ -154,5 +157,8 @@ func (s *stEnv) UnmarshalJSON(input []byte) error { if dec.ParentBeaconBlockRoot != nil { s.ParentBeaconBlockRoot = dec.ParentBeaconBlockRoot } + if dec.SlotNumber != nil { + s.SlotNumber = (*uint64)(dec.SlotNumber) + } return nil } diff --git a/cmd/evm/internal/t8ntool/transaction.go b/cmd/evm/internal/t8ntool/transaction.go index 8b84a5626b..74d3caca8e 100644 --- a/cmd/evm/internal/t8ntool/transaction.go +++ b/cmd/evm/internal/t8ntool/transaction.go @@ -27,7 +27,9 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/tests" @@ -115,9 +117,6 @@ func Transaction(ctx *cli.Context) error { } var results []result for it.Next() { - if err := it.Err(); err != nil { - return NewError(ErrorIO, err) - } var tx types.Transaction err := rlp.DecodeBytes(it.Value(), &tx) if err != nil { @@ -180,14 +179,21 @@ func Transaction(ctx *cli.Context) error { r.Error = errors.New("gas * maxFeePerGas exceeds 256 bits") } // Check whether the init code size has been exceeded. - if chainConfig.IsShanghai(new(big.Int), 0, 0) && tx.To() == nil && len(tx.Data()) > int(chainConfig.MaxInitCodeSize()) { - r.Error = errors.New("max initcode size exceeded") + if tx.To() == nil { + if err := vm.CheckMaxInitCodeSize(&rules, uint64(len(tx.Data()))); err != nil { + r.Error = err + } } + if !chainConfig.IsArbitrum() && chainConfig.IsOsaka(new(big.Int), 0, 0) && tx.Gas() > params.MaxTxGasRenamedForNitroMerges { r.Error = errors.New("gas limit exceeds maximum") } results = append(results, r) } + if err := it.Err(); err != nil { + return NewError(ErrorIO, err) + } + out, err := json.MarshalIndent(results, "", " ") fmt.Println(string(out)) return err diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index 5b358be4ef..b8f5daad08 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -28,15 +28,23 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus/misc/eip1559" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/eth/tracers/logger" + "github.com/ethereum/go-ethereum/eth/tracers/native" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/tests" + "github.com/ethereum/go-ethereum/trie/bintrie" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/database" + "github.com/holiman/uint256" "github.com/urfave/cli/v2" ) @@ -75,10 +83,11 @@ var ( ) type input struct { - Alloc types.GenesisAlloc `json:"alloc,omitempty"` - Env *stEnv `json:"env,omitempty"` - Txs []*txWithKey `json:"txs,omitempty"` - TxRlp string `json:"txsRlp,omitempty"` + Alloc types.GenesisAlloc `json:"alloc,omitempty"` + Env *stEnv `json:"env,omitempty"` + BT map[common.Hash]hexutil.Bytes `json:"vkt,omitempty"` + Txs []*txWithKey `json:"txs,omitempty"` + TxRlp string `json:"txsRlp,omitempty"` } func Transition(ctx *cli.Context) error { @@ -90,16 +99,16 @@ func Transition(ctx *cli.Context) error { // stdin input or in files. // Check if anything needs to be read from stdin var ( - prestate Prestate - txIt txIterator // txs to apply - allocStr = ctx.String(InputAllocFlag.Name) - + prestate Prestate + txIt txIterator // txs to apply + allocStr = ctx.String(InputAllocFlag.Name) + btStr = ctx.String(InputBTFlag.Name) envStr = ctx.String(InputEnvFlag.Name) txStr = ctx.String(InputTxsFlag.Name) inputData = &input{} ) // Figure out the prestate alloc - if allocStr == stdinSelector || envStr == stdinSelector || txStr == stdinSelector { + if allocStr == stdinSelector || btStr == stdinSelector || envStr == stdinSelector || txStr == stdinSelector { decoder := json.NewDecoder(os.Stdin) if err := decoder.Decode(inputData); err != nil { return NewError(ErrorJson, fmt.Errorf("failed unmarshalling stdin: %v", err)) @@ -112,6 +121,13 @@ func Transition(ctx *cli.Context) error { } prestate.Pre = inputData.Alloc + if btStr != stdinSelector && btStr != "" { + if err := readFile(btStr, "BT", &inputData.BT); err != nil { + return err + } + } + prestate.TreeLeaves = inputData.BT + // Set the block environment if envStr != stdinSelector { var env stEnv @@ -152,14 +168,15 @@ func Transition(ctx *cli.Context) error { } // Configure tracer + var tracer *tracers.Tracer if ctx.IsSet(TraceTracerFlag.Name) { // Custom tracing config := json.RawMessage(ctx.String(TraceTracerConfigFlag.Name)) - tracer, err := tracers.DefaultDirectory.New(ctx.String(TraceTracerFlag.Name), + innerTracer, err := tracers.DefaultDirectory.New(ctx.String(TraceTracerFlag.Name), nil, config, chainConfig) if err != nil { return NewError(ErrorConfig, fmt.Errorf("failed instantiating tracer: %v", err)) } - vmConfig.Tracer = newResultWriter(baseDir, tracer) + tracer = newResultWriter(baseDir, innerTracer) } else if ctx.Bool(TraceFlag.Name) { // JSON opcode tracing logConfig := &logger.Config{ DisableStack: ctx.Bool(TraceDisableStackFlag.Name), @@ -167,24 +184,61 @@ func Transition(ctx *cli.Context) error { EnableReturnData: ctx.Bool(TraceEnableReturnDataFlag.Name), } if ctx.Bool(TraceEnableCallFramesFlag.Name) { - vmConfig.Tracer = newFileWriter(baseDir, func(out io.Writer) *tracing.Hooks { + tracer = newFileWriter(baseDir, func(out io.Writer) *tracing.Hooks { return logger.NewJSONLoggerWithCallFrames(logConfig, out) }) } else { - vmConfig.Tracer = newFileWriter(baseDir, func(out io.Writer) *tracing.Hooks { + tracer = newFileWriter(baseDir, func(out io.Writer) *tracing.Hooks { return logger.NewJSONLogger(logConfig, out) }) } } + // Configure opcode counter + var opcodeTracer *tracers.Tracer + if ctx.IsSet(OpcodeCountFlag.Name) && ctx.String(OpcodeCountFlag.Name) != "" { + opcodeTracer = native.NewOpcodeCounter() + if tracer != nil { + // If we have an existing tracer, multiplex with the opcode tracer + mux, _ := native.NewMuxTracer([]string{"trace", "opcode"}, []*tracers.Tracer{tracer, opcodeTracer}) + vmConfig.Tracer = mux.Hooks + } else { + vmConfig.Tracer = opcodeTracer.Hooks + } + } else if tracer != nil { + vmConfig.Tracer = tracer.Hooks + } // Run the test and aggregate the result s, result, body, err := prestate.Apply(vmConfig, chainConfig, txIt, ctx.Int64(RewardFlag.Name)) if err != nil { return err } + // Write opcode counts if enabled + if opcodeTracer != nil { + fname := ctx.String(OpcodeCountFlag.Name) + result, err := opcodeTracer.GetResult() + if err != nil { + return NewError(ErrorJson, fmt.Errorf("failed getting opcode counts: %v", err)) + } + if err := saveFile(baseDir, fname, result); err != nil { + return err + } + } // Dump the execution result - collector := make(Alloc) - s.DumpToCollector(collector, nil) - return dispatchOutput(ctx, baseDir, result, collector, body) + var ( + collector = make(Alloc) + btleaves map[common.Hash]hexutil.Bytes + ) + isBinary := chainConfig.IsVerkle(big.NewInt(int64(prestate.Env.Number)), prestate.Env.Timestamp) + if !isBinary { + s.DumpToCollector(collector, nil) + } else { + btleaves = make(map[common.Hash]hexutil.Bytes) + if err := s.DumpBinTrieLeaves(btleaves); err != nil { + return err + } + } + + return dispatchOutput(ctx, baseDir, result, collector, body, btleaves) } func applyLondonChecks(env *stEnv, chainConfig *params.ChainConfig) error { @@ -306,7 +360,7 @@ func saveFile(baseDir, filename string, data interface{}) error { // dispatchOutput writes the output data to either stderr or stdout, or to the specified // files -func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, alloc Alloc, body hexutil.Bytes) error { +func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, alloc Alloc, body hexutil.Bytes, bt map[common.Hash]hexutil.Bytes) error { stdOutObject := make(map[string]interface{}) stdErrObject := make(map[string]interface{}) dispatch := func(baseDir, fName, name string, obj interface{}) error { @@ -333,6 +387,13 @@ func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, a if err := dispatch(baseDir, ctx.String(OutputBodyFlag.Name), "body", body); err != nil { return err } + // Only write bt output if we actually have binary trie leaves + if bt != nil { + if err := dispatch(baseDir, ctx.String(OutputBTFlag.Name), "vkt", bt); err != nil { + return err + } + } + if len(stdOutObject) > 0 { b, err := json.MarshalIndent(stdOutObject, "", " ") if err != nil { @@ -351,3 +412,168 @@ func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, a } return nil } + +// BinKey computes the tree key given an address and an optional slot number. +func BinKey(ctx *cli.Context) error { + if ctx.Args().Len() == 0 || ctx.Args().Len() > 2 { + return errors.New("invalid number of arguments: expecting an address and an optional slot number") + } + + addr, err := hexutil.Decode(ctx.Args().Get(0)) + if err != nil { + return fmt.Errorf("error decoding address: %w", err) + } + + if ctx.Args().Len() == 2 { + slot, err := hexutil.Decode(ctx.Args().Get(1)) + if err != nil { + return fmt.Errorf("error decoding slot: %w", err) + } + fmt.Printf("%#x\n", bintrie.GetBinaryTreeKeyStorageSlot(common.BytesToAddress(addr), slot)) + } else { + fmt.Printf("%#x\n", bintrie.GetBinaryTreeKeyBasicData(common.BytesToAddress(addr))) + } + return nil +} + +// BinKeys computes a set of tree keys given a genesis alloc. +func BinKeys(ctx *cli.Context) error { + var allocStr = ctx.String(InputAllocFlag.Name) + var alloc core.GenesisAlloc + // Figure out the prestate alloc + if allocStr == stdinSelector { + decoder := json.NewDecoder(os.Stdin) + if err := decoder.Decode(&alloc); err != nil { + return NewError(ErrorJson, fmt.Errorf("failed unmarshaling stdin: %v", err)) + } + } + if allocStr != stdinSelector { + if err := readFile(allocStr, "alloc", &alloc); err != nil { + return err + } + } + db := triedb.NewDatabase(rawdb.NewMemoryDatabase(), triedb.VerkleDefaults) + defer db.Close() + + bt, err := genBinTrieFromAlloc(alloc, db) + if err != nil { + return fmt.Errorf("error generating bt: %w", err) + } + + collector := make(map[common.Hash]hexutil.Bytes) + it, err := bt.NodeIterator(nil) + if err != nil { + panic(err) + } + for it.Next(true) { + if it.Leaf() { + collector[common.BytesToHash(it.LeafKey())] = it.LeafBlob() + } + } + + output, err := json.MarshalIndent(collector, "", "") + if err != nil { + return fmt.Errorf("error outputting tree: %w", err) + } + + fmt.Println(string(output)) + + return nil +} + +// BinTrieRoot computes the root of a Binary Trie from a genesis alloc. +func BinTrieRoot(ctx *cli.Context) error { + var allocStr = ctx.String(InputAllocFlag.Name) + var alloc core.GenesisAlloc + if allocStr == stdinSelector { + decoder := json.NewDecoder(os.Stdin) + if err := decoder.Decode(&alloc); err != nil { + return NewError(ErrorJson, fmt.Errorf("failed unmarshaling stdin: %v", err)) + } + } + if allocStr != stdinSelector { + if err := readFile(allocStr, "alloc", &alloc); err != nil { + return err + } + } + db := triedb.NewDatabase(rawdb.NewMemoryDatabase(), triedb.VerkleDefaults) + defer db.Close() + + bt, err := genBinTrieFromAlloc(alloc, db) + if err != nil { + return fmt.Errorf("error generating bt: %w", err) + } + fmt.Println(bt.Hash().Hex()) + + return nil +} + +// TODO(@CPerezz): Should this go to `bintrie` module? +func genBinTrieFromAlloc(alloc core.GenesisAlloc, db database.NodeDatabase) (*bintrie.BinaryTrie, error) { + bt, err := bintrie.NewBinaryTrie(types.EmptyBinaryHash, db) + if err != nil { + return nil, err + } + for addr, acc := range alloc { + for slot, value := range acc.Storage { + err := bt.UpdateStorage(addr, slot.Bytes(), value.Big().Bytes()) + if err != nil { + return nil, fmt.Errorf("error inserting storage: %w", err) + } + } + account := &types.StateAccount{ + Balance: uint256.MustFromBig(acc.Balance), + Nonce: acc.Nonce, + CodeHash: crypto.Keccak256Hash(acc.Code).Bytes(), + Root: common.Hash{}, + } + err := bt.UpdateAccount(addr, account, len(acc.Code)) + if err != nil { + return nil, fmt.Errorf("error inserting account: %w", err) + } + err = bt.UpdateContractCode(addr, common.BytesToHash(account.CodeHash), acc.Code) + if err != nil { + return nil, fmt.Errorf("error inserting code: %w", err) + } + } + return bt, nil +} + +// BinaryCodeChunkKey computes the tree key of a code-chunk for a given address. +func BinaryCodeChunkKey(ctx *cli.Context) error { + if ctx.Args().Len() == 0 || ctx.Args().Len() > 2 { + return errors.New("invalid number of arguments: expecting an address and an code-chunk number") + } + + addr, err := hexutil.Decode(ctx.Args().Get(0)) + if err != nil { + return fmt.Errorf("error decoding address: %w", err) + } + chunkNumberBytes, err := hexutil.Decode(ctx.Args().Get(1)) + if err != nil { + return fmt.Errorf("error decoding chunk number: %w", err) + } + var chunkNumber uint256.Int + chunkNumber.SetBytes(chunkNumberBytes) + + fmt.Printf("%#x\n", bintrie.GetBinaryTreeKeyCodeChunk(common.BytesToAddress(addr), &chunkNumber)) + + return nil +} + +// BinaryCodeChunkCode returns the code chunkification for a given code. +func BinaryCodeChunkCode(ctx *cli.Context) error { + if ctx.Args().Len() == 0 || ctx.Args().Len() > 1 { + return errors.New("invalid number of arguments: expecting a bytecode") + } + + bytecode, err := hexutil.Decode(ctx.Args().Get(0)) + if err != nil { + return fmt.Errorf("error decoding address: %w", err) + } + + chunkedCode := bintrie.ChunkifyCode(bytecode) + fmt.Printf("%#x\n", chunkedCode) + + return nil +} diff --git a/cmd/evm/main.go b/cmd/evm/main.go index bf5be9a359..77a06bec26 100644 --- a/cmd/evm/main.go +++ b/cmd/evm/main.go @@ -55,6 +55,11 @@ var ( Usage: "benchmark the execution", Category: flags.VMCategory, } + FuzzFlag = &cli.BoolFlag{ + Name: "fuzz", + Usage: "adapts output format for fuzzing", + Category: flags.VMCategory, + } WitnessCrossCheckFlag = &cli.BoolFlag{ Name: "cross-check", Aliases: []string{"xc"}, @@ -146,16 +151,64 @@ var ( t8ntool.TraceEnableCallFramesFlag, t8ntool.OutputBasedir, t8ntool.OutputAllocFlag, + t8ntool.OutputBTFlag, t8ntool.OutputResultFlag, t8ntool.OutputBodyFlag, t8ntool.InputAllocFlag, t8ntool.InputEnvFlag, + t8ntool.InputBTFlag, t8ntool.InputTxsFlag, t8ntool.ForknameFlag, t8ntool.ChainIDFlag, t8ntool.RewardFlag, + t8ntool.OpcodeCountFlag, + }, + } + + verkleCommand = &cli.Command{ + Name: "verkle", + Aliases: []string{"vkt"}, + Usage: "Binary Trie helpers", + Subcommands: []*cli.Command{ + { + Name: "tree-keys", + Aliases: []string{"v"}, + Usage: "compute a set of binary trie keys, given their source addresses and optional slot numbers", + Action: t8ntool.BinKeys, + Flags: []cli.Flag{ + t8ntool.InputAllocFlag, + }, + }, + { + Name: "single-key", + Aliases: []string{"vk"}, + Usage: "compute the binary trie key given an address and optional slot number", + Action: t8ntool.BinKey, + }, + { + Name: "code-chunk-key", + Aliases: []string{"vck"}, + Usage: "compute the binary trie key given an address and chunk number", + Action: t8ntool.BinaryCodeChunkKey, + }, + { + Name: "chunkify-code", + Aliases: []string{"vcc"}, + Usage: "chunkify a given bytecode for a binary trie", + Action: t8ntool.BinaryCodeChunkCode, + }, + { + Name: "state-root", + Aliases: []string{"vsr"}, + Usage: "compute the state-root of a binary trie for the given alloc", + Action: t8ntool.BinTrieRoot, + Flags: []cli.Flag{ + t8ntool.InputAllocFlag, + }, + }, }, } + transactionCommand = &cli.Command{ Name: "transaction", Aliases: []string{"t9n"}, @@ -210,6 +263,7 @@ func init() { stateTransitionCommand, transactionCommand, blockBuilderCommand, + verkleCommand, } app.Before = func(ctx *cli.Context) error { flags.MigrateGlobalFlags(ctx) diff --git a/cmd/evm/testdata/1/exp.json b/cmd/evm/testdata/1/exp.json index d83e89f7bd..d0ae130c79 100644 --- a/cmd/evm/testdata/1/exp.json +++ b/cmd/evm/testdata/1/exp.json @@ -38,7 +38,7 @@ "status": "0x1", "cumulativeGasUsed": "0x5208", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "logs": null, + "logs": [], "transactionHash": "0x0557bacce3375c98d806609b8d5043072f0b6a8bae45ae5a67a00d3a1a18d673", "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x5208", diff --git a/cmd/evm/testdata/13/exp2.json b/cmd/evm/testdata/13/exp2.json index fe7db58bf4..b77d61b9a8 100644 --- a/cmd/evm/testdata/13/exp2.json +++ b/cmd/evm/testdata/13/exp2.json @@ -26,7 +26,7 @@ "status": "0x0", "cumulativeGasUsed": "0x84d0", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "logs": null, + "logs": [], "transactionHash": "0xa98a24882ea90916c6a86da650fbc6b14238e46f0af04a131ce92be897507476", "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x84d0", @@ -55,7 +55,7 @@ "status": "0x0", "cumulativeGasUsed": "0x109a0", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "logs": null, + "logs": [], "transactionHash": "0x36bad80acce7040c45fd32764b5c2b2d2e6f778669fb41791f73f546d56e739a", "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x84d0", diff --git a/cmd/evm/testdata/23/exp.json b/cmd/evm/testdata/23/exp.json index 616ee61238..a4d0dbdfd2 100644 --- a/cmd/evm/testdata/23/exp.json +++ b/cmd/evm/testdata/23/exp.json @@ -25,7 +25,7 @@ "status": "0x1", "cumulativeGasUsed": "0x520b", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "logs": null, + "logs": [], "transactionHash": "0x72fadbef39cd251a437eea619cfeda752271a5faaaa2147df012e112159ffb81", "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x520b", diff --git a/cmd/evm/testdata/24/exp.json b/cmd/evm/testdata/24/exp.json index aa3fac506f..fb3aa45732 100644 --- a/cmd/evm/testdata/24/exp.json +++ b/cmd/evm/testdata/24/exp.json @@ -41,7 +41,7 @@ "status": "0x1", "cumulativeGasUsed": "0xa861", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "logs": null, + "logs": [], "transactionHash": "0x92ea4a28224d033afb20e0cc2b290d4c7c2d61f6a4800a680e4e19ac962ee941", "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0xa861", @@ -69,7 +69,7 @@ "status": "0x1", "cumulativeGasUsed": "0x10306", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "logs": null, + "logs": [], "transactionHash": "0x16b1d912f1d664f3f60f4e1b5f296f3c82a64a1a253117b4851d18bc03c4f1da", "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x5aa5", diff --git a/cmd/evm/testdata/25/exp.json b/cmd/evm/testdata/25/exp.json index 0891688361..d0fc7b28b6 100644 --- a/cmd/evm/testdata/25/exp.json +++ b/cmd/evm/testdata/25/exp.json @@ -37,7 +37,7 @@ "status": "0x1", "cumulativeGasUsed": "0x5208", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "logs": null, + "logs": [], "transactionHash": "0x92ea4a28224d033afb20e0cc2b290d4c7c2d61f6a4800a680e4e19ac962ee941", "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x5208", diff --git a/cmd/evm/testdata/28/exp.json b/cmd/evm/testdata/28/exp.json index 9f3c90d133..40ae9451a2 100644 --- a/cmd/evm/testdata/28/exp.json +++ b/cmd/evm/testdata/28/exp.json @@ -42,7 +42,7 @@ "status": "0x1", "cumulativeGasUsed": "0xa865", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "logs": null, + "logs": [], "transactionHash": "0x7508d7139d002a4b3a26a4f12dec0d87cb46075c78bf77a38b569a133b509262", "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0xa865", diff --git a/cmd/evm/testdata/29/exp.json b/cmd/evm/testdata/29/exp.json index d6fd37a097..ee64e2f8f6 100644 --- a/cmd/evm/testdata/29/exp.json +++ b/cmd/evm/testdata/29/exp.json @@ -40,7 +40,7 @@ "status": "0x1", "cumulativeGasUsed": "0x5208", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "logs": null, + "logs": [], "transactionHash": "0x84f70aba406a55628a0620f26d260f90aeb6ccc55fed6ec2ac13dd4f727032ed", "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x5208", diff --git a/cmd/evm/testdata/3/exp.json b/cmd/evm/testdata/3/exp.json index 35113b303e..71cdbf2614 100644 --- a/cmd/evm/testdata/3/exp.json +++ b/cmd/evm/testdata/3/exp.json @@ -38,7 +38,7 @@ "status": "0x1", "cumulativeGasUsed": "0x521f", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "logs": null, + "logs": [], "transactionHash": "0x72fadbef39cd251a437eea619cfeda752271a5faaaa2147df012e112159ffb81", "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x521f", diff --git a/cmd/evm/testdata/30/exp.json b/cmd/evm/testdata/30/exp.json index c1443532d7..ca3c9655a6 100644 --- a/cmd/evm/testdata/30/exp.json +++ b/cmd/evm/testdata/30/exp.json @@ -39,7 +39,7 @@ "status": "0x1", "cumulativeGasUsed": "0x5208", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "logs": null, + "logs": [], "transactionHash": "0xa98a24882ea90916c6a86da650fbc6b14238e46f0af04a131ce92be897507476", "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x5208", @@ -68,7 +68,7 @@ "status": "0x1", "cumulativeGasUsed": "0xa410", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "logs": null, + "logs": [], "transactionHash": "0x36bad80acce7040c45fd32764b5c2b2d2e6f778669fb41791f73f546d56e739a", "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x5208", diff --git a/cmd/evm/testdata/33/exp.json b/cmd/evm/testdata/33/exp.json index 5d09ce19b1..564807ab9d 100644 --- a/cmd/evm/testdata/33/exp.json +++ b/cmd/evm/testdata/33/exp.json @@ -58,7 +58,7 @@ "root": "0x", "status": "0x1", "cumulativeGasUsed": "0x15fa9", - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","logs": null,"transactionHash": "0x0417aab7c1d8a3989190c3167c132876ce9b8afd99262c5a0f9d06802de3d7ef", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","logs": [],"transactionHash": "0x0417aab7c1d8a3989190c3167c132876ce9b8afd99262c5a0f9d06802de3d7ef", "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x15fa9", "effectiveGasPrice": null, diff --git a/cmd/fetchpayload/main.go b/cmd/fetchpayload/main.go new file mode 100644 index 0000000000..eafc05fbe8 --- /dev/null +++ b/cmd/fetchpayload/main.go @@ -0,0 +1,177 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +// fetchpayload queries an Ethereum node over RPC, fetches a block and its +// execution witness, and writes the combined Payload (ChainID + Block + +// Witness) to disk in the format consumed by cmd/keeper. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "math/big" + "os" + "path/filepath" + "strings" + "time" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/stateless" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/rpc" +) + +// Payload is duplicated from cmd/keeper/main.go (package main, not importable). +type Payload struct { + ChainID uint64 + Block *types.Block + Witness *stateless.Witness +} + +func main() { + var ( + rpcURL = flag.String("rpc", "http://localhost:8545", "RPC endpoint URL") + blockArg = flag.String("block", "latest", `Block number: decimal, 0x-hex, or "latest"`) + format = flag.String("format", "rlp", "Comma-separated output formats: rlp, hex, json") + outDir = flag.String("out", "", "Output directory (default: current directory)") + ) + flag.Parse() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Parse block number (nil means "latest" in ethclient). + blockNum, err := parseBlockNumber(*blockArg) + if err != nil { + fatal("invalid block number %q: %v", *blockArg, err) + } + + // Connect to the node. + client, err := ethclient.DialContext(ctx, *rpcURL) + if err != nil { + fatal("failed to connect to %s: %v", *rpcURL, err) + } + defer client.Close() + + chainID, err := client.ChainID(ctx) + if err != nil { + fatal("failed to get chain ID: %v", err) + } + + // Fetch the block first so we have a concrete number for the witness call, + // avoiding a race where "latest" advances between the two RPCs. + block, err := client.BlockByNumber(ctx, blockNum) + if err != nil { + fatal("failed to fetch block: %v", err) + } + fmt.Printf("Fetched block %d (%#x)\n", block.NumberU64(), block.Hash()) + + // Fetch the execution witness via the debug namespace. + var extWitness stateless.ExtWitness + err = client.Client().CallContext(ctx, &extWitness, "debug_executionWitness", rpc.BlockNumber(block.NumberU64())) + if err != nil { + fatal("failed to fetch execution witness: %v", err) + } + + witness := new(stateless.Witness) + err = witness.FromExtWitness(&extWitness) + if err != nil { + fatal("failed to convert witness: %v", err) + } + + payload := Payload{ + ChainID: chainID.Uint64(), + Block: block, + Witness: witness, + } + + // Encode payload as RLP (shared by "rlp" and "hex" formats). + rlpBytes, err := rlp.EncodeToBytes(payload) + if err != nil { + fatal("failed to RLP-encode payload: %v", err) + } + + // Write one output file per requested format. + blockHex := fmt.Sprintf("%x", block.NumberU64()) + for f := range strings.SplitSeq(*format, ",") { + f = strings.TrimSpace(f) + outPath := filepath.Join(*outDir, fmt.Sprintf("%s_payload.%s", blockHex, f)) + + var data []byte + switch f { + case "rlp": + data = rlpBytes + case "hex": + data = []byte(hexutil.Encode(rlpBytes)) + case "json": + data, err = marshalJSONPayload(chainID, block, &extWitness) + if err != nil { + fatal("failed to JSON-encode payload: %v", err) + } + default: + fatal("unknown format %q (valid: rlp, hex, json)", f) + } + + if err := os.WriteFile(outPath, data, 0644); err != nil { + fatal("failed to write %s: %v", outPath, err) + } + fmt.Printf("Wrote %s (%d bytes)\n", outPath, len(data)) + } +} + +// parseBlockNumber converts a CLI string to *big.Int. +// Returns nil for "latest" (ethclient convention for the head block). +func parseBlockNumber(s string) (*big.Int, error) { + if strings.EqualFold(s, "latest") { + return nil, nil + } + n := new(big.Int) + if strings.HasPrefix(s, "0x") || strings.HasPrefix(s, "0X") { + if _, ok := n.SetString(s[2:], 16); !ok { + return nil, fmt.Errorf("invalid hex number") + } + return n, nil + } + if _, ok := n.SetString(s, 10); !ok { + return nil, fmt.Errorf("invalid decimal number") + } + return n, nil +} + +// jsonPayload is a JSON-friendly representation of Payload. It uses ExtWitness +// instead of the internal Witness (which has no JSON marshaling). +type jsonPayload struct { + ChainID uint64 `json:"chainId"` + Block *types.Block `json:"block"` + Witness *stateless.ExtWitness `json:"witness"` +} + +func marshalJSONPayload(chainID *big.Int, block *types.Block, ext *stateless.ExtWitness) ([]byte, error) { + return json.MarshalIndent(jsonPayload{ + ChainID: chainID.Uint64(), + Block: block, + Witness: ext, + }, "", " ") +} + +func fatal(format string, args ...any) { + fmt.Fprintf(os.Stderr, format+"\n", args...) + os.Exit(1) +} diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index c5145bbfb7..1084100f39 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -20,6 +20,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "os" "path/filepath" "regexp" @@ -43,6 +44,8 @@ import ( "github.com/ethereum/go-ethereum/internal/debug" "github.com/ethereum/go-ethereum/internal/era" "github.com/ethereum/go-ethereum/internal/era/eradl" + "github.com/ethereum/go-ethereum/internal/era/execdb" + "github.com/ethereum/go-ethereum/internal/era/onedb" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -96,6 +99,7 @@ if one is set. Otherwise it prints the genesis from the datadir.`, utils.CacheNoPrefetchFlag, utils.CachePreimagesFlag, utils.NoCompactionFlag, + utils.LogSlowBlockFlag, utils.MetricsEnabledFlag, utils.MetricsEnabledExpensiveFlag, utils.MetricsHTTPFlag, @@ -107,6 +111,7 @@ if one is set. Otherwise it prints the genesis from the datadir.`, utils.MetricsInfluxDBUsernameFlag, utils.MetricsInfluxDBPasswordFlag, utils.MetricsInfluxDBTagsFlag, + utils.MetricsInfluxDBIntervalFlag, utils.MetricsInfluxDBTokenFlag, utils.MetricsInfluxDBBucketFlag, utils.MetricsInfluxDBOrganizationFlag, @@ -119,6 +124,8 @@ if one is set. Otherwise it prints the genesis from the datadir.`, utils.LogNoHistoryFlag, utils.LogExportCheckpointsFlag, utils.StateHistoryFlag, + utils.TrienodeHistoryFlag, + utils.TrienodeHistoryFullValueCheckpointFlag, }, utils.DatabaseFlags, debug.Flags), Before: func(ctx *cli.Context) error { flags.MigrateGlobalFlags(ctx) @@ -150,7 +157,7 @@ be gzipped.`, Name: "import-history", Usage: "Import an Era archive", ArgsUsage: "", - Flags: slices.Concat([]cli.Flag{utils.TxLookupLimitFlag, utils.TransactionHistoryFlag}, utils.DatabaseFlags, utils.NetworkFlags), + Flags: slices.Concat([]cli.Flag{utils.TxLookupLimitFlag, utils.TransactionHistoryFlag, utils.EraFormatFlag}, utils.DatabaseFlags, utils.NetworkFlags), Description: ` The import-history command will import blocks and their corresponding receipts from Era archives. @@ -161,7 +168,7 @@ from Era archives. Name: "export-history", Usage: "Export blockchain history to Era archives", ArgsUsage: " ", - Flags: utils.DatabaseFlags, + Flags: slices.Concat([]cli.Flag{utils.EraFormatFlag}, utils.DatabaseFlags), Description: ` The export-history command will export blocks and their corresponding receipts into Era archives. Eras are typically packaged in steps of 8192 blocks. @@ -201,13 +208,19 @@ This command dumps out the state for a given block (or latest, if none provided) pruneHistoryCommand = &cli.Command{ Action: pruneHistory, Name: "prune-history", - Usage: "Prune blockchain history (block bodies and receipts) up to the merge block", + Usage: "Prune blockchain history (block bodies and receipts) up to a specified point", ArgsUsage: "", - Flags: utils.DatabaseFlags, + Flags: slices.Concat(utils.DatabaseFlags, []cli.Flag{ + utils.ChainHistoryFlag, + }), Description: ` The prune-history command removes historical block bodies and receipts from the -blockchain database up to the merge block, while preserving block headers. This -helps reduce storage requirements for nodes that don't need full historical data.`, +blockchain database up to a specified point, while preserving block headers. This +helps reduce storage requirements for nodes that don't need full historical data. + +The --history.chain flag is required to specify the pruning target: + - postmerge: Prune up to the merge block. The node will keep the merge block and everything thereafter. + - postprague: Prune up to the Prague (Pectra) upgrade block. The node will keep the prague block and everything thereafter.`, } downloadEraCommand = &cli.Command{ @@ -295,7 +308,7 @@ func initGenesis(ctx *cli.Context) error { triedb := utils.MakeTrieDatabase(ctx, stack, chaindb, ctx.Bool(utils.CachePreimagesFlag.Name), false, genesis.IsVerkle()) defer triedb.Close() - _, hash, compatErr, err := core.SetupGenesisBlockWithOverride(chaindb, triedb, genesis, &overrides) + _, hash, compatErr, err := core.SetupGenesisBlockWithOverride(chaindb, triedb, genesis, &overrides, nil) if err != nil { utils.Fatalf("Failed to write genesis block: %v", err) } @@ -513,15 +526,27 @@ func importHistory(ctx *cli.Context) error { network = networks[0] } - if err := utils.ImportHistory(chain, dir, network); err != nil { + var ( + format = ctx.String(utils.EraFormatFlag.Name) + from func(era.ReadAtSeekCloser) (era.Era, error) + ) + switch format { + case "era1", "era": + from = onedb.From + case "erae": + from = execdb.From + default: + return fmt.Errorf("unknown --era.format %q (expected 'era1' or 'erae')", format) + } + if err := utils.ImportHistory(chain, dir, network, from); err != nil { return err } + fmt.Printf("Import done in %v\n", time.Since(start)) return nil } -// exportHistory exports chain history in Era archives at a specified -// directory. +// exportHistory exports chain history in Era archives at a specified directory. func exportHistory(ctx *cli.Context) error { if ctx.Args().Len() != 3 { utils.Fatalf("usage: %s", ctx.Command.ArgsUsage) @@ -547,10 +572,26 @@ func exportHistory(ctx *cli.Context) error { if head := chain.CurrentSnapBlock(); uint64(last) > head.Number.Uint64() { utils.Fatalf("Export error: block number %d larger than head block %d\n", uint64(last), head.Number.Uint64()) } - err := utils.ExportHistory(chain, dir, uint64(first), uint64(last), uint64(era.MaxEra1Size)) - if err != nil { + + var ( + format = ctx.String(utils.EraFormatFlag.Name) + filename func(network string, epoch int, root common.Hash) string + newBuilder func(w io.Writer) era.Builder + ) + switch format { + case "era1", "era": + newBuilder = func(w io.Writer) era.Builder { return onedb.NewBuilder(w) } + filename = func(network string, epoch int, root common.Hash) string { return onedb.Filename(network, epoch, root) } + case "erae": + newBuilder = func(w io.Writer) era.Builder { return execdb.NewBuilder(w) } + filename = func(network string, epoch int, root common.Hash) string { return execdb.Filename(network, epoch, root) } + default: + return fmt.Errorf("unknown archive format %q (use 'era1' or 'erae')", format) + } + if err := utils.ExportHistory(chain, dir, uint64(first), uint64(last), newBuilder, filename); err != nil { utils.Fatalf("Export error: %v\n", err) } + fmt.Printf("Export done in %v\n", time.Since(start)) return nil } @@ -668,47 +709,77 @@ func hashish(x string) bool { } func pruneHistory(ctx *cli.Context) error { + // Parse and validate the history mode flag. + if !ctx.IsSet(utils.ChainHistoryFlag.Name) { + return errors.New("--history.chain flag is required") + } + var mode history.HistoryMode + if err := mode.UnmarshalText([]byte(ctx.String(utils.ChainHistoryFlag.Name))); err != nil { + return err + } + if mode == history.KeepAll { + return errors.New("--history.chain=all is not valid for pruning. To restore history, use 'geth import-history'") + } + stack, _ := makeConfigNode(ctx) defer stack.Close() - // Open the chain database + // Open the chain database. chain, chaindb := utils.MakeChain(ctx, stack, false) defer chaindb.Close() defer chain.Stop() - // Determine the prune point. This will be the first PoS block. - prunePoint, ok := history.PrunePoints[chain.Genesis().Hash()] - if !ok || prunePoint == nil { - return errors.New("prune point not found") + // Determine the prune point based on the history mode. + genesisHash := chain.Genesis().Hash() + policy, err := history.NewPolicy(mode, genesisHash) + if err != nil { + return err + } + if policy.Target == nil { + return fmt.Errorf("prune point for %q not found for this network", mode.String()) } var ( - mergeBlock = prunePoint.BlockNumber - mergeBlockHash = prunePoint.BlockHash.Hex() + targetBlock = policy.Target.BlockNumber + targetBlockHash = policy.Target.BlockHash ) - // Check we're far enough past merge to ensure all data is in freezer + // Check the current freezer tail to see if pruning is needed/possible. + freezerTail, _ := chaindb.Tail() + if freezerTail > 0 { + if freezerTail == targetBlock { + log.Info("Database already pruned to target block", "tail", freezerTail) + return nil + } + if freezerTail > targetBlock { + // Database is pruned beyond the target - can't unprune. + return fmt.Errorf("database is already pruned to block %d, which is beyond target %d. Cannot unprune. To restore history, use 'geth import-history'", freezerTail, targetBlock) + } + // freezerTail < targetBlock: we can prune further, continue below. + } + + // Check we're far enough past the target to ensure all data is in freezer. currentHeader := chain.CurrentHeader() if currentHeader == nil { return errors.New("current header not found") } - if currentHeader.Number.Uint64() < mergeBlock+params.FullImmutabilityThreshold { - return fmt.Errorf("chain not far enough past merge block, need %d more blocks", - mergeBlock+params.FullImmutabilityThreshold-currentHeader.Number.Uint64()) + if currentHeader.Number.Uint64() < targetBlock+params.FullImmutabilityThreshold { + return fmt.Errorf("chain not far enough past target block %d, need %d more blocks", + targetBlock, targetBlock+params.FullImmutabilityThreshold-currentHeader.Number.Uint64()) } - // Double-check the prune block in db has the expected hash. - hash := rawdb.ReadCanonicalHash(chaindb, mergeBlock) - if hash != common.HexToHash(mergeBlockHash) { - return fmt.Errorf("merge block hash mismatch: got %s, want %s", hash.Hex(), mergeBlockHash) + // Double-check the target block in db has the expected hash. + hash := rawdb.ReadCanonicalHash(chaindb, targetBlock) + if hash != targetBlockHash { + return fmt.Errorf("target block hash mismatch: got %s, want %s", hash.Hex(), targetBlockHash.Hex()) } - log.Info("Starting history pruning", "head", currentHeader.Number, "tail", mergeBlock, "tailHash", mergeBlockHash) + log.Info("Starting history pruning", "head", currentHeader.Number, "target", targetBlock, "targetHash", targetBlockHash.Hex()) start := time.Now() - rawdb.PruneTransactionIndex(chaindb, mergeBlock) - if _, err := chaindb.TruncateTail(mergeBlock); err != nil { + rawdb.PruneTransactionIndex(chaindb, targetBlock) + if _, err := chaindb.TruncateTail(targetBlock); err != nil { return fmt.Errorf("failed to truncate ancient data: %v", err) } - log.Info("History pruning completed", "tail", mergeBlock, "elapsed", common.PrettyDuration(time.Since(start))) + log.Info("History pruning completed", "tail", targetBlock, "elapsed", common.PrettyDuration(time.Since(start))) // TODO(s1na): what if there is a crash between the two prune operations? diff --git a/cmd/geth/config.go b/cmd/geth/config.go index fcb315af97..720d1ef9fc 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -35,11 +35,11 @@ import ( "github.com/ethereum/go-ethereum/beacon/blsync" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/catalyst" "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/internal/flags" + "github.com/ethereum/go-ethereum/internal/telemetry/tracesetup" "github.com/ethereum/go-ethereum/internal/version" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" @@ -240,9 +240,15 @@ func makeFullNode(ctx *cli.Context) *node.Node { cfg.Eth.OverrideVerkle = &v } - // Start metrics export if enabled + // Start metrics export if enabled. utils.SetupMetrics(&cfg.Metrics) + // Setup OpenTelemetry reporting if enabled. + if err := tracesetup.SetupTelemetry(cfg.Node.OpenTelemetry, stack); err != nil { + utils.Fatalf("failed to setup OpenTelemetry: %v", err) + } + + // Add Ethereum service. backend, eth := utils.RegisterEthService(stack, &cfg.Eth) // Create gauge with geth system and build information @@ -273,11 +279,11 @@ func makeFullNode(ctx *cli.Context) *node.Node { // Configure synchronization override service var synctarget common.Hash if ctx.IsSet(utils.SyncTargetFlag.Name) { - hex := hexutil.MustDecode(ctx.String(utils.SyncTargetFlag.Name)) - if len(hex) != common.HashLength { - utils.Fatalf("invalid sync target length: have %d, want %d", len(hex), common.HashLength) + target := ctx.String(utils.SyncTargetFlag.Name) + if !common.IsHexHash(target) { + utils.Fatalf("sync target hash is not a valid hex hash: %s", target) } - synctarget = common.BytesToHash(hex) + synctarget = common.HexToHash(target) } utils.RegisterSyncOverrideService(stack, eth, synctarget, ctx.Bool(utils.ExitWhenSyncedFlag.Name)) @@ -371,6 +377,9 @@ func applyMetricConfig(ctx *cli.Context, cfg *gethConfig) { if ctx.IsSet(utils.MetricsInfluxDBTagsFlag.Name) { cfg.Metrics.InfluxDBTags = ctx.String(utils.MetricsInfluxDBTagsFlag.Name) } + if ctx.IsSet(utils.MetricsInfluxDBIntervalFlag.Name) { + cfg.Metrics.InfluxDBInterval = ctx.Duration(utils.MetricsInfluxDBIntervalFlag.Name) + } if ctx.IsSet(utils.MetricsEnableInfluxDBV2Flag.Name) { cfg.Metrics.EnableInfluxDBV2 = ctx.Bool(utils.MetricsEnableInfluxDBV2Flag.Name) } @@ -399,9 +408,9 @@ func applyMetricConfig(ctx *cli.Context, cfg *gethConfig) { ctx.IsSet(utils.MetricsInfluxDBBucketFlag.Name) if enableExport && v2FlagIsSet { - utils.Fatalf("Flags --influxdb.metrics.organization, --influxdb.metrics.token, --influxdb.metrics.bucket are only available for influxdb-v2") + utils.Fatalf("Flags --%s, --%s, --%s are only available for influxdb-v2", utils.MetricsInfluxDBOrganizationFlag.Name, utils.MetricsInfluxDBTokenFlag.Name, utils.MetricsInfluxDBBucketFlag.Name) } else if enableExportV2 && v1FlagIsSet { - utils.Fatalf("Flags --influxdb.metrics.username, --influxdb.metrics.password are only available for influxdb-v1") + utils.Fatalf("Flags --%s, --%s are only available for influxdb-v1", utils.MetricsInfluxDBUsernameFlag.Name, utils.MetricsInfluxDBPasswordFlag.Name) } } } diff --git a/cmd/geth/consolecmd_test.go b/cmd/geth/consolecmd_test.go index 42d6b43ade..c6282176e1 100644 --- a/cmd/geth/consolecmd_test.go +++ b/cmd/geth/consolecmd_test.go @@ -30,7 +30,7 @@ import ( ) const ( - ipcAPIs = "admin:1.0 debug:1.0 engine:1.0 eth:1.0 miner:1.0 net:1.0 rpc:1.0 txpool:1.0 web3:1.0" + ipcAPIs = "admin:1.0 debug:1.0 engine:1.0 eth:1.0 miner:1.0 net:1.0 rpc:1.0 testing:1.0 txpool:1.0 web3:1.0" httpAPIs = "eth:1.0 net:1.0 rpc:1.0 web3:1.0" ) @@ -39,8 +39,9 @@ const ( // child g gets a temporary data directory. func runMinimalGeth(t *testing.T, args ...string) *testgeth { // --holesky to make the 'writing genesis to disk' faster (no accounts) + // --networkid=1337 to avoid cache bump // --syncmode=full to avoid allocating fast sync bloom - allArgs := []string{"--holesky", "--authrpc.port", "0", "--syncmode=full", "--port", "0", + allArgs := []string{"--holesky", "--networkid", "1337", "--authrpc.port", "0", "--syncmode=full", "--port", "0", "--nat", "none", "--nodiscover", "--maxpeers", "0", "--cache", "64", "--datadir.minfreedisk", "0"} return runGeth(t, append(allArgs, args...)...) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index c57add0656..455dd05aca 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -19,6 +19,7 @@ package main import ( "bytes" "fmt" + "math" "os" "os/signal" "path/filepath" @@ -37,11 +38,11 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/internal/tablewriter" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/triedb" - "github.com/olekukonko/tablewriter" "github.com/urfave/cli/v2" ) @@ -52,7 +53,24 @@ var ( } removeChainDataFlag = &cli.BoolFlag{ Name: "remove.chain", - Usage: "If set, selects the state data for removal", + Usage: "If set, selects the chain data for removal", + } + inspectTrieTopFlag = &cli.IntFlag{ + Name: "top", + Usage: "Print the top N results per ranking category", + Value: 10, + } + inspectTrieDumpPathFlag = &cli.StringFlag{ + Name: "dump-path", + Usage: "Path for the trie statistics dump file", + } + inspectTrieSummarizeFlag = &cli.StringFlag{ + Name: "summarize", + Usage: "Summarize an existing trie dump file (skip trie traversal)", + } + inspectTrieContractFlag = &cli.StringFlag{ + Name: "contract", + Usage: "Inspect only the storage of the given contract address (skips full account trie walk)", } removedbCommand = &cli.Command{ @@ -75,6 +93,7 @@ Remove blockchain and state databases`, dbCompactCmd, dbGetCmd, dbDeleteCmd, + dbInspectTrieCmd, dbPutCmd, dbGetSlotsCmd, dbDumpFreezerIndex, @@ -93,6 +112,22 @@ Remove blockchain and state databases`, Usage: "Inspect the storage size for each type of data in the database", Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`, } + dbInspectTrieCmd = &cli.Command{ + Action: inspectTrie, + Name: "inspect-trie", + ArgsUsage: "", + Flags: slices.Concat([]cli.Flag{ + utils.ExcludeStorageFlag, + inspectTrieTopFlag, + utils.OutputFileFlag, + inspectTrieDumpPathFlag, + inspectTrieSummarizeFlag, + inspectTrieContractFlag, + }, utils.NetworkFlags, utils.DatabaseFlags), + Usage: "Print detailed trie information about the structure of account trie and storage tries.", + Description: `This commands iterates the entrie trie-backed state. If the 'blocknum' is not specified, +the latest block number will be used by default.`, + } dbCheckStateContentCmd = &cli.Command{ Action: checkStateContent, Name: "check-state-content", @@ -386,6 +421,88 @@ func checkStateContent(ctx *cli.Context) error { return nil } +func inspectTrie(ctx *cli.Context) error { + topN := ctx.Int(inspectTrieTopFlag.Name) + if topN <= 0 { + return fmt.Errorf("invalid --%s value %d (must be > 0)", inspectTrieTopFlag.Name, topN) + } + config := &trie.InspectConfig{ + NoStorage: ctx.Bool(utils.ExcludeStorageFlag.Name), + TopN: topN, + Path: ctx.String(utils.OutputFileFlag.Name), + } + + if summarizePath := ctx.String(inspectTrieSummarizeFlag.Name); summarizePath != "" { + if ctx.NArg() > 0 { + return fmt.Errorf("block number argument is not supported with --%s", inspectTrieSummarizeFlag.Name) + } + config.DumpPath = summarizePath + log.Info("Summarizing trie dump", "path", summarizePath, "top", topN) + return trie.Summarize(summarizePath, config) + } + if ctx.NArg() > 1 { + return fmt.Errorf("excessive number of arguments: %v", ctx.Command.ArgsUsage) + } + + stack, _ := makeConfigNode(ctx) + db := utils.MakeChainDatabase(ctx, stack, false) + defer stack.Close() + defer db.Close() + + var ( + trieRoot common.Hash + hash common.Hash + number uint64 + ) + switch { + case ctx.NArg() == 0 || ctx.Args().Get(0) == "latest": + head := rawdb.ReadHeadHeaderHash(db) + n, ok := rawdb.ReadHeaderNumber(db, head) + if !ok { + return fmt.Errorf("could not load head block hash") + } + number = n + case ctx.Args().Get(0) == "snapshot": + trieRoot = rawdb.ReadSnapshotRoot(db) + number = math.MaxUint64 + default: + var err error + number, err = strconv.ParseUint(ctx.Args().Get(0), 10, 64) + if err != nil { + return fmt.Errorf("failed to parse blocknum, Args[0]: %v, err: %v", ctx.Args().Get(0), err) + } + } + + if number != math.MaxUint64 { + hash = rawdb.ReadCanonicalHash(db, number) + if hash == (common.Hash{}) { + return fmt.Errorf("canonical hash for block %d not found", number) + } + blockHeader := rawdb.ReadHeader(db, hash, number) + trieRoot = blockHeader.Root + } + if trieRoot == (common.Hash{}) { + log.Error("Empty root hash") + } + + config.DumpPath = ctx.String(inspectTrieDumpPathFlag.Name) + if config.DumpPath == "" { + config.DumpPath = stack.ResolvePath("trie-dump.bin") + } + + triedb := utils.MakeTrieDatabase(ctx, stack, db, false, true, false) + defer triedb.Close() + + if contractAddr := ctx.String(inspectTrieContractFlag.Name); contractAddr != "" { + address := common.HexToAddress(contractAddr) + log.Info("Inspecting contract", "address", address, "root", trieRoot, "block", number) + return trie.InspectContract(triedb, db, trieRoot, address) + } + + log.Info("Inspecting trie", "root", trieRoot, "block", number, "dump", config.DumpPath, "top", topN) + return trie.Inspect(triedb, trieRoot, config) +} + func showDBStats(db ethdb.KeyValueStater) { stats, err := db.Stat() if err != nil { diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 109b36836a..b72cbb9885 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -22,7 +22,6 @@ import ( "os" "slices" "sort" - "strconv" "time" "github.com/ethereum/go-ethereum/accounts" @@ -94,6 +93,8 @@ var ( utils.LogNoHistoryFlag, utils.LogExportCheckpointsFlag, utils.StateHistoryFlag, + utils.TrienodeHistoryFlag, + utils.TrienodeHistoryFullValueCheckpointFlag, utils.LightKDFFlag, utils.EthRequiredBlocksFlag, utils.LegacyWhitelistFlag, // deprecated @@ -118,6 +119,7 @@ var ( utils.MinerGasPriceFlag, utils.MinerEtherbaseFlag, // deprecated utils.MinerExtraDataFlag, + utils.MinerMaxBlobsFlag, utils.MinerRecommitIntervalFlag, utils.MinerPendingFeeRecipientFlag, utils.MinerNewPayloadTimeoutFlag, // deprecated @@ -156,6 +158,7 @@ var ( utils.BeaconGenesisTimeFlag, utils.BeaconCheckpointFlag, utils.BeaconCheckpointFileFlag, + utils.LogSlowBlockFlag, }, utils.NetworkFlags, utils.DatabaseFlags) rpcFlags = []cli.Flag{ @@ -191,6 +194,14 @@ var ( utils.BatchResponseMaxSize, utils.RPCTxSyncDefaultTimeoutFlag, utils.RPCTxSyncMaxTimeoutFlag, + utils.RPCGlobalRangeLimitFlag, + utils.RPCTelemetryFlag, + utils.RPCTelemetryEndpointFlag, + utils.RPCTelemetryUserFlag, + utils.RPCTelemetryPasswordFlag, + utils.RPCTelemetryInstanceIDFlag, + utils.RPCTelemetryTagsFlag, + utils.RPCTelemetrySampleRatioFlag, } metricsFlags = []cli.Flag{ @@ -204,6 +215,7 @@ var ( utils.MetricsInfluxDBUsernameFlag, utils.MetricsInfluxDBPasswordFlag, utils.MetricsInfluxDBTagsFlag, + utils.MetricsInfluxDBIntervalFlag, utils.MetricsEnableInfluxDBV2Flag, utils.MetricsInfluxDBTokenFlag, utils.MetricsInfluxDBBucketFlag, @@ -239,7 +251,6 @@ func init() { javascriptCommand, // See misccmd.go: versionCommand, - versionCheckCommand, licenseCommand, // See config.go dumpConfigCommand, @@ -249,8 +260,6 @@ func init() { utils.ShowDeprecated, // See snapshot.go snapshotCommand, - // See verkle.go - verkleCommand, } if logTestCommand != nil { app.Commands = append(app.Commands, logTestCommand) @@ -306,18 +315,6 @@ func prepare(ctx *cli.Context) { case !ctx.IsSet(utils.NetworkIdFlag.Name): log.Info("Starting Geth on Ethereum mainnet...") } - // If we're a full node on mainnet without --cache specified, bump default cache allowance - if !ctx.IsSet(utils.CacheFlag.Name) && !ctx.IsSet(utils.NetworkIdFlag.Name) { - // Make sure we're not on any supported preconfigured testnet either - if !ctx.IsSet(utils.HoleskyFlag.Name) && - !ctx.IsSet(utils.SepoliaFlag.Name) && - !ctx.IsSet(utils.HoodiFlag.Name) && - !ctx.IsSet(utils.DeveloperFlag.Name) { - // Nope, we're really on mainnet. Bump that cache up! - log.Info("Bumping default cache on mainnet", "provided", ctx.Int(utils.CacheFlag.Name), "updated", 4096) - ctx.Set(utils.CacheFlag.Name, strconv.Itoa(4096)) - } - } } // geth is the main entry point into the system if no special subcommand is run. diff --git a/cmd/geth/misccmd.go b/cmd/geth/misccmd.go index 2d31f3abe7..f5c0d55ebb 100644 --- a/cmd/geth/misccmd.go +++ b/cmd/geth/misccmd.go @@ -27,16 +27,6 @@ import ( ) var ( - VersionCheckUrlFlag = &cli.StringFlag{ - Name: "check.url", - Usage: "URL to use when checking vulnerabilities", - Value: "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities.json", - } - VersionCheckVersionFlag = &cli.StringFlag{ - Name: "check.version", - Usage: "Version to check", - Value: version.ClientName(clientIdentifier), - } versionCommand = &cli.Command{ Action: printVersion, Name: "version", @@ -44,20 +34,6 @@ var ( ArgsUsage: " ", Description: ` The output of this command is supposed to be machine-readable. -`, - } - versionCheckCommand = &cli.Command{ - Action: versionCheck, - Flags: []cli.Flag{ - VersionCheckUrlFlag, - VersionCheckVersionFlag, - }, - Name: "version-check", - Usage: "Checks (online) for known Geth security vulnerabilities", - ArgsUsage: "", - Description: ` -The version-check command fetches vulnerability-information from https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities.json, -and displays information about any security vulnerabilities that affect the currently executing version. `, } licenseCommand = &cli.Command{ diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index fe0bf97db8..3f4c77e84d 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -36,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" "github.com/urfave/cli/v2" ) @@ -105,7 +106,9 @@ information about the specified address. Usage: "Traverse the state with given root hash and perform quick verification", ArgsUsage: "", Action: traverseState, - Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), + Flags: slices.Concat([]cli.Flag{ + utils.AccountFlag, + }, utils.NetworkFlags, utils.DatabaseFlags), Description: ` geth snapshot traverse-state will traverse the whole state from the given state root and will abort if any @@ -113,6 +116,8 @@ referenced trie node or contract code is missing. This command can be used for state integrity verification. The default checking target is the HEAD state. It's also usable without snapshot enabled. + +If --account is specified, only the storage trie of that account is traversed. `, }, { @@ -120,7 +125,9 @@ It's also usable without snapshot enabled. Usage: "Traverse the state with given root hash and perform detailed verification", ArgsUsage: "", Action: traverseRawState, - Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), + Flags: slices.Concat([]cli.Flag{ + utils.AccountFlag, + }, utils.NetworkFlags, utils.DatabaseFlags), Description: ` geth snapshot traverse-rawstate will traverse the whole state from the given root and will abort if any referenced @@ -129,6 +136,8 @@ verification. The default checking target is the HEAD state. It's basically iden to traverse-state, but the check granularity is smaller. It's also usable without snapshot enabled. + +If --account is specified, only the storage trie of that account is traversed. `, }, { @@ -276,6 +285,120 @@ func checkDanglingStorage(ctx *cli.Context) error { return snapshot.CheckDanglingStorage(db) } +// parseAccount parses the account flag value as either an address (20 bytes) +// or an account hash (32 bytes) and returns the hashed account key. +func parseAccount(input string) (common.Hash, error) { + switch len(input) { + case 40, 42: // address + return crypto.Keccak256Hash(common.HexToAddress(input).Bytes()), nil + case 64, 66: // hash + return common.HexToHash(input), nil + default: + return common.Hash{}, errors.New("malformed account address or hash") + } +} + +// lookupAccount resolves the account from the state trie using the given +// account hash. +func lookupAccount(accountHash common.Hash, tr *trie.Trie) (*types.StateAccount, error) { + accData, err := tr.Get(accountHash.Bytes()) + if err != nil { + return nil, fmt.Errorf("failed to get account %s: %w", accountHash, err) + } + if accData == nil { + return nil, fmt.Errorf("account not found: %s", accountHash) + } + var acc types.StateAccount + if err := rlp.DecodeBytes(accData, &acc); err != nil { + return nil, fmt.Errorf("invalid account data %s: %w", accountHash, err) + } + return &acc, nil +} + +func traverseStorage(id *trie.ID, db *triedb.Database, report bool, detail bool) error { + tr, err := trie.NewStateTrie(id, db) + if err != nil { + log.Error("Failed to open storage trie", "account", id.Owner, "root", id.Root, "err", err) + return err + } + var ( + slots int + nodes int + lastReport time.Time + start = time.Now() + ) + it, err := tr.NodeIterator(nil) + if err != nil { + log.Error("Failed to open storage iterator", "account", id.Owner, "root", id.Root, "err", err) + return err + } + logger := log.Debug + if report { + logger = log.Info + } + logger("Start traversing storage trie", "account", id.Owner, "storageRoot", id.Root) + + if !detail { + iter := trie.NewIterator(it) + for iter.Next() { + slots += 1 + if time.Since(lastReport) > time.Second*8 { + logger("Traversing storage", "account", id.Owner, "slots", slots, "elapsed", common.PrettyDuration(time.Since(start))) + lastReport = time.Now() + } + } + if iter.Err != nil { + log.Error("Failed to traverse storage trie", "root", id.Root, "err", iter.Err) + return iter.Err + } + logger("Storage is complete", "account", id.Owner, "slots", slots, "elapsed", common.PrettyDuration(time.Since(start))) + } else { + reader, err := db.NodeReader(id.StateRoot) + if err != nil { + log.Error("Failed to open state reader", "err", err) + return err + } + var ( + buffer = make([]byte, 32) + hasher = crypto.NewKeccakState() + ) + for it.Next(true) { + nodes += 1 + node := it.Hash() + + // Check the presence for non-empty hash node(embedded node doesn't + // have their own hash). + if node != (common.Hash{}) { + blob, _ := reader.Node(id.Owner, it.Path(), node) + if len(blob) == 0 { + log.Error("Missing trie node(storage)", "hash", node) + return errors.New("missing storage") + } + hasher.Reset() + hasher.Write(blob) + hasher.Read(buffer) + if !bytes.Equal(buffer, node.Bytes()) { + log.Error("Invalid trie node(storage)", "hash", node.Hex(), "value", blob) + return errors.New("invalid storage node") + } + } + if it.Leaf() { + slots += 1 + } + if time.Since(lastReport) > time.Second*8 { + logger("Traversing storage", "account", id.Owner, "nodes", nodes, "slots", slots, "elapsed", common.PrettyDuration(time.Since(start))) + lastReport = time.Now() + } + } + if err := it.Error(); err != nil { + log.Error("Failed to traverse storage trie", "root", id.Root, "err", err) + return err + } + logger("Storage is complete", "account", id.Owner, "nodes", nodes, "slots", slots, "elapsed", common.PrettyDuration(time.Since(start))) + } + return nil +} + // traverseState is a helper function used for pruning verification. // Basically it just iterates the trie, ensure all nodes and associated // contract codes are present. @@ -313,6 +436,30 @@ func traverseState(ctx *cli.Context) error { root = headBlock.Root() log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) } + // If --account is specified, only traverse the storage trie of that account. + if accountStr := ctx.String(utils.AccountFlag.Name); accountStr != "" { + accountHash, err := parseAccount(accountStr) + if err != nil { + log.Error("Failed to parse account", "err", err) + return err + } + // Use raw trie since the account key is already hashed. + t, err := trie.New(trie.StateTrieID(root), triedb) + if err != nil { + log.Error("Failed to open state trie", "root", root, "err", err) + return err + } + acc, err := lookupAccount(accountHash, t) + if err != nil { + log.Error("Failed to look up account", "hash", accountHash, "err", err) + return err + } + if acc.Root == types.EmptyRootHash { + log.Info("Account has no storage", "hash", accountHash) + return nil + } + return traverseStorage(trie.StorageTrieID(root, accountHash, acc.Root), triedb, true, false) + } t, err := trie.NewStateTrie(trie.StateTrieID(root), triedb) if err != nil { log.Error("Failed to open trie", "root", root, "err", err) @@ -339,30 +486,10 @@ func traverseState(ctx *cli.Context) error { return err } if acc.Root != types.EmptyRootHash { - id := trie.StorageTrieID(root, common.BytesToHash(accIter.Key), acc.Root) - storageTrie, err := trie.NewStateTrie(id, triedb) + err := traverseStorage(trie.StorageTrieID(root, common.BytesToHash(accIter.Key), acc.Root), triedb, false, false) if err != nil { - log.Error("Failed to open storage trie", "root", acc.Root, "err", err) return err } - storageIt, err := storageTrie.NodeIterator(nil) - if err != nil { - log.Error("Failed to open storage iterator", "root", acc.Root, "err", err) - return err - } - storageIter := trie.NewIterator(storageIt) - for storageIter.Next() { - slots += 1 - - if time.Since(lastReport) > time.Second*8 { - log.Info("Traversing state", "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start))) - lastReport = time.Now() - } - } - if storageIter.Err != nil { - log.Error("Failed to traverse storage trie", "root", acc.Root, "err", storageIter.Err) - return storageIter.Err - } } if !bytes.Equal(acc.CodeHash, types.EmptyCodeHash.Bytes()) { if !rawdb.HasCode(chaindb, common.BytesToHash(acc.CodeHash)) { @@ -422,6 +549,30 @@ func traverseRawState(ctx *cli.Context) error { root = headBlock.Root() log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) } + // If --account is specified, only traverse the storage trie of that account. + if accountStr := ctx.String(utils.AccountFlag.Name); accountStr != "" { + accountHash, err := parseAccount(accountStr) + if err != nil { + log.Error("Failed to parse account", "err", err) + return err + } + // Use raw trie since the account key is already hashed. + t, err := trie.New(trie.StateTrieID(root), triedb) + if err != nil { + log.Error("Failed to open state trie", "root", root, "err", err) + return err + } + acc, err := lookupAccount(accountHash, t) + if err != nil { + log.Error("Failed to look up account", "hash", accountHash, "err", err) + return err + } + if acc.Root == types.EmptyRootHash { + log.Info("Account has no storage", "hash", accountHash) + return nil + } + return traverseStorage(trie.StorageTrieID(root, accountHash, acc.Root), triedb, true, true) + } t, err := trie.NewStateTrie(trie.StateTrieID(root), triedb) if err != nil { log.Error("Failed to open trie", "root", root, "err", err) @@ -477,50 +628,10 @@ func traverseRawState(ctx *cli.Context) error { return errors.New("invalid account") } if acc.Root != types.EmptyRootHash { - id := trie.StorageTrieID(root, common.BytesToHash(accIter.LeafKey()), acc.Root) - storageTrie, err := trie.NewStateTrie(id, triedb) - if err != nil { - log.Error("Failed to open storage trie", "root", acc.Root, "err", err) - return errors.New("missing storage trie") - } - storageIter, err := storageTrie.NodeIterator(nil) + err := traverseStorage(trie.StorageTrieID(root, common.BytesToHash(accIter.LeafKey()), acc.Root), triedb, false, true) if err != nil { - log.Error("Failed to open storage iterator", "root", acc.Root, "err", err) return err } - for storageIter.Next(true) { - nodes += 1 - node := storageIter.Hash() - - // Check the presence for non-empty hash node(embedded node doesn't - // have their own hash). - if node != (common.Hash{}) { - blob, _ := reader.Node(common.BytesToHash(accIter.LeafKey()), storageIter.Path(), node) - if len(blob) == 0 { - log.Error("Missing trie node(storage)", "hash", node) - return errors.New("missing storage") - } - hasher.Reset() - hasher.Write(blob) - hasher.Read(got) - if !bytes.Equal(got, node.Bytes()) { - log.Error("Invalid trie node(storage)", "hash", node.Hex(), "value", blob) - return errors.New("invalid storage node") - } - } - // Bump the counter if it's leaf node. - if storageIter.Leaf() { - slots += 1 - } - if time.Since(lastReport) > time.Second*8 { - log.Info("Traversing state", "nodes", nodes, "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start))) - lastReport = time.Now() - } - } - if storageIter.Error() != nil { - log.Error("Failed to traverse storage trie", "root", acc.Root, "err", storageIter.Error()) - return storageIter.Error() - } } if !bytes.Equal(acc.CodeHash, types.EmptyCodeHash.Bytes()) { if !rawdb.HasCode(chaindb, common.BytesToHash(acc.CodeHash)) { @@ -643,11 +754,11 @@ func snapshotExportPreimages(ctx *cli.Context) error { var root common.Hash if ctx.NArg() > 1 { - rootBytes := common.FromHex(ctx.Args().Get(1)) - if len(rootBytes) != common.HashLength { + hash := ctx.Args().Get(1) + if !common.IsHexHash(hash) { return fmt.Errorf("invalid hash: %s", ctx.Args().Get(1)) } - root = common.BytesToHash(rootBytes) + root = common.HexToHash(hash) } else { headBlock := rawdb.ReadHeadBlock(chaindb) if headBlock == nil { diff --git a/cmd/geth/testdata/vcheck/data.json b/cmd/geth/testdata/vcheck/data.json deleted file mode 100644 index e52fd84e67..0000000000 --- a/cmd/geth/testdata/vcheck/data.json +++ /dev/null @@ -1,202 +0,0 @@ -[ - { - "name": "CorruptedDAG", - "uid": "GETH-2020-01", - "summary": "Mining nodes will generate erroneous PoW on epochs > `385`.", - "description": "A mining flaw could cause miners to erroneously calculate PoW, due to an index overflow, if DAG size is exceeding the maximum 32 bit unsigned value.\n\nThis occurred on the ETC chain on 2020-11-06. This is likely to trigger for ETH mainnet around block `11550000`/epoch `385`, slated to occur early January 2021.\n\nThis issue is relevant only for miners, non-mining nodes are unaffected, since non-mining nodes use a smaller verification cache instead of a full DAG.", - "links": [ - "https://github.com/ethereum/go-ethereum/pull/21793", - "https://blog.ethereum.org/2020/11/12/geth-security-release", - "https://github.com/ethereum/go-ethereum/commit/567d41d9363706b4b13ce0903804e8acf214af49", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-v592-xf75-856p" - ], - "introduced": "v1.6.0", - "fixed": "v1.9.24", - "published": "2020-11-12", - "severity": "Medium", - "CVE": "CVE-2020-26240", - "check": "Geth\\/v1\\.(6|7|8)\\..*|Geth\\/v1\\.9\\.\\d-.*|Geth\\/v1\\.9\\.1.*|Geth\\/v1\\.9\\.2(0|1|2|3)-.*" - }, - { - "name": "Denial of service due to Go CVE-2020-28362", - "uid": "GETH-2020-02", - "summary": "A denial-of-service issue can be used to crash Geth nodes during block processing, due to an underlying bug in Go (CVE-2020-28362) versions < `1.15.5`, or `<1.14.12`", - "description": "The DoS issue can be used to crash all Geth nodes during block processing, the effects of which would be that a major part of the Ethereum network went offline.\n\nOutside of Go-Ethereum, the issue is most likely relevant for all forks of Geth (such as TurboGeth or ETC’s core-geth) which is built with versions of Go which contains the vulnerability.", - "links": [ - "https://blog.ethereum.org/2020/11/12/geth-security-release", - "https://groups.google.com/g/golang-announce/c/NpBGTTmKzpM", - "https://github.com/golang/go/issues/42552", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-m6gx-rhvj-fh52" - ], - "introduced": "v0.0.0", - "fixed": "v1.9.24", - "published": "2020-11-12", - "severity": "Critical", - "CVE": "CVE-2020-28362", - "check": "Geth.*\\/go1\\.(11(.*)|12(.*)|13(.*)|14|14\\.(\\d|10|11|)|15|15\\.[0-4])$" - }, - { - "name": "ShallowCopy", - "uid": "GETH-2020-03", - "summary": "A consensus flaw in Geth, related to `datacopy` precompile", - "description": "Geth erroneously performed a 'shallow' copy when the precompiled `datacopy` (at `0x00...04`) was invoked. An attacker could deploy a contract that uses the shallow copy to corrupt the contents of the `RETURNDATA`, thus causing a consensus failure.", - "links": [ - "https://blog.ethereum.org/2020/11/12/geth-security-release", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-69v6-xc2j-r2jf" - ], - "introduced": "v1.9.7", - "fixed": "v1.9.17", - "published": "2020-11-12", - "severity": "Critical", - "CVE": "CVE-2020-26241", - "check": "Geth\\/v1\\.9\\.(7|8|9|10|11|12|13|14|15|16).*$" - }, - { - "name": "Geth DoS via MULMOD", - "uid": "GETH-2020-04", - "summary": "A denial-of-service issue can be used to crash Geth nodes during block processing", - "description": "Affected versions suffer from a vulnerability which can be exploited through the `MULMOD` operation, by specifying a modulo of `0`: `mulmod(a,b,0)`, causing a `panic` in the underlying library. \nThe crash was in the `uint256` library, where a buffer [underflowed](https://github.com/holiman/uint256/blob/4ce82e695c10ddad57215bdbeafb68b8c5df2c30/uint256.go#L442).\n\n\tif `d == 0`, `dLen` remains `0`\n\nand https://github.com/holiman/uint256/blob/4ce82e695c10ddad57215bdbeafb68b8c5df2c30/uint256.go#L451 will try to access index `[-1]`.\n\nThe `uint256` library was first merged in this [commit](https://github.com/ethereum/go-ethereum/commit/cf6674539c589f80031f3371a71c6a80addbe454), on 2020-06-08. \nExploiting this vulnerabilty would cause all vulnerable nodes to drop off the network. \n\nThe issue was brought to our attention through a [bug report](https://github.com/ethereum/go-ethereum/issues/21367), showing a `panic` occurring on sync from genesis on the Ropsten network.\n \nIt was estimated that the least obvious way to fix this would be to merge the fix into `uint256`, make a new release of that library and then update the geth-dependency.\n", - "links": [ - "https://blog.ethereum.org/2020/11/12/geth-security-release", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-jm5c-rv3w-w83m", - "https://github.com/holiman/uint256/releases/tag/v1.1.1", - "https://github.com/holiman/uint256/pull/80", - "https://github.com/ethereum/go-ethereum/pull/21368" - ], - "introduced": "v1.9.16", - "fixed": "v1.9.18", - "published": "2020-11-12", - "severity": "Critical", - "CVE": "CVE-2020-26242", - "check": "Geth\\/v1\\.9.(16|17).*$" - }, - { - "name": "LES Server DoS via GetProofsV2", - "uid": "GETH-2020-05", - "summary": "A DoS vulnerability can make a LES server crash.", - "description": "A DoS vulnerability can make a LES server crash via malicious GetProofsV2 request from a connected LES client.\n\nThe vulnerability was patched in #21896.\n\nThis vulnerability only concern users explicitly running geth as a light server", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-r33q-22hv-j29q", - "https://github.com/ethereum/go-ethereum/pull/21896" - ], - "introduced": "v1.8.0", - "fixed": "v1.9.25", - "published": "2020-12-10", - "severity": "Medium", - "CVE": "CVE-2020-26264", - "check": "(Geth\\/v1\\.8\\.*)|(Geth\\/v1\\.9\\.\\d-.*)|(Geth\\/v1\\.9\\.1\\d-.*)|(Geth\\/v1\\.9\\.(20|21|22|23|24)-.*)$" - }, - { - "name": "SELFDESTRUCT-recreate consensus flaw", - "uid": "GETH-2020-06", - "introduced": "v1.9.4", - "fixed": "v1.9.20", - "summary": "A consensus-vulnerability in Geth could cause a chain split, where vulnerable versions refuse to accept the canonical chain.", - "description": "A flaw was repoted at 2020-08-11 by John Youngseok Yang (Software Platform Lab), where a particular sequence of transactions could cause a consensus failure.\n\n- Tx 1:\n - `sender` invokes `caller`.\n - `caller` invokes `0xaa`. `0xaa` has 3 wei, does a self-destruct-to-self\n - `caller` does a `1 wei` -call to `0xaa`, who thereby has 1 wei (the code in `0xaa` still executed, since the tx is still ongoing, but doesn't redo the selfdestruct, it takes a different path if callvalue is non-zero)\n\n-Tx 2:\n - `sender` does a 5-wei call to 0xaa. No exec (since no code). \n\nIn geth, the result would be that `0xaa` had `6 wei`, whereas OE reported (correctly) `5` wei. Furthermore, in geth, if the second tx was not executed, the `0xaa` would be destructed, resulting in `0 wei`. Thus obviously wrong. \n\nIt was determined that the root cause was this [commit](https://github.com/ethereum/go-ethereum/commit/223b950944f494a5b4e0957fd9f92c48b09037ad) from [this PR](https://github.com/ethereum/go-ethereum/pull/19953). The semantics of `createObject` was subtly changd, into returning a non-nil object (with `deleted=true`) where it previously did not if the account had been destructed. This return value caused the new object to inherit the old `balance`.\n", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-xw37-57qp-9mm4" - ], - "published": "2020-12-10", - "severity": "High", - "CVE": "CVE-2020-26265", - "check": "(Geth\\/v1\\.9\\.(4|5|6|7|8|9)-.*)|(Geth\\/v1\\.9\\.1\\d-.*)$" - }, - { - "name": "Not ready for London upgrade", - "uid": "GETH-2021-01", - "summary": "The client is not ready for the 'London' technical upgrade, and will deviate from the canonical chain when the London upgrade occurs (at block '12965000' around August 4, 2021.", - "description": "At (or around) August 4, Ethereum will undergo a technical upgrade called 'London'. Clients not upgraded will fail to progress on the canonical chain.", - "links": [ - "https://github.com/ethereum/eth1.0-specs/blob/master/network-upgrades/mainnet-upgrades/london.md", - "https://notes.ethereum.org/@timbeiko/ropsten-postmortem" - ], - "introduced": "v1.10.1", - "fixed": "v1.10.6", - "published": "2021-07-22", - "severity": "High", - "check": "(Geth\\/v1\\.10\\.(1|2|3|4|5)-.*)$" - }, - { - "name": "RETURNDATA corruption via datacopy", - "uid": "GETH-2021-02", - "summary": "A consensus-flaw in the Geth EVM could cause a node to deviate from the canonical chain.", - "description": "A memory-corruption bug within the EVM can cause a consensus error, where vulnerable nodes obtain a different `stateRoot` when processing a maliciously crafted transaction. This, in turn, would lead to the chain being split: mainnet splitting in two forks.\n\nAll Geth versions supporting the London hard fork are vulnerable (the bug is older than London), so all users should update.\n\nThis bug was exploited on Mainnet at block 13107518.\n\nCredits for the discovery go to @guidovranken (working for Sentnl during an audit of the Telos EVM) and reported via bounty@ethereum.org.", - "links": [ - "https://github.com/ethereum/go-ethereum/blob/master/docs/postmortems/2021-08-22-split-postmortem.md", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-9856-9gg9-qcmq", - "https://github.com/ethereum/go-ethereum/releases/tag/v1.10.8" - ], - "introduced": "v1.10.0", - "fixed": "v1.10.8", - "published": "2021-08-24", - "severity": "High", - "CVE": "CVE-2021-39137", - "check": "(Geth\\/v1\\.10\\.(0|1|2|3|4|5|6|7)-.*)$" - }, - { - "name": "DoS via malicious `snap/1` request", - "uid": "GETH-2021-03", - "summary": "A vulnerable node is susceptible to crash when processing a maliciously crafted message from a peer, via the snap/1 protocol. The crash can be triggered by sending a malicious snap/1 GetTrieNodes package.", - "description": "The `snap/1` protocol handler contains two vulnerabilities related to the `GetTrieNodes` packet, which can be exploited to crash the node. Full details are available at the Github security [advisory](https://github.com/ethereum/go-ethereum/security/advisories/GHSA-59hh-656j-3p7v)", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-59hh-656j-3p7v", - "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities", - "https://github.com/ethereum/go-ethereum/pull/23657" - ], - "introduced": "v1.10.0", - "fixed": "v1.10.9", - "published": "2021-10-24", - "severity": "Medium", - "CVE": "CVE-2021-41173", - "check": "(Geth\\/v1\\.10\\.(0|1|2|3|4|5|6|7|8)-.*)$" - }, - { - "name": "DoS via malicious p2p message", - "uid": "GETH-2022-01", - "summary": "A vulnerable node can crash via p2p messages sent from an attacker node, if running with non-default log options.", - "description": "A vulnerable node, if configured to use high verbosity logging, can be made to crash when handling specially crafted p2p messages sent from an attacker node. Full details are available at the Github security [advisory](https://github.com/ethereum/go-ethereum/security/advisories/GHSA-wjxw-gh3m-7pm5)", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-wjxw-gh3m-7pm5", - "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities", - "https://github.com/ethereum/go-ethereum/pull/24507" - ], - "introduced": "v1.10.0", - "fixed": "v1.10.17", - "published": "2022-05-11", - "severity": "Low", - "CVE": "CVE-2022-29177", - "check": "(Geth\\/v1\\.10\\.(0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16)-.*)$" - }, - { - "name": "DoS via malicious p2p message", - "uid": "GETH-2023-01", - "summary": "A vulnerable node can be made to consume unbounded amounts of memory when handling specially crafted p2p messages sent from an attacker node.", - "description": "The p2p handler spawned a new goroutine to respond to ping requests. By flooding a node with ping requests, an unbounded number of goroutines can be created, leading to resource exhaustion and potentially crash due to OOM.", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-ppjg-v974-84cm", - "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities" - ], - "introduced": "v1.10.0", - "fixed": "v1.12.1", - "published": "2023-09-06", - "severity": "High", - "CVE": "CVE-2023-40591", - "check": "(Geth\\/v1\\.(10|11)\\..*)|(Geth\\/v1\\.12\\.0-.*)$" - }, - { - "name": "DoS via malicious p2p message", - "uid": "GETH-2024-01", - "summary": "A vulnerable node can be made to consume very large amounts of memory when handling specially crafted p2p messages sent from an attacker node.", - "description": "A vulnerable node can be made to consume very large amounts of memory when handling specially crafted p2p messages sent from an attacker node. Full details will be available at the Github security [advisory](https://github.com/ethereum/go-ethereum/security/advisories/GHSA-4xc9-8hmq-j652)", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-4xc9-8hmq-j652", - "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities" - ], - "introduced": "v1.10.0", - "fixed": "v1.13.15", - "published": "2024-05-06", - "severity": "High", - "CVE": "CVE-2024-32972", - "check": "(Geth\\/v1\\.(10|11|12)\\..*)|(Geth\\/v1\\.13\\.\\d-.*)|(Geth\\/v1\\.13\\.1(0|1|2|3|4)-.*)$" - } -] diff --git a/cmd/geth/testdata/vcheck/minisig-sigs-new/data.json.minisig b/cmd/geth/testdata/vcheck/minisig-sigs-new/data.json.minisig deleted file mode 100644 index 987ffe92bb..0000000000 --- a/cmd/geth/testdata/vcheck/minisig-sigs-new/data.json.minisig +++ /dev/null @@ -1,4 +0,0 @@ -untrusted comment: signature from minisign secret key -RUQkliYstQBOKHklFEYCUjepz81dyUuDmIAxjAvXa+icjGuKcjtVfV06G7qfOMSpplS5EcntU12n+AnGNyuOM8zIctaIWcfG2w0= -trusted comment: timestamp:1752094689 file:data.json hashed -u2e4wo4HBTU6viQTSY/NVBHoWoPFJnnTvLZS0FYl3JdvSOYi6+qpbEsDhAIFqq/n8VmlS/fPqqf7vKCNiAgjAA== diff --git a/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.1 b/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.1 deleted file mode 100644 index 6b6aa900e3..0000000000 --- a/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.1 +++ /dev/null @@ -1,4 +0,0 @@ -untrusted comment: signature from minisign secret key -RWQkliYstQBOKNoyq2O98hPmeVJQ6ShQLM58+4n0gkY0y0trFMDAsHuN/l4IyHfh8dDQ1ry0+IuZVrf/i8M/P3YFzFfAymDYCQ0= -trusted comment: timestamp:1752094703 file:data.json -cNyq3ZGlqo785HtWODb9ejWqF0HhSeXuLGXzC7z1IhnDrBObWBJngYd3qBG1dQcYlHQ+bgB/On5mSyMFn4UoCQ== diff --git a/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.2 b/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.2 deleted file mode 100644 index 704437de39..0000000000 --- a/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.2 +++ /dev/null @@ -1,4 +0,0 @@ -untrusted comment: Here's a comment -RWQkliYstQBOKNoyq2O98hPmeVJQ6ShQLM58+4n0gkY0y0trFMDAsHuN/l4IyHfh8dDQ1ry0+IuZVrf/i8M/P3YFzFfAymDYCQ0= -trusted comment: Here's a trusted comment -dL7lO8sqFFCOXJO/u8SgoDk2nlXGWPRDbOTJkChMbmtUp9PB7sG831basXkZ/0CQ/l/vG7AbPyMNEVZyJn5NCg== diff --git a/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.3 b/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.3 deleted file mode 100644 index 806cd07316..0000000000 --- a/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.3 +++ /dev/null @@ -1,4 +0,0 @@ -untrusted comment: One more (untrusted™) comment -RWQkliYstQBOKNoyq2O98hPmeVJQ6ShQLM58+4n0gkY0y0trFMDAsHuN/l4IyHfh8dDQ1ry0+IuZVrf/i8M/P3YFzFfAymDYCQ0= -trusted comment: Here's a trusted comment -dL7lO8sqFFCOXJO/u8SgoDk2nlXGWPRDbOTJkChMbmtUp9PB7sG831basXkZ/0CQ/l/vG7AbPyMNEVZyJn5NCg== diff --git a/cmd/geth/testdata/vcheck/minisign.pub b/cmd/geth/testdata/vcheck/minisign.pub deleted file mode 100644 index 183dce5f6b..0000000000 --- a/cmd/geth/testdata/vcheck/minisign.pub +++ /dev/null @@ -1,2 +0,0 @@ -untrusted comment: minisign public key 284E00B52C269624 -RWQkliYstQBOKOdtClfgC3IypIPX6TAmoEi7beZ4gyR3wsaezvqOMWsp diff --git a/cmd/geth/testdata/vcheck/minisign.sec b/cmd/geth/testdata/vcheck/minisign.sec deleted file mode 100644 index 5c50715b20..0000000000 --- a/cmd/geth/testdata/vcheck/minisign.sec +++ /dev/null @@ -1,2 +0,0 @@ -untrusted comment: minisign encrypted secret key -RWRTY0Iyz8kmPMKrqk6DCtlO9a33akKiaOQG1aLolqDxs52qvPoAAAACAAAAAAAAAEAAAAAArEiggdvyn6+WzTprirLtgiYQoU+ihz/HyGgjhuF+Pz2ddMduyCO+xjCHeq+vgVVW039fbsI8hW6LRGJZLBKV5/jdxCXAVVQE7qTQ6xpEdO0z8Z731/pV1hlspQXG2PNd16NMtwd9dWw= diff --git a/cmd/geth/testdata/vcheck/signify-sigs/data.json.sig b/cmd/geth/testdata/vcheck/signify-sigs/data.json.sig deleted file mode 100644 index d704af7709..0000000000 --- a/cmd/geth/testdata/vcheck/signify-sigs/data.json.sig +++ /dev/null @@ -1,2 +0,0 @@ -untrusted comment: verify with signifykey.pub -RWSKLNhZb0KdARbMcGN40hbHzKQYZDgDOFhEUT1YpzMnqre/mbKJ8td/HVlG03Am1YCszATiI0DbnljjTy4iNHYwqBfzrFUqUg0= diff --git a/cmd/geth/testdata/vcheck/signifykey.pub b/cmd/geth/testdata/vcheck/signifykey.pub deleted file mode 100644 index 328f973ab4..0000000000 --- a/cmd/geth/testdata/vcheck/signifykey.pub +++ /dev/null @@ -1,2 +0,0 @@ -untrusted comment: signify public key -RWSKLNhZb0KdATtRT7mZC/bybI3t3+Hv/O2i3ye04Dq9fnT9slpZ1a2/ diff --git a/cmd/geth/testdata/vcheck/signifykey.sec b/cmd/geth/testdata/vcheck/signifykey.sec deleted file mode 100644 index 3279a2e58b..0000000000 --- a/cmd/geth/testdata/vcheck/signifykey.sec +++ /dev/null @@ -1,2 +0,0 @@ -untrusted comment: signify secret key -RWRCSwAAACpLQDLawSQCtI7eAVIvaiHzjTsTyJsfV5aKLNhZb0KdAWeICXJGa93/bHAcsY6jUh9I8RdEcDWEoGxmaXZC+IdVBPxDpkix9fBRGEUdKWHi3dOfqME0YRzErWI5AVg3cRw= diff --git a/cmd/geth/testdata/vcheck/vulnerabilities.json b/cmd/geth/testdata/vcheck/vulnerabilities.json deleted file mode 100644 index e52fd84e67..0000000000 --- a/cmd/geth/testdata/vcheck/vulnerabilities.json +++ /dev/null @@ -1,202 +0,0 @@ -[ - { - "name": "CorruptedDAG", - "uid": "GETH-2020-01", - "summary": "Mining nodes will generate erroneous PoW on epochs > `385`.", - "description": "A mining flaw could cause miners to erroneously calculate PoW, due to an index overflow, if DAG size is exceeding the maximum 32 bit unsigned value.\n\nThis occurred on the ETC chain on 2020-11-06. This is likely to trigger for ETH mainnet around block `11550000`/epoch `385`, slated to occur early January 2021.\n\nThis issue is relevant only for miners, non-mining nodes are unaffected, since non-mining nodes use a smaller verification cache instead of a full DAG.", - "links": [ - "https://github.com/ethereum/go-ethereum/pull/21793", - "https://blog.ethereum.org/2020/11/12/geth-security-release", - "https://github.com/ethereum/go-ethereum/commit/567d41d9363706b4b13ce0903804e8acf214af49", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-v592-xf75-856p" - ], - "introduced": "v1.6.0", - "fixed": "v1.9.24", - "published": "2020-11-12", - "severity": "Medium", - "CVE": "CVE-2020-26240", - "check": "Geth\\/v1\\.(6|7|8)\\..*|Geth\\/v1\\.9\\.\\d-.*|Geth\\/v1\\.9\\.1.*|Geth\\/v1\\.9\\.2(0|1|2|3)-.*" - }, - { - "name": "Denial of service due to Go CVE-2020-28362", - "uid": "GETH-2020-02", - "summary": "A denial-of-service issue can be used to crash Geth nodes during block processing, due to an underlying bug in Go (CVE-2020-28362) versions < `1.15.5`, or `<1.14.12`", - "description": "The DoS issue can be used to crash all Geth nodes during block processing, the effects of which would be that a major part of the Ethereum network went offline.\n\nOutside of Go-Ethereum, the issue is most likely relevant for all forks of Geth (such as TurboGeth or ETC’s core-geth) which is built with versions of Go which contains the vulnerability.", - "links": [ - "https://blog.ethereum.org/2020/11/12/geth-security-release", - "https://groups.google.com/g/golang-announce/c/NpBGTTmKzpM", - "https://github.com/golang/go/issues/42552", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-m6gx-rhvj-fh52" - ], - "introduced": "v0.0.0", - "fixed": "v1.9.24", - "published": "2020-11-12", - "severity": "Critical", - "CVE": "CVE-2020-28362", - "check": "Geth.*\\/go1\\.(11(.*)|12(.*)|13(.*)|14|14\\.(\\d|10|11|)|15|15\\.[0-4])$" - }, - { - "name": "ShallowCopy", - "uid": "GETH-2020-03", - "summary": "A consensus flaw in Geth, related to `datacopy` precompile", - "description": "Geth erroneously performed a 'shallow' copy when the precompiled `datacopy` (at `0x00...04`) was invoked. An attacker could deploy a contract that uses the shallow copy to corrupt the contents of the `RETURNDATA`, thus causing a consensus failure.", - "links": [ - "https://blog.ethereum.org/2020/11/12/geth-security-release", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-69v6-xc2j-r2jf" - ], - "introduced": "v1.9.7", - "fixed": "v1.9.17", - "published": "2020-11-12", - "severity": "Critical", - "CVE": "CVE-2020-26241", - "check": "Geth\\/v1\\.9\\.(7|8|9|10|11|12|13|14|15|16).*$" - }, - { - "name": "Geth DoS via MULMOD", - "uid": "GETH-2020-04", - "summary": "A denial-of-service issue can be used to crash Geth nodes during block processing", - "description": "Affected versions suffer from a vulnerability which can be exploited through the `MULMOD` operation, by specifying a modulo of `0`: `mulmod(a,b,0)`, causing a `panic` in the underlying library. \nThe crash was in the `uint256` library, where a buffer [underflowed](https://github.com/holiman/uint256/blob/4ce82e695c10ddad57215bdbeafb68b8c5df2c30/uint256.go#L442).\n\n\tif `d == 0`, `dLen` remains `0`\n\nand https://github.com/holiman/uint256/blob/4ce82e695c10ddad57215bdbeafb68b8c5df2c30/uint256.go#L451 will try to access index `[-1]`.\n\nThe `uint256` library was first merged in this [commit](https://github.com/ethereum/go-ethereum/commit/cf6674539c589f80031f3371a71c6a80addbe454), on 2020-06-08. \nExploiting this vulnerabilty would cause all vulnerable nodes to drop off the network. \n\nThe issue was brought to our attention through a [bug report](https://github.com/ethereum/go-ethereum/issues/21367), showing a `panic` occurring on sync from genesis on the Ropsten network.\n \nIt was estimated that the least obvious way to fix this would be to merge the fix into `uint256`, make a new release of that library and then update the geth-dependency.\n", - "links": [ - "https://blog.ethereum.org/2020/11/12/geth-security-release", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-jm5c-rv3w-w83m", - "https://github.com/holiman/uint256/releases/tag/v1.1.1", - "https://github.com/holiman/uint256/pull/80", - "https://github.com/ethereum/go-ethereum/pull/21368" - ], - "introduced": "v1.9.16", - "fixed": "v1.9.18", - "published": "2020-11-12", - "severity": "Critical", - "CVE": "CVE-2020-26242", - "check": "Geth\\/v1\\.9.(16|17).*$" - }, - { - "name": "LES Server DoS via GetProofsV2", - "uid": "GETH-2020-05", - "summary": "A DoS vulnerability can make a LES server crash.", - "description": "A DoS vulnerability can make a LES server crash via malicious GetProofsV2 request from a connected LES client.\n\nThe vulnerability was patched in #21896.\n\nThis vulnerability only concern users explicitly running geth as a light server", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-r33q-22hv-j29q", - "https://github.com/ethereum/go-ethereum/pull/21896" - ], - "introduced": "v1.8.0", - "fixed": "v1.9.25", - "published": "2020-12-10", - "severity": "Medium", - "CVE": "CVE-2020-26264", - "check": "(Geth\\/v1\\.8\\.*)|(Geth\\/v1\\.9\\.\\d-.*)|(Geth\\/v1\\.9\\.1\\d-.*)|(Geth\\/v1\\.9\\.(20|21|22|23|24)-.*)$" - }, - { - "name": "SELFDESTRUCT-recreate consensus flaw", - "uid": "GETH-2020-06", - "introduced": "v1.9.4", - "fixed": "v1.9.20", - "summary": "A consensus-vulnerability in Geth could cause a chain split, where vulnerable versions refuse to accept the canonical chain.", - "description": "A flaw was repoted at 2020-08-11 by John Youngseok Yang (Software Platform Lab), where a particular sequence of transactions could cause a consensus failure.\n\n- Tx 1:\n - `sender` invokes `caller`.\n - `caller` invokes `0xaa`. `0xaa` has 3 wei, does a self-destruct-to-self\n - `caller` does a `1 wei` -call to `0xaa`, who thereby has 1 wei (the code in `0xaa` still executed, since the tx is still ongoing, but doesn't redo the selfdestruct, it takes a different path if callvalue is non-zero)\n\n-Tx 2:\n - `sender` does a 5-wei call to 0xaa. No exec (since no code). \n\nIn geth, the result would be that `0xaa` had `6 wei`, whereas OE reported (correctly) `5` wei. Furthermore, in geth, if the second tx was not executed, the `0xaa` would be destructed, resulting in `0 wei`. Thus obviously wrong. \n\nIt was determined that the root cause was this [commit](https://github.com/ethereum/go-ethereum/commit/223b950944f494a5b4e0957fd9f92c48b09037ad) from [this PR](https://github.com/ethereum/go-ethereum/pull/19953). The semantics of `createObject` was subtly changd, into returning a non-nil object (with `deleted=true`) where it previously did not if the account had been destructed. This return value caused the new object to inherit the old `balance`.\n", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-xw37-57qp-9mm4" - ], - "published": "2020-12-10", - "severity": "High", - "CVE": "CVE-2020-26265", - "check": "(Geth\\/v1\\.9\\.(4|5|6|7|8|9)-.*)|(Geth\\/v1\\.9\\.1\\d-.*)$" - }, - { - "name": "Not ready for London upgrade", - "uid": "GETH-2021-01", - "summary": "The client is not ready for the 'London' technical upgrade, and will deviate from the canonical chain when the London upgrade occurs (at block '12965000' around August 4, 2021.", - "description": "At (or around) August 4, Ethereum will undergo a technical upgrade called 'London'. Clients not upgraded will fail to progress on the canonical chain.", - "links": [ - "https://github.com/ethereum/eth1.0-specs/blob/master/network-upgrades/mainnet-upgrades/london.md", - "https://notes.ethereum.org/@timbeiko/ropsten-postmortem" - ], - "introduced": "v1.10.1", - "fixed": "v1.10.6", - "published": "2021-07-22", - "severity": "High", - "check": "(Geth\\/v1\\.10\\.(1|2|3|4|5)-.*)$" - }, - { - "name": "RETURNDATA corruption via datacopy", - "uid": "GETH-2021-02", - "summary": "A consensus-flaw in the Geth EVM could cause a node to deviate from the canonical chain.", - "description": "A memory-corruption bug within the EVM can cause a consensus error, where vulnerable nodes obtain a different `stateRoot` when processing a maliciously crafted transaction. This, in turn, would lead to the chain being split: mainnet splitting in two forks.\n\nAll Geth versions supporting the London hard fork are vulnerable (the bug is older than London), so all users should update.\n\nThis bug was exploited on Mainnet at block 13107518.\n\nCredits for the discovery go to @guidovranken (working for Sentnl during an audit of the Telos EVM) and reported via bounty@ethereum.org.", - "links": [ - "https://github.com/ethereum/go-ethereum/blob/master/docs/postmortems/2021-08-22-split-postmortem.md", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-9856-9gg9-qcmq", - "https://github.com/ethereum/go-ethereum/releases/tag/v1.10.8" - ], - "introduced": "v1.10.0", - "fixed": "v1.10.8", - "published": "2021-08-24", - "severity": "High", - "CVE": "CVE-2021-39137", - "check": "(Geth\\/v1\\.10\\.(0|1|2|3|4|5|6|7)-.*)$" - }, - { - "name": "DoS via malicious `snap/1` request", - "uid": "GETH-2021-03", - "summary": "A vulnerable node is susceptible to crash when processing a maliciously crafted message from a peer, via the snap/1 protocol. The crash can be triggered by sending a malicious snap/1 GetTrieNodes package.", - "description": "The `snap/1` protocol handler contains two vulnerabilities related to the `GetTrieNodes` packet, which can be exploited to crash the node. Full details are available at the Github security [advisory](https://github.com/ethereum/go-ethereum/security/advisories/GHSA-59hh-656j-3p7v)", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-59hh-656j-3p7v", - "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities", - "https://github.com/ethereum/go-ethereum/pull/23657" - ], - "introduced": "v1.10.0", - "fixed": "v1.10.9", - "published": "2021-10-24", - "severity": "Medium", - "CVE": "CVE-2021-41173", - "check": "(Geth\\/v1\\.10\\.(0|1|2|3|4|5|6|7|8)-.*)$" - }, - { - "name": "DoS via malicious p2p message", - "uid": "GETH-2022-01", - "summary": "A vulnerable node can crash via p2p messages sent from an attacker node, if running with non-default log options.", - "description": "A vulnerable node, if configured to use high verbosity logging, can be made to crash when handling specially crafted p2p messages sent from an attacker node. Full details are available at the Github security [advisory](https://github.com/ethereum/go-ethereum/security/advisories/GHSA-wjxw-gh3m-7pm5)", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-wjxw-gh3m-7pm5", - "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities", - "https://github.com/ethereum/go-ethereum/pull/24507" - ], - "introduced": "v1.10.0", - "fixed": "v1.10.17", - "published": "2022-05-11", - "severity": "Low", - "CVE": "CVE-2022-29177", - "check": "(Geth\\/v1\\.10\\.(0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16)-.*)$" - }, - { - "name": "DoS via malicious p2p message", - "uid": "GETH-2023-01", - "summary": "A vulnerable node can be made to consume unbounded amounts of memory when handling specially crafted p2p messages sent from an attacker node.", - "description": "The p2p handler spawned a new goroutine to respond to ping requests. By flooding a node with ping requests, an unbounded number of goroutines can be created, leading to resource exhaustion and potentially crash due to OOM.", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-ppjg-v974-84cm", - "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities" - ], - "introduced": "v1.10.0", - "fixed": "v1.12.1", - "published": "2023-09-06", - "severity": "High", - "CVE": "CVE-2023-40591", - "check": "(Geth\\/v1\\.(10|11)\\..*)|(Geth\\/v1\\.12\\.0-.*)$" - }, - { - "name": "DoS via malicious p2p message", - "uid": "GETH-2024-01", - "summary": "A vulnerable node can be made to consume very large amounts of memory when handling specially crafted p2p messages sent from an attacker node.", - "description": "A vulnerable node can be made to consume very large amounts of memory when handling specially crafted p2p messages sent from an attacker node. Full details will be available at the Github security [advisory](https://github.com/ethereum/go-ethereum/security/advisories/GHSA-4xc9-8hmq-j652)", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-4xc9-8hmq-j652", - "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities" - ], - "introduced": "v1.10.0", - "fixed": "v1.13.15", - "published": "2024-05-06", - "severity": "High", - "CVE": "CVE-2024-32972", - "check": "(Geth\\/v1\\.(10|11|12)\\..*)|(Geth\\/v1\\.13\\.\\d-.*)|(Geth\\/v1\\.13\\.1(0|1|2|3|4)-.*)$" - } -] diff --git a/cmd/geth/verkle.go b/cmd/geth/verkle.go deleted file mode 100644 index 67dc7257c0..0000000000 --- a/cmd/geth/verkle.go +++ /dev/null @@ -1,214 +0,0 @@ -// Copyright 2022 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package main - -import ( - "bytes" - "encoding/hex" - "errors" - "fmt" - "os" - "slices" - - "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-verkle" - "github.com/urfave/cli/v2" -) - -var ( - zero [32]byte - - verkleCommand = &cli.Command{ - Name: "verkle", - Usage: "A set of experimental verkle tree management commands", - Description: "", - Subcommands: []*cli.Command{ - { - Name: "verify", - Usage: "verify the conversion of a MPT into a verkle tree", - ArgsUsage: "", - Action: verifyVerkle, - Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), - Description: ` -geth verkle verify -This command takes a root commitment and attempts to rebuild the tree. - `, - }, - { - Name: "dump", - Usage: "Dump a verkle tree to a DOT file", - ArgsUsage: " [ ...]", - Action: expandVerkle, - Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), - Description: ` -geth verkle dump [ ...] -This command will produce a dot file representing the tree, rooted at . -in which key1, key2, ... are expanded. - `, - }, - }, - } -) - -// recurse into each child to ensure they can be loaded from the db. The tree isn't rebuilt -// (only its nodes are loaded) so there is no need to flush them, the garbage collector should -// take care of that for us. -func checkChildren(root verkle.VerkleNode, resolver verkle.NodeResolverFn) error { - switch node := root.(type) { - case *verkle.InternalNode: - for i, child := range node.Children() { - childC := child.Commit().Bytes() - - childS, err := resolver(childC[:]) - if bytes.Equal(childC[:], zero[:]) { - continue - } - if err != nil { - return fmt.Errorf("could not find child %x in db: %w", childC, err) - } - // depth is set to 0, the tree isn't rebuilt so it's not a problem - childN, err := verkle.ParseNode(childS, 0) - if err != nil { - return fmt.Errorf("decode error child %x in db: %w", child.Commitment().Bytes(), err) - } - if err := checkChildren(childN, resolver); err != nil { - return fmt.Errorf("%x%w", i, err) // write the path to the erroring node - } - } - case *verkle.LeafNode: - // sanity check: ensure at least one value is non-zero - - for i := 0; i < verkle.NodeWidth; i++ { - if len(node.Value(i)) != 0 { - return nil - } - } - return errors.New("both balance and nonce are 0") - case verkle.Empty: - // nothing to do - default: - return fmt.Errorf("unsupported type encountered %v", root) - } - - return nil -} - -func verifyVerkle(ctx *cli.Context) error { - stack, _ := makeConfigNode(ctx) - defer stack.Close() - - chaindb := utils.MakeChainDatabase(ctx, stack, true) - defer chaindb.Close() - headBlock := rawdb.ReadHeadBlock(chaindb) - if headBlock == nil { - log.Error("Failed to load head block") - return errors.New("no head block") - } - if ctx.NArg() > 1 { - log.Error("Too many arguments given") - return errors.New("too many arguments") - } - var ( - rootC common.Hash - err error - ) - if ctx.NArg() == 1 { - rootC, err = parseRoot(ctx.Args().First()) - if err != nil { - log.Error("Failed to resolve state root", "error", err) - return err - } - log.Info("Rebuilding the tree", "root", rootC) - } else { - rootC = headBlock.Root() - log.Info("Rebuilding the tree", "root", rootC, "number", headBlock.NumberU64()) - } - - serializedRoot, err := chaindb.Get(rootC[:]) - if err != nil { - return err - } - root, err := verkle.ParseNode(serializedRoot, 0) - if err != nil { - return err - } - - if err := checkChildren(root, chaindb.Get); err != nil { - log.Error("Could not rebuild the tree from the database", "err", err) - return err - } - - log.Info("Tree was rebuilt from the database") - return nil -} - -func expandVerkle(ctx *cli.Context) error { - stack, _ := makeConfigNode(ctx) - defer stack.Close() - - chaindb := utils.MakeChainDatabase(ctx, stack, true) - defer chaindb.Close() - var ( - rootC common.Hash - keylist [][]byte - err error - ) - if ctx.NArg() >= 2 { - rootC, err = parseRoot(ctx.Args().First()) - if err != nil { - log.Error("Failed to resolve state root", "error", err) - return err - } - keylist = make([][]byte, 0, ctx.Args().Len()-1) - args := ctx.Args().Slice() - for i := range args[1:] { - key, err := hex.DecodeString(args[i+1]) - log.Info("decoded key", "arg", args[i+1], "key", key) - if err != nil { - return fmt.Errorf("error decoding key #%d: %w", i+1, err) - } - keylist = append(keylist, key) - } - log.Info("Rebuilding the tree", "root", rootC) - } else { - return fmt.Errorf("usage: %s root key1 [key 2...]", ctx.App.Name) - } - - serializedRoot, err := chaindb.Get(rootC[:]) - if err != nil { - return err - } - root, err := verkle.ParseNode(serializedRoot, 0) - if err != nil { - return err - } - - for i, key := range keylist { - log.Info("Reading key", "index", i, "key", key) - root.Get(key, chaindb.Get) - } - - if err := os.WriteFile("dump.dot", []byte(verkle.ToDot(root)), 0600); err != nil { - log.Error("Failed to dump file", "err", err) - } else { - log.Info("Tree was dumped to file", "file", "dump.dot") - } - return nil -} diff --git a/cmd/geth/version_check.go b/cmd/geth/version_check.go deleted file mode 100644 index 237556788e..0000000000 --- a/cmd/geth/version_check.go +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package main - -import ( - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "os" - "regexp" - "strings" - - "github.com/ethereum/go-ethereum/log" - "github.com/jedisct1/go-minisign" - "github.com/urfave/cli/v2" -) - -var gethPubKeys []string = []string{ - //@holiman, minisign public key FB1D084D39BAEC24 - "RWQk7Lo5TQgd+wxBNZM+Zoy+7UhhMHaWKzqoes9tvSbFLJYZhNTbrIjx", - //minisign public key 138B1CA303E51687 - "RWSHFuUDoxyLEzjszuWZI1xStS66QTyXFFZG18uDfO26CuCsbckX1e9J", - //minisign public key FD9813B2D2098484 - "RWSEhAnSshOY/b+GmaiDkObbCWefsAoavjoLcPjBo1xn71yuOH5I+Lts", -} - -type vulnJson struct { - Name string - Uid string - Summary string - Description string - Links []string - Introduced string - Fixed string - Published string - Severity string - Check string - CVE string -} - -func versionCheck(ctx *cli.Context) error { - url := ctx.String(VersionCheckUrlFlag.Name) - version := ctx.String(VersionCheckVersionFlag.Name) - log.Info("Checking vulnerabilities", "version", version, "url", url) - return checkCurrent(url, version) -} - -func checkCurrent(url, current string) error { - var ( - data []byte - sig []byte - err error - ) - if data, err = fetch(url); err != nil { - return fmt.Errorf("could not retrieve data: %w", err) - } - if sig, err = fetch(fmt.Sprintf("%v.minisig", url)); err != nil { - return fmt.Errorf("could not retrieve signature: %w", err) - } - if err = verifySignature(gethPubKeys, data, sig); err != nil { - return err - } - var vulns []vulnJson - if err = json.Unmarshal(data, &vulns); err != nil { - return err - } - allOk := true - for _, vuln := range vulns { - r, err := regexp.Compile(vuln.Check) - if err != nil { - return err - } - if r.MatchString(current) { - allOk = false - fmt.Printf("## Vulnerable to %v (%v)\n\n", vuln.Uid, vuln.Name) - fmt.Printf("Severity: %v\n", vuln.Severity) - fmt.Printf("Summary : %v\n", vuln.Summary) - fmt.Printf("Fixed in: %v\n", vuln.Fixed) - if len(vuln.CVE) > 0 { - fmt.Printf("CVE: %v\n", vuln.CVE) - } - if len(vuln.Links) > 0 { - fmt.Printf("References:\n") - for _, ref := range vuln.Links { - fmt.Printf("\t- %v\n", ref) - } - } - fmt.Println() - } - } - if allOk { - fmt.Println("No vulnerabilities found") - } - return nil -} - -// fetch makes an HTTP request to the given url and returns the response body -func fetch(url string) ([]byte, error) { - if filep := strings.TrimPrefix(url, "file://"); filep != url { - return os.ReadFile(filep) - } - res, err := http.Get(url) - if err != nil { - return nil, err - } - defer res.Body.Close() - body, err := io.ReadAll(res.Body) - if err != nil { - return nil, err - } - return body, nil -} - -// verifySignature checks that the sigData is a valid signature of the given -// data, for pubkey GethPubkey -func verifySignature(pubkeys []string, data, sigdata []byte) error { - sig, err := minisign.DecodeSignature(string(sigdata)) - if err != nil { - return err - } - // find the used key - var key *minisign.PublicKey - for _, pubkey := range pubkeys { - pub, err := minisign.NewPublicKey(pubkey) - if err != nil { - // our pubkeys should be parseable - return err - } - if pub.KeyId != sig.KeyId { - continue - } - key = &pub - break - } - if key == nil { - log.Info("Signing key not trusted", "keyid", keyID(sig.KeyId), "error", err) - return errors.New("signature could not be verified") - } - if ok, err := key.Verify(data, sig); !ok || err != nil { - log.Info("Verification failed error", "keyid", keyID(key.KeyId), "error", err) - return errors.New("signature could not be verified") - } - return nil -} - -// keyID turns a binary minisign key ID into a hex string. -// Note: key IDs are printed in reverse byte order. -func keyID(id [8]byte) string { - var rev [8]byte - for i := range id { - rev[len(rev)-1-i] = id[i] - } - return fmt.Sprintf("%X", rev) -} diff --git a/cmd/geth/version_check_test.go b/cmd/geth/version_check_test.go deleted file mode 100644 index fb5d1b2d69..0000000000 --- a/cmd/geth/version_check_test.go +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package main - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "regexp" - "strconv" - "strings" - "testing" - - "github.com/jedisct1/go-minisign" -) - -func TestVerification(t *testing.T) { - t.Parallel() - // Signatures generated with `minisign`. Legacy format, not pre-hashed file. - t.Run("minisig-legacy", func(t *testing.T) { - t.Parallel() - // For this test, the pubkey is in testdata/vcheck/minisign.pub - // (the privkey is `minisign.sec`, if we want to expand this test. Password 'test' ) - // 1. `minisign -S -l -s ./minisign.sec -m data.json -x ./minisig-sigs/vulnerabilities.json.minisig.1 -c "signature from minisign secret key"` - // 2. `minisign -S -l -s ./minisign.sec -m vulnerabilities.json -x ./minisig-sigs/vulnerabilities.json.minisig.2 -c "Here's a comment" -t "Here's a trusted comment"` - // 3. minisign -S -l -s ./minisign.sec -m vulnerabilities.json -x ./minisig-sigs/vulnerabilities.json.minisig.3 -c "One more (untrusted™) comment" -t "Here's a trusted comment" - pub := "RWQkliYstQBOKOdtClfgC3IypIPX6TAmoEi7beZ4gyR3wsaezvqOMWsp" - testVerification(t, pub, "./testdata/vcheck/minisig-sigs/") - }) - t.Run("minisig-new", func(t *testing.T) { - t.Parallel() - // For this test, the pubkey is in testdata/vcheck/minisign.pub - // (the privkey is `minisign.sec`, if we want to expand this test. Password 'test' ) - // `minisign -S -s ./minisign.sec -m data.json -x ./minisig-sigs-new/data.json.minisig` - pub := "RWQkliYstQBOKOdtClfgC3IypIPX6TAmoEi7beZ4gyR3wsaezvqOMWsp" - testVerification(t, pub, "./testdata/vcheck/minisig-sigs-new/") - }) - // Signatures generated with `signify-openbsd` - t.Run("signify-openbsd", func(t *testing.T) { - t.Parallel() - t.Skip("This currently fails, minisign expects 4 lines of data, signify provides only 2") - // For this test, the pubkey is in testdata/vcheck/signifykey.pub - // (the privkey is `signifykey.sec`, if we want to expand this test. Password 'test' ) - // `signify -S -s signifykey.sec -m data.json -x ./signify-sigs/data.json.sig` - pub := "RWSKLNhZb0KdATtRT7mZC/bybI3t3+Hv/O2i3ye04Dq9fnT9slpZ1a2/" - testVerification(t, pub, "./testdata/vcheck/signify-sigs/") - }) -} - -func testVerification(t *testing.T, pubkey, sigdir string) { - // Data to verify - data, err := os.ReadFile("./testdata/vcheck/data.json") - if err != nil { - t.Fatal(err) - } - // Signatures, with and without comments, both trusted and untrusted - files, err := os.ReadDir(sigdir) - if err != nil { - t.Fatal(err) - } - if len(files) == 0 { - t.Fatal("Missing tests") - } - for _, f := range files { - sig, err := os.ReadFile(filepath.Join(sigdir, f.Name())) - if err != nil { - t.Fatal(err) - } - err = verifySignature([]string{pubkey}, data, sig) - if err != nil { - t.Fatal(err) - } - } -} - -func versionUint(v string) int { - mustInt := func(s string) int { - a, err := strconv.Atoi(s) - if err != nil { - panic(v) - } - return a - } - components := strings.Split(strings.TrimPrefix(v, "v"), ".") - a := mustInt(components[0]) - b := mustInt(components[1]) - c := mustInt(components[2]) - return a*100*100 + b*100 + c -} - -// TestMatching can be used to check that the regexps are correct -func TestMatching(t *testing.T) { - t.Parallel() - data, _ := os.ReadFile("./testdata/vcheck/vulnerabilities.json") - var vulns []vulnJson - if err := json.Unmarshal(data, &vulns); err != nil { - t.Fatal(err) - } - check := func(version string) { - vFull := fmt.Sprintf("Geth/%v-unstable-15339cf1-20201204/linux-amd64/go1.15.4", version) - for _, vuln := range vulns { - r, err := regexp.Compile(vuln.Check) - vulnIntro := versionUint(vuln.Introduced) - vulnFixed := versionUint(vuln.Fixed) - current := versionUint(version) - if err != nil { - t.Fatal(err) - } - if vuln.Name == "Denial of service due to Go CVE-2020-28362" { - // this one is not tied to geth-versions - continue - } - if vulnIntro <= current && vulnFixed > current { - // Should be vulnerable - if !r.MatchString(vFull) { - t.Errorf("Should be vulnerable, version %v, intro: %v, fixed: %v %v %v", - version, vuln.Introduced, vuln.Fixed, vuln.Name, vuln.Check) - } - } else { - if r.MatchString(vFull) { - t.Errorf("Should not be flagged vulnerable, version %v, intro: %v, fixed: %v %v %d %d %d", - version, vuln.Introduced, vuln.Fixed, vuln.Name, vulnIntro, current, vulnFixed) - } - } - } - } - for major := 1; major < 2; major++ { - for minor := 0; minor < 30; minor++ { - for patch := 0; patch < 30; patch++ { - vShort := fmt.Sprintf("v%d.%d.%d", major, minor, patch) - check(vShort) - } - } - } -} - -func TestGethPubKeysParseable(t *testing.T) { - t.Parallel() - for _, pubkey := range gethPubKeys { - _, err := minisign.NewPublicKey(pubkey) - if err != nil { - t.Errorf("Should be parseable") - } - } -} - -func TestKeyID(t *testing.T) { - t.Parallel() - type args struct { - id [8]byte - } - tests := []struct { - name string - args args - want string - }{ - {"@holiman key", args{id: extractKeyId(gethPubKeys[0])}, "FB1D084D39BAEC24"}, - {"second key", args{id: extractKeyId(gethPubKeys[1])}, "138B1CA303E51687"}, - {"third key", args{id: extractKeyId(gethPubKeys[2])}, "FD9813B2D2098484"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - if got := keyID(tt.args.id); got != tt.want { - t.Errorf("keyID() = %v, want %v", got, tt.want) - } - }) - } -} - -func extractKeyId(pubkey string) [8]byte { - p, _ := minisign.NewPublicKey(pubkey) - return p.KeyId -} diff --git a/cmd/keeper/getpayload_example.go b/cmd/keeper/getpayload_example.go index 683cc79248..8f40a7bd11 100644 --- a/cmd/keeper/getpayload_example.go +++ b/cmd/keeper/getpayload_example.go @@ -15,6 +15,7 @@ // along with the go-ethereum library. If not, see . //go:build example +// +build example package main diff --git a/cmd/keeper/getpayload_wasm.go b/cmd/keeper/getpayload_wasm.go new file mode 100644 index 0000000000..5024ac7d49 --- /dev/null +++ b/cmd/keeper/getpayload_wasm.go @@ -0,0 +1,36 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +//go:build wasm && !womir +// +build wasm,!womir + +package main + +import ( + "unsafe" +) + +//go:wasmimport geth_io len +func hintLen() uint32 + +//go:wasmimport geth_io read +func hintRead(data unsafe.Pointer) + +func getInput() []byte { + data := make([]byte, hintLen()) + hintRead(unsafe.Pointer(&data[0])) + return data +} diff --git a/cmd/keeper/getpayload_womir.go b/cmd/keeper/getpayload_womir.go new file mode 100644 index 0000000000..8645dc7c26 --- /dev/null +++ b/cmd/keeper/getpayload_womir.go @@ -0,0 +1,49 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +//go:build womir + +package main + +import "unsafe" + +// These match the WOMIR guest-io imports (env module). +// Protocol: __hint_input prepares next item, __hint_buffer reads words. +// Each item has format: [byte_len_u32_le, ...data_words_padded_to_4bytes] +// +//go:wasmimport env __hint_input +func hintInput() + +//go:wasmimport env __hint_buffer +func hintBuffer(ptr unsafe.Pointer, numWords uint32) +func readWord() uint32 { + var buf [4]byte + hintBuffer(unsafe.Pointer(&buf[0]), 1) + return uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 +} +func readBytes() []byte { + hintInput() + byteLen := readWord() + numWords := (byteLen + 3) / 4 + data := make([]byte, numWords*4) + hintBuffer(unsafe.Pointer(&data[0]), numWords) + return data[:byteLen] +} + +// getInput reads the RLP-encoded Payload from the WOMIR hint stream. +func getInput() []byte { + return readBytes() +} diff --git a/cmd/keeper/go.mod b/cmd/keeper/go.mod index ceae65d491..24a2823b91 100644 --- a/cmd/keeper/go.mod +++ b/cmd/keeper/go.mod @@ -21,18 +21,18 @@ require ( github.com/cockroachdb/pebble v1.1.5 // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect - github.com/consensys/gnark-crypto v0.18.0 // indirect - github.com/crate-crypto/go-eth-kzg v1.4.0 // indirect - github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a // indirect + github.com/consensys/gnark-crypto v0.18.1 // indirect + github.com/crate-crypto/go-eth-kzg v1.5.0 // indirect github.com/deckarep/golang-set/v2 v2.6.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/emicklei/dot v1.6.2 // indirect - github.com/ethereum/c-kzg-4844/v2 v2.1.5 // indirect + github.com/ethereum/c-kzg-4844/v2 v2.1.6 // indirect github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab // indirect - github.com/ethereum/go-verkle v0.2.2 // indirect github.com/ferranbt/fastssz v0.1.4 // indirect github.com/gammazero/deque v1.1.0 // indirect github.com/getsentry/sentry-go v0.27.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/gofrs/flock v0.12.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -41,33 +41,34 @@ require ( github.com/gorilla/websocket v1.4.2 // indirect github.com/holiman/bloomfilter/v2 v2.0.3 // indirect github.com/holiman/uint256 v1.3.2 // indirect - github.com/klauspost/compress v1.16.0 // indirect + github.com/klauspost/compress v1.17.8 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect - github.com/mattn/go-runewidth v0.0.13 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect - github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.15.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect - github.com/rivo/uniseg v0.2.0 // indirect - github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect - github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe // indirect + github.com/supranational/blst v0.3.16 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect - golang.org/x/crypto v0.36.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel v1.40.0 // indirect + go.opentelemetry.io/otel/metric v1.40.0 // indirect + go.opentelemetry.io/otel/trace v1.40.0 // indirect + golang.org/x/crypto v0.44.0 // indirect golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df // indirect - golang.org/x/sync v0.12.0 // indirect - golang.org/x/sys v0.36.0 // indirect - golang.org/x/text v0.23.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/sys v0.40.0 // indirect + golang.org/x/text v0.31.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/cmd/keeper/go.sum b/cmd/keeper/go.sum index 846ad62362..e83d3e15c4 100644 --- a/cmd/keeper/go.sum +++ b/cmd/keeper/go.sum @@ -30,12 +30,10 @@ github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwP github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= -github.com/consensys/gnark-crypto v0.18.0 h1:vIye/FqI50VeAr0B3dx+YjeIvmc3LWz4yEfbWBpTUf0= -github.com/consensys/gnark-crypto v0.18.0/go.mod h1:L3mXGFTe1ZN+RSJ+CLjUt9x7PNdx8ubaYfDROyp2Z8c= -github.com/crate-crypto/go-eth-kzg v1.4.0 h1:WzDGjHk4gFg6YzV0rJOAsTK4z3Qkz5jd4RE3DAvPFkg= -github.com/crate-crypto/go-eth-kzg v1.4.0/go.mod h1:J9/u5sWfznSObptgfa92Jq8rTswn6ahQWEuiLHOjCUI= -github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a h1:W8mUrRp6NOVl3J+MYp5kPMoUZPp7aOYHtaua31lwRHg= -github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a/go.mod h1:sTwzHBvIzm2RfVCGNEBZgRyjwK40bVoun3ZnGOCafNM= +github.com/consensys/gnark-crypto v0.18.1 h1:RyLV6UhPRoYYzaFnPQA4qK3DyuDgkTgskDdoGqFt3fI= +github.com/consensys/gnark-crypto v0.18.1/go.mod h1:L3mXGFTe1ZN+RSJ+CLjUt9x7PNdx8ubaYfDROyp2Z8c= +github.com/crate-crypto/go-eth-kzg v1.5.0 h1:FYRiJMJG2iv+2Dy3fi14SVGjcPteZ5HAAUe4YWlJygc= +github.com/crate-crypto/go-eth-kzg v1.5.0/go.mod h1:J9/u5sWfznSObptgfa92Jq8rTswn6ahQWEuiLHOjCUI= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -47,12 +45,10 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/emicklei/dot v1.6.2 h1:08GN+DD79cy/tzN6uLCT84+2Wk9u+wvqP+Hkx/dIR8A= github.com/emicklei/dot v1.6.2/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= -github.com/ethereum/c-kzg-4844/v2 v2.1.5 h1:aVtoLK5xwJ6c5RiqO8g8ptJ5KU+2Hdquf6G3aXiHh5s= -github.com/ethereum/c-kzg-4844/v2 v2.1.5/go.mod h1:u59hRTTah4Co6i9fDWtiCjTrblJv0UwsqZKCc0GfgUs= +github.com/ethereum/c-kzg-4844/v2 v2.1.6 h1:xQymkKCT5E2Jiaoqf3v4wsNgjZLY0lRSkZn27fRjSls= +github.com/ethereum/c-kzg-4844/v2 v2.1.6/go.mod h1:8HMkUZ5JRv4hpw/XUrYWSQNAUzhHMg2UDb/U+5m+XNw= github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab h1:rvv6MJhy07IMfEKuARQ9TKojGqLVNxQajaXEp/BoqSk= github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab/go.mod h1:IuLm4IsPipXKF7CW5Lzf68PIbZ5yl7FFd74l/E0o9A8= -github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= -github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= github.com/ferranbt/fastssz v0.1.4 h1:OCDB+dYDEQDvAgtAGnTSidK1Pe2tW3nFV40XyMkTeDY= github.com/ferranbt/fastssz v0.1.4/go.mod h1:Ea3+oeoRGGLGm5shYAeDgu6PGUlcvQhE2fILyD9+tGg= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -65,6 +61,11 @@ github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= @@ -88,8 +89,10 @@ github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= @@ -99,8 +102,8 @@ github.com/holiman/uint256 v1.3.2/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXei github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= -github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -112,9 +115,6 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leanovate/gopter v0.2.11 h1:vRjThO1EKPb/1NsDXuDrzldR28RLkBflWYcU9CvzWu4= github.com/leanovate/gopter v0.2.11/go.mod h1:aK3tzZP/C+p1m3SPRE4SYZFGP7jjkuSI4f7Xvpt0S9c= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= -github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= @@ -123,8 +123,6 @@ github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxd github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= -github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= @@ -149,17 +147,15 @@ github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJf github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/prysmaticlabs/gohashtree v0.0.4-beta h1:H/EbCuXPeTV3lpKeXGPpEV9gsUpkqOOVnWapUyeWro4= github.com/prysmaticlabs/gohashtree v0.0.4-beta/go.mod h1:BFdtALS+Ffhg3lGQIHv9HDWuHS8cTvHZzrHWxwOtGOs= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= -github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe h1:nbdqkIGOGfUAD54q1s2YBcBz/WcsxCO9HUQ4aGV5hUw= -github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/supranational/blst v0.3.16 h1:bTDadT+3fK497EvLdWRQEjiGnUtzJ7jjIUMF0jqwYhE= +github.com/supranational/blst v0.3.16/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= @@ -168,11 +164,21 @@ github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+F github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= +go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= +go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= +go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= +go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= +go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= +go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= +go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU= +golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc= golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -184,15 +190,15 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -207,13 +213,13 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= -golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -229,8 +235,8 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/cmd/keeper/main.go b/cmd/keeper/main.go index 9b459f6f36..df6881acbf 100644 --- a/cmd/keeper/main.go +++ b/cmd/keeper/main.go @@ -17,6 +17,7 @@ package main import ( + "context" "fmt" "os" "runtime/debug" @@ -52,7 +53,7 @@ func main() { } vmConfig := vm.Config{} - crossStateRoot, crossReceiptRoot, err := core.ExecuteStateless(chainConfig, vmConfig, payload.Block, payload.Witness) + crossStateRoot, crossReceiptRoot, err := core.ExecuteStateless(context.Background(), chainConfig, vmConfig, payload.Block, payload.Witness) if err != nil { fmt.Fprintf(os.Stderr, "stateless self-validation failed: %v\n", err) os.Exit(10) diff --git a/cmd/keeper/stubs.go b/cmd/keeper/stubs.go index 04a3bc735b..de7ee64353 100644 --- a/cmd/keeper/stubs.go +++ b/cmd/keeper/stubs.go @@ -14,7 +14,8 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//go:build !example && !ziren +//go:build !example && !ziren && !wasm && !womir +// +build !example,!ziren,!wasm,!womir package main diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 3e337a3d00..e490f613b3 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -57,6 +57,8 @@ const ( importBatchSize = 2500 ) +type EraFileFormat int + // ErrImportInterrupted is returned when the user interrupts the import process. var ErrImportInterrupted = errors.New("interrupted") @@ -250,7 +252,7 @@ func readList(filename string) ([]string, error) { // ImportHistory imports Era1 files containing historical block information, // starting from genesis. The assumption is held that the provided chain // segment in Era1 file should all be canonical and verified. -func ImportHistory(chain *core.BlockChain, dir string, network string) error { +func ImportHistory(chain *core.BlockChain, dir string, network string, from func(f era.ReadAtSeekCloser) (era.Era, error)) error { if chain.CurrentSnapBlock().Number.BitLen() != 0 { return errors.New("history import only supported when starting from genesis") } @@ -263,8 +265,10 @@ func ImportHistory(chain *core.BlockChain, dir string, network string) error { return fmt.Errorf("unable to read checksums.txt: %w", err) } if len(checksums) != len(entries) { - return fmt.Errorf("expected equal number of checksums and entries, have: %d checksums, %d entries", len(checksums), len(entries)) + return fmt.Errorf("expected equal number of checksums and entries, have: %d checksums, %d entries", + len(checksums), len(entries)) } + var ( start = time.Now() reported = time.Now() @@ -272,33 +276,64 @@ func ImportHistory(chain *core.BlockChain, dir string, network string) error { h = sha256.New() buf = bytes.NewBuffer(nil) ) - for i, filename := range entries { + + for i, file := range entries { err := func() error { - f, err := os.Open(filepath.Join(dir, filename)) + path := filepath.Join(dir, file) + + // Validate against checksum file in directory. + f, err := os.Open(path) if err != nil { - return fmt.Errorf("unable to open era: %w", err) + return fmt.Errorf("open %s: %w", path, err) } defer f.Close() - // Validate checksum. if _, err := io.Copy(h, f); err != nil { - return fmt.Errorf("unable to recalculate checksum: %w", err) - } - if have, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; have != want { - return fmt.Errorf("checksum mismatch: have %s, want %s", have, want) + return fmt.Errorf("checksum %s: %w", path, err) } + got := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex() h.Reset() buf.Reset() - + if got != checksums[i] { + return fmt.Errorf("%s checksum mismatch: have %s want %s", file, got, checksums[i]) + } // Import all block data from Era1. - e, err := era.From(f) + e, err := from(f) if err != nil { return fmt.Errorf("error opening era: %w", err) } - it, err := era.NewIterator(e) + defer e.Close() + + it, err := e.Iterator() if err != nil { - return fmt.Errorf("error making era reader: %w", err) + return fmt.Errorf("error creating iterator: %w", err) } + + var ( + blocks = make([]*types.Block, 0, importBatchSize) + receiptsList = make([]types.Receipts, 0, importBatchSize) + flush = func() error { + if len(blocks) == 0 { + return nil + } + enc := types.EncodeBlockReceiptLists(receiptsList) + if _, err := chain.InsertReceiptChain(blocks, enc, math.MaxUint64); err != nil { + return fmt.Errorf("error inserting blocks %d-%d: %w", + blocks[0].NumberU64(), blocks[len(blocks)-1].NumberU64(), err) + } + imported += len(blocks) + if time.Since(reported) >= 8*time.Second { + head := blocks[len(blocks)-1].NumberU64() + log.Info("Importing Era files", "head", head, "imported", imported, + "elapsed", common.PrettyDuration(time.Since(start))) + imported = 0 + reported = time.Now() + } + blocks = blocks[:0] + receiptsList = receiptsList[:0] + return nil + } + ) for it.Next() { block, err := it.Block() if err != nil { @@ -311,26 +346,23 @@ func ImportHistory(chain *core.BlockChain, dir string, network string) error { if err != nil { return fmt.Errorf("error reading receipts %d: %w", it.Number(), err) } - encReceipts := types.EncodeBlockReceiptLists([]types.Receipts{receipts}) - if _, err := chain.InsertReceiptChain([]*types.Block{block}, encReceipts, math.MaxUint64); err != nil { - return fmt.Errorf("error inserting body %d: %w", it.Number(), err) - } - imported += 1 - - // Give the user some feedback that something is happening. - if time.Since(reported) >= 8*time.Second { - log.Info("Importing Era files", "head", it.Number(), "imported", imported, "elapsed", common.PrettyDuration(time.Since(start))) - imported = 0 - reported = time.Now() + blocks = append(blocks, block) + receiptsList = append(receiptsList, receipts) + if len(blocks) == importBatchSize { + if err := flush(); err != nil { + return err + } } } - return nil + if err := it.Error(); err != nil { + return err + } + return flush() }() if err != nil { return err } } - return nil } @@ -389,7 +421,6 @@ func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, las return err } defer fh.Close() - var writer io.Writer = fh if strings.HasSuffix(fn, ".gz") { writer = gzip.NewWriter(writer) @@ -405,7 +436,7 @@ func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, las // ExportHistory exports blockchain history into the specified directory, // following the Era format. -func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64) error { +func ExportHistory(bc *core.BlockChain, dir string, first, last uint64, newBuilder func(io.Writer) era.Builder, filename func(network string, epoch int, lastBlockHash common.Hash) string) error { log.Info("Exporting blockchain history", "dir", dir) if head := bc.CurrentBlock().Number.Uint64(); head < last { log.Warn("Last block beyond head, setting last = head", "head", head, "last", last) @@ -418,76 +449,100 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64) er if err := os.MkdirAll(dir, os.ModePerm); err != nil { return fmt.Errorf("error creating output directory: %w", err) } + var ( start = time.Now() reported = time.Now() h = sha256.New() buf = bytes.NewBuffer(nil) + td = new(big.Int) checksums []string ) - td := new(big.Int) - for i := uint64(0); i < first; i++ { - td.Add(td, bc.GetHeaderByNumber(i).Difficulty) + + // Compute initial TD by accumulating difficulty from genesis to first-1. + // This is necessary because TD is no longer stored in the database. Only + // compute if a segment of the export is pre-merge. + b := bc.GetBlockByNumber(first) + if b == nil { + return fmt.Errorf("block #%d not found", first) + } + if first > 0 && b.Difficulty().Sign() != 0 { + log.Info("Computing initial total difficulty", "from", 0, "to", first-1) + for i := uint64(0); i < first; i++ { + b := bc.GetBlockByNumber(i) + if b == nil { + return fmt.Errorf("block #%d not found while computing initial TD", i) + } + td.Add(td, b.Difficulty()) + } + log.Info("Initial total difficulty computed", "td", td) } - for i := first; i <= last; i += step { - err := func() error { - filename := filepath.Join(dir, era.Filename(network, int(i/step), common.Hash{})) - f, err := os.Create(filename) + + for batch := first; batch <= last; batch += uint64(era.MaxSize) { + idx := int(batch / uint64(era.MaxSize)) + tmpPath := filepath.Join(dir, filename(network, idx, common.Hash{})) + + if err := func() error { + f, err := os.Create(tmpPath) if err != nil { - return fmt.Errorf("could not create era file: %w", err) + return err } defer f.Close() - w := era.NewBuilder(f) - for j := uint64(0); j < step && j <= last-i; j++ { - var ( - n = i + j - block = bc.GetBlockByNumber(n) - ) + builder := newBuilder(f) + + for j := uint64(0); j < uint64(era.MaxSize) && batch+j <= last; j++ { + n := batch + j + block := bc.GetBlockByNumber(n) if block == nil { - return fmt.Errorf("export failed on #%d: not found", n) + return fmt.Errorf("block #%d not found", n) } - receipts := bc.GetReceiptsByHash(block.Hash()) - if receipts == nil { - return fmt.Errorf("export failed on #%d: receipts not found", n) + receipt := bc.GetReceiptsByHash(block.Hash()) + if receipt == nil { + return fmt.Errorf("receipts for #%d missing", n) } - td.Add(td, block.Difficulty()) - if err := w.Add(block, receipts, new(big.Int).Set(td)); err != nil { + + // For pre-merge blocks, pass accumulated TD. + // For post-merge blocks (difficulty == 0), pass nil TD. + var blockTD *big.Int + if block.Difficulty().Sign() != 0 { + td.Add(td, block.Difficulty()) + blockTD = new(big.Int).Set(td) + } + + if err := builder.Add(block, receipt, blockTD); err != nil { return err } } - root, err := w.Finalize() + id, err := builder.Finalize() if err != nil { - return fmt.Errorf("export failed to finalize %d: %w", step/i, err) + return err } - // Set correct filename with root. - os.Rename(filename, filepath.Join(dir, era.Filename(network, int(i/step), root))) - - // Compute checksum of entire Era1. if _, err := f.Seek(0, io.SeekStart); err != nil { return err } + h.Reset() + buf.Reset() if _, err := io.Copy(h, f); err != nil { - return fmt.Errorf("unable to calculate checksum: %w", err) + return err } checksums = append(checksums, common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex()) - h.Reset() - buf.Reset() - return nil - }() - if err != nil { + + // Close before rename. It's required on Windows. + f.Close() + final := filepath.Join(dir, filename(network, idx, id)) + return os.Rename(tmpPath, final) + }(); err != nil { return err } + if time.Since(reported) >= 8*time.Second { - log.Info("Exporting blocks", "exported", i, "elapsed", common.PrettyDuration(time.Since(start))) + log.Info("export progress", "exported", batch, "elapsed", common.PrettyDuration(time.Since(start))) reported = time.Now() } } - os.WriteFile(filepath.Join(dir, "checksums.txt"), []byte(strings.Join(checksums, "\n")), os.ModePerm) - - log.Info("Exported blockchain to", "dir", dir) - + _ = os.WriteFile(filepath.Join(dir, "checksums.txt"), []byte(strings.Join(checksums, "\n")), os.ModePerm) return nil } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index f22db37971..7fdb7e1317 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -114,7 +114,7 @@ var ( Usage: "Root directory for era1 history (default = inside ancient/chain)", Category: flags.EthCategory, } - MinFreeDiskSpaceFlag = &flags.DirectoryFlag{ + MinFreeDiskSpaceFlag = &cli.IntFlag{ Name: "datadir.minfreedisk", Usage: "Minimum free disk space in MB, once reached triggers auto shut down (default = --cache.gc converted to MB, 0 = disabled)", Category: flags.EthCategory, @@ -137,7 +137,7 @@ var ( } NetworkIdFlag = &cli.Uint64Flag{ Name: "networkid", - Usage: "Explicitly set network id (integer)(For testnets: use --sepolia, --holesky, --hoodi instead)", + Usage: "Explicitly set network ID (integer)(For testnets: use --sepolia, --holesky, --hoodi instead)", Value: ethconfig.Defaults.NetworkId, Category: flags.EthCategory, } @@ -218,7 +218,15 @@ var ( Usage: "Max number of elements (0 = no limit)", Value: 0, } - + AccountFlag = &cli.StringFlag{ + Name: "account", + Usage: "Specifies the account address or hash to traverse a single storage trie", + } + OutputFileFlag = &cli.StringFlag{ + Name: "output", + Usage: "Writes the result in json to the output", + Value: "", + } SnapshotFlag = &cli.BoolFlag{ Name: "snapshot", Usage: `Enables snapshot-database mode (default = enable)`, @@ -295,6 +303,18 @@ var ( Value: ethconfig.Defaults.StateHistory, Category: flags.StateCategory, } + TrienodeHistoryFlag = &cli.Int64Flag{ + Name: "history.trienode", + Usage: "Number of recent blocks to retain trienode history for, only relevant in state.scheme=path (default/negative = disabled, 0 = entire chain)", + Value: ethconfig.Defaults.TrienodeHistory, + Category: flags.StateCategory, + } + TrienodeHistoryFullValueCheckpointFlag = &cli.UintFlag{ + Name: "history.trienode.full-value-checkpoint", + Usage: "The frequency of full-value encoding. Every n-th node is stored in full-value format; all other nodes are stored as diffs relative to their predecessor", + Value: uint(ethconfig.Defaults.NodeFullValueCheckpoint), + Category: flags.StateCategory, + } TransactionHistoryFlag = &cli.Uint64Flag{ Name: "history.transactions", Usage: "Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain)", @@ -303,7 +323,7 @@ var ( } ChainHistoryFlag = &cli.StringFlag{ Name: "history.chain", - Usage: `Blockchain history retention ("all" or "postmerge")`, + Usage: `Blockchain history retention ("all", "postmerge", or "postprague")`, Value: ethconfig.Defaults.HistoryMode.String(), Category: flags.StateCategory, } @@ -468,8 +488,8 @@ var ( // Performance tuning settings CacheFlag = &cli.IntFlag{ Name: "cache", - Usage: "Megabytes of memory allocated to internal caching (default = 4096 mainnet full node, 128 light mode)", - Value: 1024, + Usage: "Megabytes of memory allocated to internal caching", + Value: 4096, Category: flags.PerfCategory, } CacheDatabaseFlag = &cli.IntFlag{ @@ -553,6 +573,11 @@ var ( Usage: "0x prefixed public address for the pending block producer (not used for actual block production)", Category: flags.MinerCategory, } + MinerMaxBlobsFlag = &cli.IntFlag{ + Name: "miner.maxblobs", + Usage: "Maximum number of blobs per block (falls back to protocol maximum if unspecified)", + Category: flags.MinerCategory, + } // Account settings PasswordFileFlag = &cli.PathFlag{ @@ -631,6 +656,12 @@ var ( Value: ethconfig.Defaults.TxSyncMaxTimeout, Category: flags.APICategory, } + RPCGlobalRangeLimitFlag = &cli.Uint64Flag{ + Name: "rpc.rangelimit", + Usage: "Maximum block range (end - begin) allowed for range queries (0 = unlimited)", + Value: ethconfig.Defaults.RangeLimit, + Category: flags.APICategory, + } // Authenticated RPC HTTP settings AuthListenFlag = &cli.StringFlag{ Name: "authrpc.addr", @@ -667,6 +698,12 @@ var ( Usage: "Disables db compaction after import", Category: flags.LoggingCategory, } + LogSlowBlockFlag = &cli.DurationFlag{ + Name: "debug.logslowblock", + Usage: "Block execution time threshold beyond which detailed statistics will be logged (0 logs all blocks, negative means disable)", + Value: ethconfig.Defaults.SlowBlockThreshold, + Category: flags.LoggingCategory, + } // MISC settings SyncTargetFlag = &cli.StringFlag{ @@ -859,14 +896,14 @@ var ( Aliases: []string{"discv4"}, Usage: "Enables the V4 discovery mechanism", Category: flags.NetworkingCategory, - Value: true, + Value: node.DefaultConfig.P2P.DiscoveryV4, } DiscoveryV5Flag = &cli.BoolFlag{ Name: "discovery.v5", Aliases: []string{"discv5"}, Usage: "Enables the V5 discovery mechanism", Category: flags.NetworkingCategory, - Value: true, + Value: node.DefaultConfig.P2P.DiscoveryV5, } NetrestrictFlag = &cli.StringFlag{ Name: "netrestrict", @@ -987,6 +1024,13 @@ Please note that --` + MetricsHTTPFlag.Name + ` must be set to start the server. Category: flags.MetricsCategory, } + MetricsInfluxDBIntervalFlag = &cli.DurationFlag{ + Name: "metrics.influxdb.interval", + Usage: "Interval between metrics reports to InfluxDB (with time unit, e.g. 10s)", + Value: metrics.DefaultConfig.InfluxDBInterval, + Category: flags.MetricsCategory, + } + MetricsEnableInfluxDBV2Flag = &cli.BoolFlag{ Name: "metrics.influxdbv2", Usage: "Enable metrics export/push to an external InfluxDB v2 database", @@ -1013,6 +1057,55 @@ Please note that --` + MetricsHTTPFlag.Name + ` must be set to start the server. Value: metrics.DefaultConfig.InfluxDBOrganization, Category: flags.MetricsCategory, } + + // RPC Telemetry + RPCTelemetryFlag = &cli.BoolFlag{ + Name: "rpc.telemetry", + Usage: "Enable RPC telemetry", + Category: flags.APICategory, + } + + RPCTelemetryEndpointFlag = &cli.StringFlag{ + Name: "rpc.telemetry.endpoint", + Usage: "Defines where RPC telemetry is sent (e.g., http://localhost:4318)", + Category: flags.APICategory, + } + + RPCTelemetryUserFlag = &cli.StringFlag{ + Name: "rpc.telemetry.username", + Usage: "HTTP Basic Auth username for OpenTelemetry", + Category: flags.APICategory, + } + + RPCTelemetryPasswordFlag = &cli.StringFlag{ + Name: "rpc.telemetry.password", + Usage: "HTTP Basic Auth password for OpenTelemetry", + Category: flags.APICategory, + } + + RPCTelemetryInstanceIDFlag = &cli.StringFlag{ + Name: "rpc.telemetry.instance-id", + Usage: "OpenTelemetry instance ID", + Category: flags.APICategory, + } + + RPCTelemetryTagsFlag = &cli.StringFlag{ + Name: "rpc.telemetry.tags", + Usage: "Comma-separated tags (key/values) added as attributes to the OpenTelemetry resource struct", + Category: flags.APICategory, + } + + RPCTelemetrySampleRatioFlag = &cli.Float64Flag{ + Name: "rpc.telemetry.sample-ratio", + Usage: "Defines the sampling ratio for RPC telemetry (0.0 to 1.0)", + Value: 1.0, + Category: flags.APICategory, + } + // Era flags are a group of flags related to the era archive format. + EraFormatFlag = &cli.StringFlag{ + Name: "era.format", + Usage: "Archive format: 'era1' or 'erae'", + } ) var ( @@ -1363,13 +1456,17 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) { cfg.MaxPendingPeers = ctx.Int(MaxPendingPeersFlag.Name) } if ctx.IsSet(NoDiscoverFlag.Name) { - cfg.NoDiscovery = true + cfg.NoDiscovery = ctx.Bool(NoDiscoverFlag.Name) } flags.CheckExclusive(ctx, DiscoveryV4Flag, NoDiscoverFlag) flags.CheckExclusive(ctx, DiscoveryV5Flag, NoDiscoverFlag) - cfg.DiscoveryV4 = ctx.Bool(DiscoveryV4Flag.Name) - cfg.DiscoveryV5 = ctx.Bool(DiscoveryV5Flag.Name) + if ctx.IsSet(DiscoveryV4Flag.Name) { + cfg.DiscoveryV4 = ctx.Bool(DiscoveryV4Flag.Name) + } + if ctx.IsSet(DiscoveryV5Flag.Name) { + cfg.DiscoveryV5 = ctx.Bool(DiscoveryV5Flag.Name) + } if netrestrict := ctx.String(NetrestrictFlag.Name); netrestrict != "" { list, err := netutil.ParseNetlist(netrestrict) @@ -1399,6 +1496,7 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { setNodeUserIdent(ctx, cfg) SetDataDir(ctx, cfg) setSmartCard(ctx, cfg) + setOpenTelemetry(ctx, cfg) if ctx.IsSet(JWTSecretFlag.Name) { cfg.JWTSecret = ctx.String(JWTSecretFlag.Name) @@ -1415,7 +1513,7 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { cfg.KeyStoreDir = ctx.String(KeyStoreDirFlag.Name) } if ctx.IsSet(DeveloperFlag.Name) { - cfg.UseLightweightKDF = true + cfg.UseLightweightKDF = ctx.Bool(DeveloperFlag.Name) } if ctx.IsSet(LightKDFFlag.Name) { cfg.UseLightweightKDF = ctx.Bool(LightKDFFlag.Name) @@ -1466,6 +1564,35 @@ func setSmartCard(ctx *cli.Context, cfg *node.Config) { cfg.SmartCardDaemonPath = path } +func setOpenTelemetry(ctx *cli.Context, cfg *node.Config) { + tcfg := &cfg.OpenTelemetry + if ctx.IsSet(RPCTelemetryFlag.Name) { + tcfg.Enabled = ctx.Bool(RPCTelemetryFlag.Name) + } + if ctx.IsSet(RPCTelemetryEndpointFlag.Name) { + tcfg.Endpoint = ctx.String(RPCTelemetryEndpointFlag.Name) + } + if ctx.IsSet(RPCTelemetryUserFlag.Name) { + tcfg.AuthUser = ctx.String(RPCTelemetryUserFlag.Name) + } + if ctx.IsSet(RPCTelemetryPasswordFlag.Name) { + tcfg.AuthPassword = ctx.String(RPCTelemetryPasswordFlag.Name) + } + if ctx.IsSet(RPCTelemetryInstanceIDFlag.Name) { + tcfg.InstanceID = ctx.String(RPCTelemetryInstanceIDFlag.Name) + } + if ctx.IsSet(RPCTelemetryTagsFlag.Name) { + tcfg.Tags = ctx.String(RPCTelemetryTagsFlag.Name) + } + if ctx.IsSet(RPCTelemetrySampleRatioFlag.Name) { + tcfg.SampleRatio = ctx.Float64(RPCTelemetrySampleRatioFlag.Name) + } + + if tcfg.Endpoint != "" && !tcfg.Enabled { + log.Warn(fmt.Sprintf("OpenTelemetry endpoint configured but telemetry is not enabled, use --%s to enable.", RPCTelemetryFlag.Name)) + } +} + func SetDataDir(ctx *cli.Context, cfg *node.Config) { switch { case ctx.IsSet(DataDirFlag.Name): @@ -1571,6 +1698,9 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) { log.Warn("The flag --miner.newpayload-timeout is deprecated and will be removed, please use --miner.recommit") cfg.Recommit = ctx.Duration(MinerNewPayloadTimeoutFlag.Name) } + if ctx.IsSet(MinerMaxBlobsFlag.Name) { + cfg.MaxBlobsPerBlock = ctx.Int(MinerMaxBlobsFlag.Name) + } } func setRequiredBlocks(ctx *cli.Context, cfg *ethconfig.Config) { @@ -1603,8 +1733,8 @@ func setRequiredBlocks(ctx *cli.Context, cfg *ethconfig.Config) { // SetEthConfig applies eth-related command line flags to the config. func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { - // Avoid conflicting network flags, don't allow network id override on preset networks - flags.CheckExclusive(ctx, MainnetFlag, DeveloperFlag, SepoliaFlag, HoleskyFlag, HoodiFlag, NetworkIdFlag, OverrideGenesisFlag) + // Avoid conflicting network flags + flags.CheckExclusive(ctx, MainnetFlag, DeveloperFlag, SepoliaFlag, HoleskyFlag, HoodiFlag, OverrideGenesisFlag) flags.CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer // Set configurations from CLI flags @@ -1651,9 +1781,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { } } - if ctx.IsSet(NetworkIdFlag.Name) { - cfg.NetworkId = ctx.Uint64(NetworkIdFlag.Name) - } if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheDatabaseFlag.Name) { cfg.DatabaseCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheDatabaseFlag.Name) / 100 } @@ -1674,8 +1801,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.IsSet(CacheNoPrefetchFlag.Name) { cfg.NoPrefetch = ctx.Bool(CacheNoPrefetchFlag.Name) } - // Read the value from the flag no matter if it's set or not. - cfg.Preimages = ctx.Bool(CachePreimagesFlag.Name) + if ctx.IsSet(CachePreimagesFlag.Name) { + cfg.Preimages = ctx.Bool(CachePreimagesFlag.Name) + } if cfg.NoPruning && !cfg.Preimages { cfg.Preimages = true log.Info("Enabling recording of key preimages since archive mode is used") @@ -1683,6 +1811,12 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.IsSet(StateHistoryFlag.Name) { cfg.StateHistory = ctx.Uint64(StateHistoryFlag.Name) } + if ctx.IsSet(TrienodeHistoryFlag.Name) { + cfg.TrienodeHistory = ctx.Int64(TrienodeHistoryFlag.Name) + } + if ctx.IsSet(TrienodeHistoryFullValueCheckpointFlag.Name) { + cfg.NodeFullValueCheckpoint = uint32(ctx.Uint(TrienodeHistoryFullValueCheckpointFlag.Name)) + } if ctx.IsSet(StateSchemeFlag.Name) { cfg.StateScheme = ctx.String(StateSchemeFlag.Name) } @@ -1708,7 +1842,10 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { cfg.LogHistory = ctx.Uint64(LogHistoryFlag.Name) } if ctx.IsSet(LogNoHistoryFlag.Name) { - cfg.LogNoHistory = true + cfg.LogNoHistory = ctx.Bool(LogNoHistoryFlag.Name) + } + if ctx.IsSet(LogSlowBlockFlag.Name) { + cfg.SlowBlockThreshold = ctx.Duration(LogSlowBlockFlag.Name) } if ctx.IsSet(LogExportCheckpointsFlag.Name) { cfg.LogExportCheckpoints = ctx.String(LogExportCheckpointsFlag.Name) @@ -1734,6 +1871,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.IsSet(RPCTxSyncMaxTimeoutFlag.Name) { cfg.TxSyncMaxTimeout = ctx.Duration(RPCTxSyncMaxTimeoutFlag.Name) } + if ctx.IsSet(RPCGlobalRangeLimitFlag.Name) { + cfg.RangeLimit = ctx.Uint64(RPCGlobalRangeLimitFlag.Name) + } if !ctx.Bool(SnapshotFlag.Name) || cfg.SnapshotCache == 0 { // If snap-sync is requested, this flag is also required if cfg.SyncMode == ethconfig.SnapSync { @@ -1903,10 +2043,18 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { } cfg.Genesis = genesis default: - if cfg.NetworkId == 1 { + if ctx.Uint64(NetworkIdFlag.Name) == 1 { SetDNSDiscoveryDefaults(cfg, params.MainnetGenesisHash) } } + if ctx.IsSet(NetworkIdFlag.Name) { + // Typically it's best to automatically set the network ID to the chainID, + // by not passing the --networkid flag at all. Emit a warning when set + // explicitly in case overriding the network ID is not the user's intention. + id := ctx.Uint64(NetworkIdFlag.Name) + log.Warn("Setting network ID with command-line flag", "id", id) + cfg.NetworkId = id + } // Set any dangling config values if ctx.String(CryptoKZGFlag.Name) != "gokzg" && ctx.String(CryptoKZGFlag.Name) != "ckzg" { Fatalf("--%s flag must be 'gokzg' or 'ckzg'", CryptoKZGFlag.Name) @@ -2070,6 +2218,7 @@ func RegisterFilterAPI(stack *node.Node, backend ethapi.Backend, ethcfg *ethconf filterSystem := filters.NewFilterSystem(backend, filters.Config{ LogCacheSize: ethcfg.FilterLogCacheSize, LogQueryLimit: ethcfg.LogQueryLimit, + RangeLimit: ethcfg.RangeLimit, }) stack.RegisterAPIs([]rpc.API{{ Namespace: "eth", @@ -2114,13 +2263,14 @@ func SetupMetrics(cfg *metrics.Config) { bucket = cfg.InfluxDBBucket organization = cfg.InfluxDBOrganization tagsMap = SplitTagsFlag(cfg.InfluxDBTags) + interval = cfg.InfluxDBInterval ) if enableExport { - log.Info("Enabling metrics export to InfluxDB") - go influxdb.InfluxDBWithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "geth.", tagsMap) + log.Info("Enabling metrics export to InfluxDB", "interval", interval) + go influxdb.InfluxDBWithTags(metrics.DefaultRegistry, interval, endpoint, database, username, password, "geth.", tagsMap) } else if enableExportV2 { - log.Info("Enabling metrics export to InfluxDB (v2)") - go influxdb.InfluxDBV2WithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, token, bucket, organization, "geth.", tagsMap) + log.Info("Enabling metrics export to InfluxDB (v2)", "interval", interval) + go influxdb.InfluxDBV2WithTags(metrics.DefaultRegistry, interval, endpoint, token, bucket, organization, "geth.", tagsMap) } // Expvar exporter. @@ -2272,15 +2422,18 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh Fatalf("%v", err) } options := &core.BlockChainConfig{ - TrieCleanLimit: ethconfig.Defaults.TrieCleanCache, - NoPrefetch: ctx.Bool(CacheNoPrefetchFlag.Name), - TrieDirtyLimit: ethconfig.Defaults.TrieDirtyCache, - ArchiveMode: ctx.String(GCModeFlag.Name) == "archive", - TrieTimeLimit: ethconfig.Defaults.TrieTimeout, - SnapshotLimit: ethconfig.Defaults.SnapshotCache, - Preimages: ctx.Bool(CachePreimagesFlag.Name), - StateScheme: scheme, - StateHistory: ctx.Uint64(StateHistoryFlag.Name), + TrieCleanLimit: ethconfig.Defaults.TrieCleanCache, + NoPrefetch: ctx.Bool(CacheNoPrefetchFlag.Name), + TrieDirtyLimit: ethconfig.Defaults.TrieDirtyCache, + ArchiveMode: ctx.String(GCModeFlag.Name) == "archive", + TrieTimeLimit: ethconfig.Defaults.TrieTimeout, + SnapshotLimit: ethconfig.Defaults.SnapshotCache, + Preimages: ctx.Bool(CachePreimagesFlag.Name), + StateScheme: scheme, + StateHistory: ctx.Uint64(StateHistoryFlag.Name), + TrienodeHistory: ctx.Int64(TrienodeHistoryFlag.Name), + NodeFullValueCheckpoint: uint32(ctx.Uint(TrienodeHistoryFullValueCheckpointFlag.Name)), + // Disable transaction indexing/unindexing. TxIndexer: nil, @@ -2292,6 +2445,13 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh // Enable state size tracking if enabled StateSizeTracking: ctx.Bool(StateSizeTrackingFlag.Name), + + // Configure the slow block statistic logger (disabled by default) + SlowBlockThreshold: ethconfig.Defaults.SlowBlockThreshold, + } + // Only enable slow block logging if the flag was explicitly set + if ctx.IsSet(LogSlowBlockFlag.Name) { + options.SlowBlockThreshold = ctx.Duration(LogSlowBlockFlag.Name) } if options.ArchiveMode && !options.Preimages { options.Preimages = true @@ -2315,8 +2475,6 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh } vmcfg := vm.Config{ EnablePreimageRecording: ctx.Bool(VMEnableDebugFlag.Name), - EnableWitnessStats: ctx.Bool(VMWitnessStatsFlag.Name), - StatelessSelfValidation: ctx.Bool(VMStatelessSelfValidationFlag.Name) || ctx.Bool(VMWitnessStatsFlag.Name), } if ctx.IsSet(VMTraceFlag.Name) { if name := ctx.String(VMTraceFlag.Name); name != "" { @@ -2330,6 +2488,12 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh } options.VmConfig = vmcfg + options.StatelessSelfValidation = ctx.Bool(VMStatelessSelfValidationFlag.Name) || ctx.Bool(VMWitnessStatsFlag.Name) + options.EnableWitnessStats = ctx.Bool(VMWitnessStatsFlag.Name) + + options.StatelessSelfValidation = ctx.Bool(VMStatelessSelfValidationFlag.Name) || ctx.Bool(VMWitnessStatsFlag.Name) + options.EnableWitnessStats = ctx.Bool(VMWitnessStatsFlag.Name) + chain, err := core.NewBlockChain(chainDb, nil, gspec, engine, options) if err != nil { Fatalf("Can't create BlockChain: %v", err) diff --git a/cmd/utils/history_test.go b/cmd/utils/history_test.go index d2024f8611..9dbbc1c551 100644 --- a/cmd/utils/history_test.go +++ b/cmd/utils/history_test.go @@ -33,6 +33,8 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/era/execdb" + "github.com/ethereum/go-ethereum/internal/era/onedb" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/triedb" @@ -44,136 +46,148 @@ var ( ) func TestHistoryImportAndExport(t *testing.T) { - var ( - key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - address = crypto.PubkeyToAddress(key.PublicKey) - genesis = &core.Genesis{ - Config: params.TestChainConfig, - Alloc: types.GenesisAlloc{address: {Balance: big.NewInt(1000000000000000000)}}, - } - signer = types.LatestSigner(genesis.Config) - ) - - // Generate chain. - db, blocks, _ := core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), int(count), func(i int, g *core.BlockGen) { - if i == 0 { - return - } - tx, err := types.SignNewTx(key, signer, &types.DynamicFeeTx{ - ChainID: genesis.Config.ChainID, - Nonce: uint64(i - 1), - GasTipCap: common.Big0, - GasFeeCap: g.PrevBlock(0).BaseFee(), - Gas: 50000, - To: &common.Address{0xaa}, - Value: big.NewInt(int64(i)), - Data: nil, - AccessList: nil, - }) - if err != nil { - t.Fatalf("error creating tx: %v", err) - } - g.AddTx(tx) - }) + for _, tt := range []struct { + name string + builder func(io.Writer) era.Builder + filename func(network string, epoch int, root common.Hash) string + from func(f era.ReadAtSeekCloser) (era.Era, error) + }{ + {"era1", onedb.NewBuilder, onedb.Filename, onedb.From}, + {"erae", execdb.NewBuilder, execdb.Filename, execdb.From}, + } { + t.Run(tt.name, func(t *testing.T) { + var ( + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + genesis = &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{address: {Balance: big.NewInt(1000000000000000000)}}, + } + signer = types.LatestSigner(genesis.Config) + ) - // Initialize BlockChain. - chain, err := core.NewBlockChain(db, nil, genesis, ethash.NewFaker(), nil) - if err != nil { - t.Fatalf("unable to initialize chain: %v", err) - } - if _, err := chain.InsertChain(blocks); err != nil { - t.Fatalf("error inserting chain: %v", err) - } + // Generate chain. + db, blocks, _ := core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), int(count), func(i int, g *core.BlockGen) { + if i == 0 { + return + } + tx, err := types.SignNewTx(key, signer, &types.DynamicFeeTx{ + ChainID: genesis.Config.ChainID, + Nonce: uint64(i - 1), + GasTipCap: common.Big0, + GasFeeCap: g.PrevBlock(0).BaseFee(), + Gas: 50000, + To: &common.Address{0xaa}, + Value: big.NewInt(int64(i)), + Data: nil, + AccessList: nil, + }) + if err != nil { + t.Fatalf("error creating tx: %v", err) + } + g.AddTx(tx) + }) - // Make temp directory for era files. - dir := t.TempDir() + // Initialize BlockChain. + chain, err := core.NewBlockChain(db, nil, genesis, ethash.NewFaker(), nil) + if err != nil { + t.Fatalf("unable to initialize chain: %v", err) + } + if _, err := chain.InsertChain(blocks); err != nil { + t.Fatalf("error inserting chain: %v", err) + } - // Export history to temp directory. - if err := ExportHistory(chain, dir, 0, count, step); err != nil { - t.Fatalf("error exporting history: %v", err) - } + // Make temp directory for era files. + dir := t.TempDir() - // Read checksums. - b, err := os.ReadFile(filepath.Join(dir, "checksums.txt")) - if err != nil { - t.Fatalf("failed to read checksums: %v", err) - } - checksums := strings.Split(string(b), "\n") + // Export history to temp directory. + if err := ExportHistory(chain, dir, 0, count, tt.builder, tt.filename); err != nil { + t.Fatalf("error exporting history: %v", err) + } - // Verify each Era. - entries, _ := era.ReadDir(dir, "mainnet") - for i, filename := range entries { - func() { - f, err := os.Open(filepath.Join(dir, filename)) + // Read checksums. + b, err := os.ReadFile(filepath.Join(dir, "checksums.txt")) if err != nil { - t.Fatalf("error opening era file: %v", err) - } - var ( - h = sha256.New() - buf = bytes.NewBuffer(nil) - ) - if _, err := io.Copy(h, f); err != nil { - t.Fatalf("unable to recalculate checksum: %v", err) + t.Fatalf("failed to read checksums: %v", err) } - if got, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; got != want { - t.Fatalf("checksum %d does not match: got %s, want %s", i, got, want) + checksums := strings.Split(string(b), "\n") + + // Verify each Era. + entries, _ := era.ReadDir(dir, "mainnet") + for i, filename := range entries { + func() { + f, err := os.Open(filepath.Join(dir, filename)) + if err != nil { + t.Fatalf("error opening era file: %v", err) + } + var ( + h = sha256.New() + buf = bytes.NewBuffer(nil) + ) + if _, err := io.Copy(h, f); err != nil { + t.Fatalf("unable to recalculate checksum: %v", err) + } + if got, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; got != want { + t.Fatalf("checksum %d does not match: got %s, want %s", i, got, want) + } + e, err := tt.from(f) + if err != nil { + t.Fatalf("error opening era: %v", err) + } + defer e.Close() + it, err := e.Iterator() + if err != nil { + t.Fatalf("error making era reader: %v", err) + } + for j := 0; it.Next(); j++ { + n := i*int(step) + j + if it.Error() != nil { + t.Fatalf("error reading block entry %d: %v", n, it.Error()) + } + block, receipts, err := it.BlockAndReceipts() + if err != nil { + t.Fatalf("error reading block entry %d: %v", n, err) + } + want := chain.GetBlockByNumber(uint64(n)) + if want, got := uint64(n), block.NumberU64(); want != got { + t.Fatalf("blocks out of order: want %d, got %d", want, got) + } + if want.Hash() != block.Hash() { + t.Fatalf("block hash mismatch %d: want %s, got %s", n, want.Hash().Hex(), block.Hash().Hex()) + } + if got := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); got != want.TxHash() { + t.Fatalf("tx hash %d mismatch: want %s, got %s", n, want.TxHash(), got) + } + if got := types.CalcUncleHash(block.Uncles()); got != want.UncleHash() { + t.Fatalf("uncle hash %d mismatch: want %s, got %s", n, want.UncleHash(), got) + } + if got := types.DeriveSha(receipts, trie.NewStackTrie(nil)); got != want.ReceiptHash() { + t.Fatalf("receipt root %d mismatch: want %s, got %s", n, want.ReceiptHash(), got) + } + } + }() } - e, err := era.From(f) + + // Now import Era. + db2, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{}) if err != nil { - t.Fatalf("error opening era: %v", err) + panic(err) } - defer e.Close() - it, err := era.NewIterator(e) + t.Cleanup(func() { + db2.Close() + }) + + genesis.MustCommit(db2, triedb.NewDatabase(db2, triedb.HashDefaults)) + imported, err := core.NewBlockChain(db2, nil, genesis, ethash.NewFaker(), nil) if err != nil { - t.Fatalf("error making era reader: %v", err) + t.Fatalf("unable to initialize chain: %v", err) } - for j := 0; it.Next(); j++ { - n := i*int(step) + j - if it.Error() != nil { - t.Fatalf("error reading block entry %d: %v", n, it.Error()) - } - block, receipts, err := it.BlockAndReceipts() - if err != nil { - t.Fatalf("error reading block entry %d: %v", n, err) - } - want := chain.GetBlockByNumber(uint64(n)) - if want, got := uint64(n), block.NumberU64(); want != got { - t.Fatalf("blocks out of order: want %d, got %d", want, got) - } - if want.Hash() != block.Hash() { - t.Fatalf("block hash mismatch %d: want %s, got %s", n, want.Hash().Hex(), block.Hash().Hex()) - } - if got := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); got != want.TxHash() { - t.Fatalf("tx hash %d mismatch: want %s, got %s", n, want.TxHash(), got) - } - if got := types.CalcUncleHash(block.Uncles()); got != want.UncleHash() { - t.Fatalf("uncle hash %d mismatch: want %s, got %s", n, want.UncleHash(), got) - } - if got := types.DeriveSha(receipts, trie.NewStackTrie(nil)); got != want.ReceiptHash() { - t.Fatalf("receipt root %d mismatch: want %s, got %s", n, want.ReceiptHash(), got) - } + if err := ImportHistory(imported, dir, "mainnet", tt.from); err != nil { + t.Fatalf("failed to import chain: %v", err) } - }() - } - - // Now import Era. - db2, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{}) - if err != nil { - panic(err) - } - t.Cleanup(func() { - db2.Close() - }) - - genesis.MustCommit(db2, triedb.NewDatabase(db2, triedb.HashDefaults)) - imported, err := core.NewBlockChain(db2, nil, genesis, ethash.NewFaker(), nil) - if err != nil { - t.Fatalf("unable to initialize chain: %v", err) - } - if err := ImportHistory(imported, dir, "mainnet"); err != nil { - t.Fatalf("failed to import chain: %v", err) - } - if have, want := imported.CurrentHeader(), chain.CurrentHeader(); have.Hash() != want.Hash() { - t.Fatalf("imported chain does not match expected, have (%d, %s) want (%d, %s)", have.Number, have.Hash(), want.Number, want.Hash()) + if have, want := imported.CurrentHeader(), chain.CurrentHeader(); have.Hash() != want.Hash() { + t.Fatalf("imported chain does not match expected, have (%d, %s) want (%d, %s)", have.Number, have.Hash(), want.Number, want.Hash()) + } + }) } } diff --git a/cmd/workload/README.md b/cmd/workload/README.md index 1b84dd05db..ee1d6acbc9 100644 --- a/cmd/workload/README.md +++ b/cmd/workload/README.md @@ -34,4 +34,5 @@ the following commands (in this directory) against a synced mainnet node: > go run . filtergen --queries queries/filter_queries_mainnet.json http://host:8545 > go run . historygen --history-tests queries/history_mainnet.json http://host:8545 > go run . tracegen --trace-tests queries/trace_mainnet.json --trace-start 4000000 --trace-end 4000100 http://host:8545 +> go run . proofgen --proof-tests queries/proof_mainnet.json --proof-states 3000 http://host:8545 ``` diff --git a/cmd/workload/main.go b/cmd/workload/main.go index 8ac0e5b6cb..4ee894e962 100644 --- a/cmd/workload/main.go +++ b/cmd/workload/main.go @@ -48,6 +48,7 @@ func init() { historyGenerateCommand, filterGenerateCommand, traceGenerateCommand, + proofGenerateCommand, filterPerfCommand, filterFuzzCommand, } diff --git a/cmd/workload/prooftest.go b/cmd/workload/prooftest.go new file mode 100644 index 0000000000..dcc063d30e --- /dev/null +++ b/cmd/workload/prooftest.go @@ -0,0 +1,105 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "context" + "encoding/json" + "fmt" + "math/big" + "os" + "strings" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/utesting" + "github.com/urfave/cli/v2" +) + +// proofTest is the content of a state-proof test. +type proofTest struct { + BlockNumbers []uint64 `json:"blockNumbers"` + Addresses [][]common.Address `json:"addresses"` + StorageKeys [][][]string `json:"storageKeys"` + Results [][]common.Hash `json:"results"` +} + +type proofTestSuite struct { + cfg testConfig + tests proofTest + invalidDir string +} + +func newProofTestSuite(cfg testConfig, ctx *cli.Context) *proofTestSuite { + s := &proofTestSuite{ + cfg: cfg, + invalidDir: ctx.String(proofTestInvalidOutputFlag.Name), + } + if err := s.loadTests(); err != nil { + exit(err) + } + return s +} + +func (s *proofTestSuite) loadTests() error { + file, err := s.cfg.fsys.Open(s.cfg.proofTestFile) + if err != nil { + // If not found in embedded FS, try to load it from disk + if !os.IsNotExist(err) { + return err + } + file, err = os.OpenFile(s.cfg.proofTestFile, os.O_RDONLY, 0666) + if err != nil { + return fmt.Errorf("can't open proofTestFile: %v", err) + } + } + defer file.Close() + if err := json.NewDecoder(file).Decode(&s.tests); err != nil { + return fmt.Errorf("invalid JSON in %s: %v", s.cfg.proofTestFile, err) + } + if len(s.tests.BlockNumbers) == 0 { + return fmt.Errorf("proofTestFile %s has no test data", s.cfg.proofTestFile) + } + return nil +} + +func (s *proofTestSuite) allTests() []workloadTest { + return []workloadTest{ + newArchiveWorkloadTest("Proof/GetProof", s.getProof), + } +} + +func (s *proofTestSuite) getProof(t *utesting.T) { + ctx := context.Background() + for i, blockNumber := range s.tests.BlockNumbers { + for j := 0; j < len(s.tests.Addresses[i]); j++ { + res, err := s.cfg.client.Geth.GetProof(ctx, s.tests.Addresses[i][j], s.tests.StorageKeys[i][j], big.NewInt(int64(blockNumber))) + if err != nil { + t.Errorf("State proving fails, blockNumber: %d, address: %x, keys: %v, err: %v\n", blockNumber, s.tests.Addresses[i][j], strings.Join(s.tests.StorageKeys[i][j], " "), err) + continue + } + blob, err := json.Marshal(res) + if err != nil { + t.Fatalf("State proving fails: error %v", err) + continue + } + if crypto.Keccak256Hash(blob) != s.tests.Results[i][j] { + t.Errorf("State proof mismatch, %d, number: %d, address: %x, keys: %v: invalid result", i, blockNumber, s.tests.Addresses[i][j], strings.Join(s.tests.StorageKeys[i][j], " ")) + } + } + } +} diff --git a/cmd/workload/prooftestgen.go b/cmd/workload/prooftestgen.go new file mode 100644 index 0000000000..5d92eea114 --- /dev/null +++ b/cmd/workload/prooftestgen.go @@ -0,0 +1,355 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see + +package main + +import ( + "context" + "encoding/json" + "fmt" + "math/big" + "math/rand" + "os" + "path/filepath" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/tracers" + "github.com/ethereum/go-ethereum/eth/tracers/native" + "github.com/ethereum/go-ethereum/internal/flags" + "github.com/ethereum/go-ethereum/internal/testrand" + "github.com/ethereum/go-ethereum/log" + "github.com/urfave/cli/v2" +) + +var ( + proofGenerateCommand = &cli.Command{ + Name: "proofgen", + Usage: "Generates tests for state proof verification", + ArgsUsage: "", + Action: generateProofTests, + Flags: []cli.Flag{ + proofTestFileFlag, + proofTestResultOutputFlag, + proofTestStatesFlag, + proofTestStartBlockFlag, + proofTestEndBlockFlag, + }, + } + + proofTestFileFlag = &cli.StringFlag{ + Name: "proof-tests", + Usage: "JSON file containing proof test queries", + Value: "proof_tests.json", + Category: flags.TestingCategory, + } + proofTestResultOutputFlag = &cli.StringFlag{ + Name: "proof-output", + Usage: "Folder containing detailed trace output files", + Value: "", + Category: flags.TestingCategory, + } + proofTestStatesFlag = &cli.Int64Flag{ + Name: "proof-states", + Usage: "Number of states to generate proof against", + Value: 10000, + Category: flags.TestingCategory, + } + proofTestInvalidOutputFlag = &cli.StringFlag{ + Name: "proof-invalid", + Usage: "Folder containing the mismatched state proof output files", + Value: "", + Category: flags.TestingCategory, + } + proofTestStartBlockFlag = &cli.Uint64Flag{ + Name: "proof-start", + Usage: "The number of starting block for proof verification (included)", + Category: flags.TestingCategory, + } + proofTestEndBlockFlag = &cli.Uint64Flag{ + Name: "proof-end", + Usage: "The number of ending block for proof verification (excluded)", + Category: flags.TestingCategory, + } +) + +type proofGenerator func(cli *client, startBlock uint64, endBlock uint64, number int) ([]uint64, [][]common.Address, [][][]string, error) + +func genAccountProof(cli *client, startBlock uint64, endBlock uint64, number int) ([]uint64, [][]common.Address, [][][]string, error) { + var ( + blockNumbers []uint64 + accountAddresses [][]common.Address + storageKeys [][][]string + nAccounts int + ctx = context.Background() + start = time.Now() + ) + chainID, err := cli.Eth.ChainID(ctx) + if err != nil { + return nil, nil, nil, err + } + signer := types.LatestSignerForChainID(chainID) + + for { + if nAccounts >= number { + break + } + blockNumber := uint64(rand.Intn(int(endBlock-startBlock))) + startBlock + + block, err := cli.Eth.BlockByNumber(context.Background(), big.NewInt(int64(blockNumber))) + if err != nil { + continue + } + var ( + addresses []common.Address + keys [][]string + gather = func(address common.Address) { + addresses = append(addresses, address) + keys = append(keys, nil) + nAccounts++ + } + ) + for _, tx := range block.Transactions() { + if nAccounts >= number { + break + } + sender, err := signer.Sender(tx) + if err != nil { + log.Error("Failed to resolve the sender address", "hash", tx.Hash(), "err", err) + continue + } + gather(sender) + + if tx.To() != nil { + gather(*tx.To()) + } + } + blockNumbers = append(blockNumbers, blockNumber) + accountAddresses = append(accountAddresses, addresses) + storageKeys = append(storageKeys, keys) + } + log.Info("Generated tests for account proof", "blocks", len(blockNumbers), "accounts", nAccounts, "elapsed", common.PrettyDuration(time.Since(start))) + return blockNumbers, accountAddresses, storageKeys, nil +} + +func genNonExistentAccountProof(cli *client, startBlock uint64, endBlock uint64, number int) ([]uint64, [][]common.Address, [][][]string, error) { + var ( + blockNumbers []uint64 + accountAddresses [][]common.Address + storageKeys [][][]string + total int + ) + for i := 0; i < number/5; i++ { + var ( + addresses []common.Address + keys [][]string + blockNumber = uint64(rand.Intn(int(endBlock-startBlock))) + startBlock + ) + for j := 0; j < 5; j++ { + addresses = append(addresses, testrand.Address()) + keys = append(keys, nil) + } + total += len(addresses) + blockNumbers = append(blockNumbers, blockNumber) + accountAddresses = append(accountAddresses, addresses) + storageKeys = append(storageKeys, keys) + } + log.Info("Generated tests for non-existing account proof", "blocks", len(blockNumbers), "accounts", total) + return blockNumbers, accountAddresses, storageKeys, nil +} + +func genStorageProof(cli *client, startBlock uint64, endBlock uint64, number int) ([]uint64, [][]common.Address, [][][]string, error) { + var ( + blockNumbers []uint64 + accountAddresses [][]common.Address + storageKeys [][][]string + + nAccounts int + nStorages int + start = time.Now() + ) + for { + if nAccounts+nStorages >= number { + break + } + blockNumber := uint64(rand.Intn(int(endBlock-startBlock))) + startBlock + + block, err := cli.Eth.BlockByNumber(context.Background(), big.NewInt(int64(blockNumber))) + if err != nil { + continue + } + var ( + addresses []common.Address + slots [][]string + tracer = "prestateTracer" + configBlob, _ = json.Marshal(native.PrestateTracerConfig{ + DiffMode: false, + DisableCode: true, + DisableStorage: false, + }) + ) + for _, tx := range block.Transactions() { + if nAccounts+nStorages >= number { + break + } + if tx.To() == nil { + continue + } + ret, err := cli.Geth.TraceTransaction(context.Background(), tx.Hash(), &tracers.TraceConfig{ + Tracer: &tracer, + TracerConfig: configBlob, + }) + if err != nil { + log.Error("Failed to trace the transaction", "blockNumber", blockNumber, "hash", tx.Hash(), "err", err) + continue + } + blob, err := json.Marshal(ret) + if err != nil { + log.Error("Failed to marshal data", "err", err) + continue + } + var accounts map[common.Address]*types.Account + if err := json.Unmarshal(blob, &accounts); err != nil { + log.Error("Failed to decode trace result", "blockNumber", blockNumber, "hash", tx.Hash(), "err", err) + continue + } + for addr, account := range accounts { + if len(account.Storage) == 0 { + continue + } + addresses = append(addresses, addr) + nAccounts += 1 + + var keys []string + for k := range account.Storage { + keys = append(keys, k.Hex()) + } + nStorages += len(keys) + + var emptyKeys []string + for i := 0; i < 3; i++ { + emptyKeys = append(emptyKeys, testrand.Hash().Hex()) + } + nStorages += len(emptyKeys) + + slots = append(slots, append(keys, emptyKeys...)) + } + } + blockNumbers = append(blockNumbers, blockNumber) + accountAddresses = append(accountAddresses, addresses) + storageKeys = append(storageKeys, slots) + } + log.Info("Generated tests for storage proof", "blocks", len(blockNumbers), "accounts", nAccounts, "storages", nStorages, "elapsed", common.PrettyDuration(time.Since(start))) + return blockNumbers, accountAddresses, storageKeys, nil +} + +func genProofRequests(cli *client, startBlock, endBlock uint64, states int) (*proofTest, error) { + var ( + blockNumbers []uint64 + accountAddresses [][]common.Address + storageKeys [][][]string + ) + ratio := []float64{0.2, 0.1, 0.7} + for i, fn := range []proofGenerator{genAccountProof, genNonExistentAccountProof, genStorageProof} { + numbers, addresses, keys, err := fn(cli, startBlock, endBlock, int(float64(states)*ratio[i])) + if err != nil { + return nil, err + } + blockNumbers = append(blockNumbers, numbers...) + accountAddresses = append(accountAddresses, addresses...) + storageKeys = append(storageKeys, keys...) + } + return &proofTest{ + BlockNumbers: blockNumbers, + Addresses: accountAddresses, + StorageKeys: storageKeys, + }, nil +} + +func generateProofTests(clictx *cli.Context) error { + var ( + client = makeClient(clictx) + ctx = context.Background() + states = clictx.Int(proofTestStatesFlag.Name) + outputFile = clictx.String(proofTestFileFlag.Name) + outputDir = clictx.String(proofTestResultOutputFlag.Name) + startBlock = clictx.Uint64(proofTestStartBlockFlag.Name) + endBlock = clictx.Uint64(proofTestEndBlockFlag.Name) + ) + head, err := client.Eth.BlockNumber(ctx) + if err != nil { + exit(err) + } + if startBlock > head || endBlock > head { + return fmt.Errorf("chain is out of proof range, head %d, start: %d, limit: %d", head, startBlock, endBlock) + } + if endBlock == 0 { + endBlock = head + } + log.Info("Generating proof states", "startBlock", startBlock, "endBlock", endBlock, "states", states) + + test, err := genProofRequests(client, startBlock, endBlock, states) + if err != nil { + exit(err) + } + for i, blockNumber := range test.BlockNumbers { + var hashes []common.Hash + for j := 0; j < len(test.Addresses[i]); j++ { + res, err := client.Geth.GetProof(ctx, test.Addresses[i][j], test.StorageKeys[i][j], big.NewInt(int64(blockNumber))) + if err != nil { + log.Error("Failed to prove the state", "number", blockNumber, "address", test.Addresses[i][j], "slots", len(test.StorageKeys[i][j]), "err", err) + continue + } + blob, err := json.Marshal(res) + if err != nil { + return err + } + hashes = append(hashes, crypto.Keccak256Hash(blob)) + + writeStateProof(outputDir, blockNumber, test.Addresses[i][j], res) + } + test.Results = append(test.Results, hashes) + } + writeJSON(outputFile, test) + return nil +} + +func writeStateProof(dir string, blockNumber uint64, address common.Address, result any) { + if dir == "" { + return + } + // Ensure the directory exists + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + exit(fmt.Errorf("failed to create directories: %w", err)) + } + fname := fmt.Sprintf("%d-%x", blockNumber, address) + name := filepath.Join(dir, fname) + file, err := os.Create(name) + if err != nil { + exit(fmt.Errorf("error creating %s: %v", name, err)) + return + } + defer file.Close() + + data, _ := json.MarshalIndent(result, "", " ") + _, err = file.Write(data) + if err != nil { + exit(fmt.Errorf("error writing %s: %v", name, err)) + return + } +} diff --git a/cmd/workload/testsuite.go b/cmd/workload/testsuite.go index 25dc17a49e..4e33522f1b 100644 --- a/cmd/workload/testsuite.go +++ b/cmd/workload/testsuite.go @@ -50,7 +50,9 @@ var ( filterQueryFileFlag, historyTestFileFlag, traceTestFileFlag, + proofTestFileFlag, traceTestInvalidOutputFlag, + proofTestInvalidOutputFlag, }, } testPatternFlag = &cli.StringFlag{ @@ -95,6 +97,7 @@ type testConfig struct { historyTestFile string historyPruneBlock *uint64 traceTestFile string + proofTestFile string } var errPrunedHistory = errors.New("attempt to access pruned history") @@ -145,8 +148,16 @@ func testConfigFromCLI(ctx *cli.Context) (cfg testConfig) { } else { cfg.traceTestFile = "queries/trace_mainnet.json" } + if ctx.IsSet(proofTestFileFlag.Name) { + cfg.proofTestFile = ctx.String(proofTestFileFlag.Name) + } else { + cfg.proofTestFile = "queries/proof_mainnet.json" + } + cfg.historyPruneBlock = new(uint64) - *cfg.historyPruneBlock = history.PrunePoints[params.MainnetGenesisHash].BlockNumber + if p, err := history.NewPolicy(history.KeepPostMerge, params.MainnetGenesisHash); err == nil { + *cfg.historyPruneBlock = p.Target.BlockNumber + } case ctx.Bool(testSepoliaFlag.Name): cfg.fsys = builtinTestFiles if ctx.IsSet(filterQueryFileFlag.Name) { @@ -164,13 +175,22 @@ func testConfigFromCLI(ctx *cli.Context) (cfg testConfig) { } else { cfg.traceTestFile = "queries/trace_sepolia.json" } + if ctx.IsSet(proofTestFileFlag.Name) { + cfg.proofTestFile = ctx.String(proofTestFileFlag.Name) + } else { + cfg.proofTestFile = "queries/proof_sepolia.json" + } + cfg.historyPruneBlock = new(uint64) - *cfg.historyPruneBlock = history.PrunePoints[params.SepoliaGenesisHash].BlockNumber + if p, err := history.NewPolicy(history.KeepPostMerge, params.SepoliaGenesisHash); err == nil { + *cfg.historyPruneBlock = p.Target.BlockNumber + } default: cfg.fsys = os.DirFS(".") cfg.filterQueryFile = ctx.String(filterQueryFileFlag.Name) cfg.historyTestFile = ctx.String(historyTestFileFlag.Name) cfg.traceTestFile = ctx.String(traceTestFileFlag.Name) + cfg.proofTestFile = ctx.String(proofTestFileFlag.Name) } return cfg } @@ -222,11 +242,13 @@ func runTestCmd(ctx *cli.Context) error { filterSuite := newFilterTestSuite(cfg) historySuite := newHistoryTestSuite(cfg) traceSuite := newTraceTestSuite(cfg, ctx) + proofSuite := newProofTestSuite(cfg, ctx) // Filter test cases. tests := filterSuite.allTests() tests = append(tests, historySuite.allTests()...) tests = append(tests, traceSuite.allTests()...) + tests = append(tests, proofSuite.allTests()...) utests := filterTests(tests, ctx.String(testPatternFlag.Name), func(t workloadTest) bool { if t.Slow && !ctx.Bool(testSlowFlag.Name) { diff --git a/common/bitutil/bitutil.go b/common/bitutil/bitutil.go index a18a6d18ee..578da1cf49 100644 --- a/common/bitutil/bitutil.go +++ b/common/bitutil/bitutil.go @@ -8,6 +8,7 @@ package bitutil import ( + "crypto/subtle" "runtime" "unsafe" ) @@ -17,46 +18,16 @@ const supportsUnaligned = runtime.GOARCH == "386" || runtime.GOARCH == "amd64" | // XORBytes xors the bytes in a and b. The destination is assumed to have enough // space. Returns the number of bytes xor'd. +// +// If dst does not have length at least n, +// XORBytes panics without writing anything to dst. +// +// dst and x or y may overlap exactly or not at all, +// otherwise XORBytes may panic. +// +// Deprecated: use crypto/subtle.XORBytes func XORBytes(dst, a, b []byte) int { - if supportsUnaligned { - return fastXORBytes(dst, a, b) - } - return safeXORBytes(dst, a, b) -} - -// fastXORBytes xors in bulk. It only works on architectures that support -// unaligned read/writes. -func fastXORBytes(dst, a, b []byte) int { - n := len(a) - if len(b) < n { - n = len(b) - } - w := n / wordSize - if w > 0 { - dw := *(*[]uintptr)(unsafe.Pointer(&dst)) - aw := *(*[]uintptr)(unsafe.Pointer(&a)) - bw := *(*[]uintptr)(unsafe.Pointer(&b)) - for i := 0; i < w; i++ { - dw[i] = aw[i] ^ bw[i] - } - } - for i := n - n%wordSize; i < n; i++ { - dst[i] = a[i] ^ b[i] - } - return n -} - -// safeXORBytes xors one by one. It works on all architectures, independent if -// it supports unaligned read/writes or not. -func safeXORBytes(dst, a, b []byte) int { - n := len(a) - if len(b) < n { - n = len(b) - } - for i := 0; i < n; i++ { - dst[i] = a[i] ^ b[i] - } - return n + return subtle.XORBytes(dst, a, b) } // ANDBytes ands the bytes in a and b. The destination is assumed to have enough diff --git a/common/bitutil/bitutil_test.go b/common/bitutil/bitutil_test.go index 12f3fe24a6..1748029794 100644 --- a/common/bitutil/bitutil_test.go +++ b/common/bitutil/bitutil_test.go @@ -29,7 +29,7 @@ func TestXOR(t *testing.T) { d2 := make([]byte, 1023+alignD)[alignD:] XORBytes(d1, p, q) - safeXORBytes(d2, p, q) + naiveXOR(d2, p, q) if !bytes.Equal(d1, d2) { t.Error("not equal", d1, d2) } @@ -38,6 +38,18 @@ func TestXOR(t *testing.T) { } } +// naiveXOR xors bytes one by one. +func naiveXOR(dst, a, b []byte) int { + n := len(a) + if len(b) < n { + n = len(b) + } + for i := 0; i < n; i++ { + dst[i] = a[i] ^ b[i] + } + return n +} + // Tests that bitwise AND works for various alignments. func TestAND(t *testing.T) { for alignP := 0; alignP < 2; alignP++ { @@ -134,7 +146,7 @@ func benchmarkBaseXOR(b *testing.B, size int) { p, q := make([]byte, size), make([]byte, size) for i := 0; i < b.N; i++ { - safeXORBytes(p, p, q) + naiveXOR(p, p, q) } } diff --git a/common/size.go b/common/size.go index 097b6304a8..e7f504a082 100644 --- a/common/size.go +++ b/common/size.go @@ -26,13 +26,13 @@ type StorageSize float64 // String implements the stringer interface. func (s StorageSize) String() string { - if s > 1099511627776 { + if s >= 1099511627776 { return fmt.Sprintf("%.2f TiB", s/1099511627776) - } else if s > 1073741824 { + } else if s >= 1073741824 { return fmt.Sprintf("%.2f GiB", s/1073741824) - } else if s > 1048576 { + } else if s >= 1048576 { return fmt.Sprintf("%.2f MiB", s/1048576) - } else if s > 1024 { + } else if s >= 1024 { return fmt.Sprintf("%.2f KiB", s/1024) } else { return fmt.Sprintf("%.2f B", s) @@ -42,13 +42,13 @@ func (s StorageSize) String() string { // TerminalString implements log.TerminalStringer, formatting a string for console // output during logging. func (s StorageSize) TerminalString() string { - if s > 1099511627776 { + if s >= 1099511627776 { return fmt.Sprintf("%.2fTiB", s/1099511627776) - } else if s > 1073741824 { + } else if s >= 1073741824 { return fmt.Sprintf("%.2fGiB", s/1073741824) - } else if s > 1048576 { + } else if s >= 1048576 { return fmt.Sprintf("%.2fMiB", s/1048576) - } else if s > 1024 { + } else if s >= 1024 { return fmt.Sprintf("%.2fKiB", s/1024) } else { return fmt.Sprintf("%.2fB", s) diff --git a/common/types.go b/common/types.go index ebf5ca0bf6..4dedbf592a 100644 --- a/common/types.go +++ b/common/types.go @@ -71,6 +71,15 @@ func BigToHash(b *big.Int) Hash { return BytesToHash(b.Bytes()) } // If b is larger than len(h), b will be cropped from the left. func HexToHash(s string) Hash { return BytesToHash(FromHex(s)) } +// IsHexHash verifies whether a string can represent a valid hex-encoded +// Ethereum hash or not. +func IsHexHash(s string) bool { + if has0xPrefix(s) { + s = s[2:] + } + return len(s) == 2*HashLength && isHex(s) +} + // Cmp compares two hashes. func (h Hash) Cmp(other Hash) int { return bytes.Compare(h[:], other[:]) diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index be46ebe906..819a2100ac 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -17,6 +17,7 @@ package beacon import ( + "context" "errors" "fmt" "math/big" @@ -29,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/internal/telemetry" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" "github.com/holiman/uint256" @@ -259,11 +261,11 @@ func (beacon *Beacon) verifyHeader(chain consensus.ChainHeaderReader, header, pa if !cancun { switch { case header.ExcessBlobGas != nil: - return fmt.Errorf("invalid excessBlobGas: have %d, expected nil", header.ExcessBlobGas) + return fmt.Errorf("invalid excessBlobGas: have %d, expected nil", *header.ExcessBlobGas) case header.BlobGasUsed != nil: - return fmt.Errorf("invalid blobGasUsed: have %d, expected nil", header.BlobGasUsed) + return fmt.Errorf("invalid blobGasUsed: have %d, expected nil", *header.BlobGasUsed) case header.ParentBeaconRoot != nil: - return fmt.Errorf("invalid parentBeaconRoot, have %#x, expected nil", header.ParentBeaconRoot) + return fmt.Errorf("invalid parentBeaconRoot, have %#x, expected nil", *header.ParentBeaconRoot) } } else { if header.ParentBeaconRoot == nil { @@ -273,6 +275,14 @@ func (beacon *Beacon) verifyHeader(chain consensus.ChainHeaderReader, header, pa return err } } + + amsterdam := chain.Config().IsAmsterdam(header.Number, header.Time) + if amsterdam && header.SlotNumber == nil { + return errors.New("header is missing slotNumber") + } + if !amsterdam && header.SlotNumber != nil { + return fmt.Errorf("invalid slotNumber: have %d, expected nil", *header.SlotNumber) + } return nil } @@ -344,9 +354,17 @@ func (beacon *Beacon) Finalize(chain consensus.ChainHeaderReader, header *types. // FinalizeAndAssemble implements consensus.Engine, setting the final state and // assembling the block. -func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, body *types.Body, receipts []*types.Receipt) (*types.Block, error) { +func (beacon *Beacon) FinalizeAndAssemble(ctx context.Context, chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, body *types.Body, receipts []*types.Receipt) (result *types.Block, err error) { + ctx, _, spanEnd := telemetry.StartSpan(ctx, "consensus.beacon.FinalizeAndAssemble", + telemetry.Int64Attribute("block.number", int64(header.Number.Uint64())), + telemetry.Int64Attribute("txs.count", int64(len(body.Transactions))), + telemetry.Int64Attribute("withdrawals.count", int64(len(body.Withdrawals))), + ) + defer spanEnd(&err) + if !beacon.IsPoSHeader(header) { - return beacon.ethone.FinalizeAndAssemble(chain, header, state, body, receipts) + block, delegateErr := beacon.ethone.FinalizeAndAssemble(ctx, chain, header, state, body, receipts) + return block, delegateErr } shanghai := chain.Config().IsShanghai(header.Number, header.Time, types.DeserializeHeaderExtraInformation(header).ArbOSFormatVersion) if shanghai { @@ -360,51 +378,19 @@ func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, hea } } // Finalize and assemble the block. + _, _, finalizeSpanEnd := telemetry.StartSpan(ctx, "consensus.beacon.Finalize") beacon.Finalize(chain, header, state, body) + finalizeSpanEnd(nil) // Assign the final state root to header. + _, _, rootSpanEnd := telemetry.StartSpan(ctx, "consensus.beacon.IntermediateRoot") header.Root = state.IntermediateRoot(true) + rootSpanEnd(nil) // Assemble the final block. + _, _, blockSpanEnd := telemetry.StartSpan(ctx, "consensus.beacon.NewBlock") block := types.NewBlock(header, body, receipts, trie.NewStackTrie(nil)) - - // Create the block witness and attach to block. - // This step needs to happen as late as possible to catch all access events. - if chain.Config().IsVerkle(header.Number, header.Time) { - keys := state.AccessEvents().Keys() - - // Open the pre-tree to prove the pre-state against - parent := chain.GetHeaderByNumber(header.Number.Uint64() - 1) - if parent == nil { - return nil, fmt.Errorf("nil parent header for block %d", header.Number) - } - preTrie, err := state.Database().OpenTrie(parent.Root) - if err != nil { - return nil, fmt.Errorf("error opening pre-state tree root: %w", err) - } - postTrie := state.GetTrie() - if postTrie == nil { - return nil, errors.New("post-state tree is not available") - } - vktPreTrie, okpre := preTrie.(*trie.VerkleTrie) - vktPostTrie, okpost := postTrie.(*trie.VerkleTrie) - - // The witness is only attached iff both parent and current block are - // using verkle tree. - if okpre && okpost { - if len(keys) > 0 { - verkleProof, stateDiff, err := vktPreTrie.Proof(vktPostTrie, keys) - if err != nil { - return nil, fmt.Errorf("error generating verkle proof for block %d: %w", header.Number, err) - } - block = block.WithWitness(&types.ExecutionWitness{ - StateDiff: stateDiff, - VerkleProof: verkleProof, - }) - } - } - } - + blockSpanEnd(nil) return block, nil } diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index 59d027225f..878594519f 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -19,6 +19,7 @@ package clique import ( "bytes" + "context" "errors" "fmt" "io" @@ -306,11 +307,13 @@ func (c *Clique) verifyHeader(chain consensus.ChainHeaderReader, header *types.H // Verify the non-existence of cancun-specific header fields switch { case header.ExcessBlobGas != nil: - return fmt.Errorf("invalid excessBlobGas: have %d, expected nil", header.ExcessBlobGas) + return fmt.Errorf("invalid excessBlobGas: have %d, expected nil", *header.ExcessBlobGas) case header.BlobGasUsed != nil: - return fmt.Errorf("invalid blobGasUsed: have %d, expected nil", header.BlobGasUsed) + return fmt.Errorf("invalid blobGasUsed: have %d, expected nil", *header.BlobGasUsed) case header.ParentBeaconRoot != nil: - return fmt.Errorf("invalid parentBeaconRoot, have %#x, expected nil", header.ParentBeaconRoot) + return fmt.Errorf("invalid parentBeaconRoot, have %#x, expected nil", *header.ParentBeaconRoot) + case header.SlotNumber != nil: + return fmt.Errorf("invalid slotNumber, have %#x, expected nil", *header.SlotNumber) } // All basic checks passed, verify cascading fields return c.verifyCascadingFields(chain, header, parents) @@ -580,7 +583,7 @@ func (c *Clique) Finalize(chain consensus.ChainHeaderReader, header *types.Heade // FinalizeAndAssemble implements consensus.Engine, ensuring no uncles are set, // nor block rewards given, and returns the final block. -func (c *Clique) FinalizeAndAssemble(chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, body *types.Body, receipts []*types.Receipt) (*types.Block, error) { +func (c *Clique) FinalizeAndAssemble(ctx context.Context, chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, body *types.Body, receipts []*types.Receipt) (*types.Block, error) { if len(body.Withdrawals) > 0 { return nil, errors.New("clique does not support withdrawals") } @@ -695,6 +698,9 @@ func encodeSigHeader(w io.Writer, header *types.Header) { if header.ParentBeaconRoot != nil { panic("unexpected parent beacon root value in clique") } + if header.SlotNumber != nil { + panic("unexpected slot number value in clique") + } if err := rlp.Encode(w, enc); err != nil { panic("can't encode: " + err.Error()) } diff --git a/consensus/consensus.go b/consensus/consensus.go index a68351f7ff..094026b614 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -18,6 +18,7 @@ package consensus import ( + "context" "math/big" "github.com/ethereum/go-ethereum/common" @@ -92,7 +93,7 @@ type Engine interface { // // Note: The block header and state database might be updated to reflect any // consensus rules that happen at finalization (e.g. block rewards). - FinalizeAndAssemble(chain ChainHeaderReader, header *types.Header, state *state.StateDB, body *types.Body, receipts []*types.Receipt) (*types.Block, error) + FinalizeAndAssemble(ctx context.Context, chain ChainHeaderReader, header *types.Header, state *state.StateDB, body *types.Body, receipts []*types.Receipt) (*types.Block, error) // Seal generates a new sealing request for the given input block and pushes // the result into the given channel. diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index ecaaa8ef9f..7b27bd1d04 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -17,6 +17,7 @@ package ethash import ( + "context" "errors" "fmt" "math/big" @@ -280,11 +281,13 @@ func (ethash *Ethash) verifyHeader(chain consensus.ChainHeaderReader, header, pa // Verify the non-existence of cancun-specific header fields switch { case header.ExcessBlobGas != nil: - return fmt.Errorf("invalid excessBlobGas: have %d, expected nil", header.ExcessBlobGas) + return fmt.Errorf("invalid excessBlobGas: have %d, expected nil", *header.ExcessBlobGas) case header.BlobGasUsed != nil: - return fmt.Errorf("invalid blobGasUsed: have %d, expected nil", header.BlobGasUsed) + return fmt.Errorf("invalid blobGasUsed: have %d, expected nil", *header.BlobGasUsed) case header.ParentBeaconRoot != nil: - return fmt.Errorf("invalid parentBeaconRoot, have %#x, expected nil", header.ParentBeaconRoot) + return fmt.Errorf("invalid parentBeaconRoot, have %#x, expected nil", *header.ParentBeaconRoot) + case header.SlotNumber != nil: + return fmt.Errorf("invalid slotNumber, have %#x, expected nil", *header.SlotNumber) } // Add some fake checks for tests if ethash.fakeDelay != nil { @@ -513,7 +516,7 @@ func (ethash *Ethash) Finalize(chain consensus.ChainHeaderReader, header *types. // FinalizeAndAssemble implements consensus.Engine, accumulating the block and // uncle rewards, setting the final state and assembling the block. -func (ethash *Ethash) FinalizeAndAssemble(chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, body *types.Body, receipts []*types.Receipt) (*types.Block, error) { +func (ethash *Ethash) FinalizeAndAssemble(ctx context.Context, chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, body *types.Body, receipts []*types.Receipt) (*types.Block, error) { if len(body.Withdrawals) > 0 { return nil, errors.New("ethash does not support withdrawals") } @@ -561,6 +564,9 @@ func (ethash *Ethash) SealHash(header *types.Header) (hash common.Hash) { if header.ParentBeaconRoot != nil { panic("parent beacon root set on ethash") } + if header.SlotNumber != nil { + panic("slot number set on ethash") + } rlp.Encode(hasher, enc) hasher.Sum(hash[:0]) return hash diff --git a/consensus/misc/eip1559/eip1559.go b/consensus/misc/eip1559/eip1559.go index a90bd744b2..9a4cd320a8 100644 --- a/consensus/misc/eip1559/eip1559.go +++ b/consensus/misc/eip1559/eip1559.go @@ -43,6 +43,10 @@ func VerifyEIP1559Header(config *params.ChainConfig, parent, header *types.Heade if header.BaseFee == nil { return errors.New("header is missing baseFee") } + // Verify the parent header is not malformed + if config.IsLondon(parent.Number) && parent.BaseFee == nil { + return errors.New("parent header is missing baseFee") + } // Verify the baseFee is correct based on the parent header. expectedBaseFee := CalcBaseFee(config, parent) if header.BaseFee.Cmp(expectedBaseFee) != 0 { diff --git a/consensus/misc/eip4844/eip4844.go b/consensus/misc/eip4844/eip4844.go index 37af914212..ccc8321d47 100644 --- a/consensus/misc/eip4844/eip4844.go +++ b/consensus/misc/eip4844/eip4844.go @@ -53,9 +53,9 @@ func (bc *BlobConfig) blobPrice(excessBlobGas uint64) *big.Int { return new(big.Int).Mul(f, big.NewInt(params.BlobTxBlobGasPerBlob)) } -func latestBlobConfig(cfg *params.ChainConfig, time uint64) *BlobConfig { +func latestBlobConfig(cfg *params.ChainConfig, time uint64) (BlobConfig, error) { if cfg.BlobScheduleConfig == nil { - return nil + return BlobConfig{}, errors.New("no blob config") } var ( london = cfg.LondonBlock @@ -83,14 +83,14 @@ func latestBlobConfig(cfg *params.ChainConfig, time uint64) *BlobConfig { case cfg.IsCancun(london, time, 0) && s.Cancun != nil: bc = s.Cancun default: - return nil + return BlobConfig{}, errors.New("no blob config") } - return &BlobConfig{ + return BlobConfig{ Target: bc.Target, Max: bc.Max, UpdateFraction: bc.UpdateFraction, - } + }, nil } // VerifyEIP4844Header verifies the presence of the excessBlobGas field and that @@ -101,8 +101,8 @@ func VerifyEIP4844Header(config *params.ChainConfig, parent, header *types.Heade panic("bad header pair") } - bcfg := latestBlobConfig(config, header.Time) - if bcfg == nil { + bcfg, err := latestBlobConfig(config, header.Time) + if err != nil { panic("called before EIP-4844 is active") } @@ -118,7 +118,7 @@ func VerifyEIP4844Header(config *params.ChainConfig, parent, header *types.Heade return fmt.Errorf("blob gas used %d exceeds maximum allowance %d", *header.BlobGasUsed, bcfg.maxBlobGas()) } if *header.BlobGasUsed%params.BlobTxBlobGasPerBlob != 0 { - return fmt.Errorf("blob gas used %d not a multiple of blob gas per blob %d", header.BlobGasUsed, params.BlobTxBlobGasPerBlob) + return fmt.Errorf("blob gas used %d not a multiple of blob gas per blob %d", *header.BlobGasUsed, params.BlobTxBlobGasPerBlob) } // Verify the excessBlobGas is correct based on the parent header @@ -134,14 +134,14 @@ func VerifyEIP4844Header(config *params.ChainConfig, parent, header *types.Heade func CalcExcessBlobGas(config *params.ChainConfig, parent *types.Header, headTimestamp uint64) uint64 { // we can use 0 here because arbitrum doesn't support Blob transactions. isOsaka := config.IsOsaka(config.LondonBlock, headTimestamp, 0) - bcfg := latestBlobConfig(config, headTimestamp) - if bcfg == nil { - return 0 + bcfg, err := latestBlobConfig(config, headTimestamp) + if err != nil { + panic("calculating excess blob gas on nil blob config") } return calcExcessBlobGas(isOsaka, bcfg, parent) } -func calcExcessBlobGas(isOsaka bool, bcfg *BlobConfig, parent *types.Header) uint64 { +func calcExcessBlobGas(isOsaka bool, bcfg BlobConfig, parent *types.Header) uint64 { var parentExcessBlobGas, parentBlobGasUsed uint64 if parent.ExcessBlobGas != nil { parentExcessBlobGas = *parent.ExcessBlobGas @@ -180,8 +180,8 @@ func CalcBlobFee(config *params.ChainConfig, header *types.Header) *big.Int { // Arbitrum does not support blob transactions, so we return 0. return big.NewInt(0) } - blobConfig := latestBlobConfig(config, header.Time) - if blobConfig == nil { + blobConfig, err := latestBlobConfig(config, header.Time) + if err != nil { panic("calculating blob fee on unsupported fork") } return blobConfig.blobBaseFee(*header.ExcessBlobGas) @@ -189,8 +189,8 @@ func CalcBlobFee(config *params.ChainConfig, header *types.Header) *big.Int { // MaxBlobsPerBlock returns the max blobs per block for a block at the given timestamp. func MaxBlobsPerBlock(cfg *params.ChainConfig, time uint64) int { - blobConfig := latestBlobConfig(cfg, time) - if blobConfig == nil { + blobConfig, err := latestBlobConfig(cfg, time) + if err != nil { return 0 } return blobConfig.Max @@ -204,8 +204,8 @@ func MaxBlobGasPerBlock(cfg *params.ChainConfig, time uint64) uint64 { // LatestMaxBlobsPerBlock returns the latest max blobs per block defined by the // configuration, regardless of the currently active fork. func LatestMaxBlobsPerBlock(cfg *params.ChainConfig) int { - bcfg := latestBlobConfig(cfg, math.MaxUint64) - if bcfg == nil { + bcfg, err := latestBlobConfig(cfg, math.MaxUint64) + if err != nil { return 0 } return bcfg.Max @@ -226,6 +226,15 @@ func CalcBlobFeeWithConfig(bc *params.BlobConfig, excessBlobGas *uint64) *big.In return gethBc.blobBaseFee(*excessBlobGas) } +// TargetBlobsPerBlock returns the target blobs per block for a block at the given timestamp. +func TargetBlobsPerBlock(cfg *params.ChainConfig, time uint64) int { + blobConfig, err := latestBlobConfig(cfg, time) + if err != nil { + return 0 + } + return blobConfig.Target +} + // fakeExponential approximates factor * e ** (numerator / denominator) using // Taylor expansion. func fakeExponential(factor, numerator, denominator *big.Int) *big.Int { diff --git a/console/console.go b/console/console.go index b5c77bd78f..573d1da570 100644 --- a/console/console.go +++ b/console/console.go @@ -282,7 +282,7 @@ func (c *Console) AutoCompleteInput(line string, pos int) (string, []string, str for ; start > 0; start-- { // Skip all methods and namespaces (i.e. including the dot) c := line[start] - if c == '.' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '1' && c <= '9') { + if c == '.' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') { continue } // We've hit an unexpected character, autocomplete form here diff --git a/console/prompt/prompter.go b/console/prompt/prompter.go index 2a20b6906a..5a0a89e76a 100644 --- a/console/prompt/prompter.go +++ b/console/prompt/prompter.go @@ -142,7 +142,7 @@ func (p *terminalPrompter) PromptPassword(prompt string) (passwd string, err err // PromptConfirm displays the given prompt to the user and requests a boolean // choice to be made, returning that choice. func (p *terminalPrompter) PromptConfirm(prompt string) (bool, error) { - input, err := p.Prompt(prompt + " [y/n] ") + input, err := p.PromptInput(prompt + " [y/n] ") if len(input) > 0 && strings.EqualFold(input[:1], "y") { return true, nil } diff --git a/core/bintrie_witness_test.go b/core/bintrie_witness_test.go new file mode 100644 index 0000000000..0c8a038776 --- /dev/null +++ b/core/bintrie_witness_test.go @@ -0,0 +1,237 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package core + +import ( + "encoding/binary" + "fmt" + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/beacon" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/triedb" +) + +var ( + testVerkleChainConfig = ¶ms.ChainConfig{ + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), + Ethash: new(params.EthashConfig), + ShanghaiTime: u64(0), + VerkleTime: u64(0), + TerminalTotalDifficulty: common.Big0, + EnableVerkleAtGenesis: true, + BlobScheduleConfig: ¶ms.BlobScheduleConfig{ + Verkle: params.DefaultPragueBlobConfig, + }, + } +) + +func TestProcessVerkle(t *testing.T) { + var ( + code = common.FromHex(`6060604052600a8060106000396000f360606040526008565b00`) + intrinsicContractCreationGas, _ = IntrinsicGas(code, nil, nil, true, true, true, true) + // A contract creation that calls EXTCODECOPY in the constructor. Used to ensure that the witness + // will not contain that copied data. + // Source: https://gist.github.com/gballet/a23db1e1cb4ed105616b5920feb75985 + codeWithExtCodeCopy = common.FromHex(`0x60806040526040516100109061017b565b604051809103906000f08015801561002c573d6000803e3d6000fd5b506000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff16021790555034801561007857600080fd5b5060008067ffffffffffffffff8111156100955761009461024a565b5b6040519080825280601f01601f1916602001820160405280156100c75781602001600182028036833780820191505090505b50905060008060009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1690506020600083833c81610101906101e3565b60405161010d90610187565b61011791906101a3565b604051809103906000f080158015610133573d6000803e3d6000fd5b50600160006101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550505061029b565b60d58061046783390190565b6102068061053c83390190565b61019d816101d9565b82525050565b60006020820190506101b86000830184610194565b92915050565b6000819050602082019050919050565b600081519050919050565b6000819050919050565b60006101ee826101ce565b826101f8846101be565b905061020381610279565b925060208210156102435761023e7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8360200360080261028e565b831692505b5050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b600061028582516101d9565b80915050919050565b600082821b905092915050565b6101bd806102aa6000396000f3fe608060405234801561001057600080fd5b506004361061002b5760003560e01c8063f566852414610030575b600080fd5b61003861004e565b6040516100459190610146565b60405180910390f35b6000600160009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff166381ca91d36040518163ffffffff1660e01b815260040160206040518083038186803b1580156100b857600080fd5b505afa1580156100cc573d6000803e3d6000fd5b505050506040513d601f19601f820116820180604052508101906100f0919061010a565b905090565b60008151905061010481610170565b92915050565b6000602082840312156101205761011f61016b565b5b600061012e848285016100f5565b91505092915050565b61014081610161565b82525050565b600060208201905061015b6000830184610137565b92915050565b6000819050919050565b600080fd5b61017981610161565b811461018457600080fd5b5056fea2646970667358221220a6a0e11af79f176f9c421b7b12f441356b25f6489b83d38cc828a701720b41f164736f6c63430008070033608060405234801561001057600080fd5b5060b68061001f6000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c8063ab5ed15014602d575b600080fd5b60336047565b604051603e9190605d565b60405180910390f35b60006001905090565b6057816076565b82525050565b6000602082019050607060008301846050565b92915050565b600081905091905056fea26469706673582212203a14eb0d5cd07c277d3e24912f110ddda3e553245a99afc4eeefb2fbae5327aa64736f6c63430008070033608060405234801561001057600080fd5b5060405161020638038061020683398181016040528101906100329190610063565b60018160001c6100429190610090565b60008190555050610145565b60008151905061005d8161012e565b92915050565b60006020828403121561007957610078610129565b5b60006100878482850161004e565b91505092915050565b600061009b826100f0565b91506100a6836100f0565b9250827fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff038211156100db576100da6100fa565b5b828201905092915050565b6000819050919050565b6000819050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fd5b600080fd5b610137816100e6565b811461014257600080fd5b50565b60b3806101536000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c806381ca91d314602d575b600080fd5b60336047565b604051603e9190605a565b60405180910390f35b60005481565b6054816073565b82525050565b6000602082019050606d6000830184604d565b92915050565b600081905091905056fea26469706673582212209bff7098a2f526de1ad499866f27d6d0d6f17b74a413036d6063ca6a0998ca4264736f6c63430008070033`) + intrinsicCodeWithExtCodeCopyGas, _ = IntrinsicGas(codeWithExtCodeCopy, nil, nil, true, true, true, true) + signer = types.LatestSigner(testVerkleChainConfig) + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + bcdb = rawdb.NewMemoryDatabase() // Database for the blockchain + coinbase = common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7") + gspec = &Genesis{ + Config: testVerkleChainConfig, + Alloc: GenesisAlloc{ + coinbase: { + Balance: big.NewInt(1000000000000000000), // 1 ether + Nonce: 0, + }, + params.BeaconRootsAddress: {Nonce: 1, Code: params.BeaconRootsCode, Balance: common.Big0}, + params.HistoryStorageAddress: {Nonce: 1, Code: params.HistoryStorageCode, Balance: common.Big0}, + params.WithdrawalQueueAddress: {Nonce: 1, Code: params.WithdrawalQueueCode, Balance: common.Big0}, + params.ConsolidationQueueAddress: {Nonce: 1, Code: params.ConsolidationQueueCode, Balance: common.Big0}, + }, + } + ) + // Verkle trees use the snapshot, which must be enabled before the + // data is saved into the tree+database. + // genesis := gspec.MustCommit(bcdb, triedb) + options := DefaultConfig().WithStateScheme(rawdb.PathScheme) + options.SnapshotLimit = 0 + blockchain, _ := NewBlockChain(bcdb, nil, gspec, beacon.New(ethash.NewFaker()), options) + defer blockchain.Stop() + + txCost1 := params.TxGas + txCost2 := params.TxGas + contractCreationCost := intrinsicContractCreationGas + + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + params.WitnessBranchReadCost + params.WitnessBranchWriteCost + /* creation */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* creation with value */ + 739 /* execution costs */ + codeWithExtCodeCopyGas := intrinsicCodeWithExtCodeCopyGas + + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + params.WitnessBranchReadCost + params.WitnessBranchWriteCost + /* creation (tx) */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + params.WitnessBranchReadCost + params.WitnessBranchWriteCost + /* creation (CREATE at pc=0x20) */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* write code hash */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #0 */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #1 */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #2 */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #3 */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #4 */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #5 */ + params.WitnessChunkReadCost + /* SLOAD in constructor */ + params.WitnessChunkWriteCost + /* SSTORE in constructor */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + params.WitnessBranchReadCost + params.WitnessBranchWriteCost + /* creation (CREATE at PC=0x121) */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* write code hash */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #0 */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #1 */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #2 */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #3 */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #4 */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #5 */ + params.WitnessChunkReadCost + /* SLOAD in constructor */ + params.WitnessChunkWriteCost + /* SSTORE in constructor */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* write code hash for tx creation */ + 15*(params.WitnessChunkReadCost+params.WitnessChunkWriteCost) + /* code chunks #0..#14 */ + uint64(4844) /* execution costs */ + blockGasUsagesExpected := []uint64{ + txCost1*2 + txCost2, + txCost1*2 + txCost2 + contractCreationCost + codeWithExtCodeCopyGas, + } + _, chain, _ := GenerateChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { + gen.SetPoS() + + // TODO need to check that the tx cost provided is the exact amount used (no remaining left-over) + tx, _ := types.SignTx(types.NewTransaction(uint64(i)*3, common.Address{byte(i), 2, 3}, big.NewInt(999), txCost1, big.NewInt(875000000), nil), signer, testKey) + gen.AddTx(tx) + tx, _ = types.SignTx(types.NewTransaction(uint64(i)*3+1, common.Address{}, big.NewInt(999), txCost1, big.NewInt(875000000), nil), signer, testKey) + gen.AddTx(tx) + tx, _ = types.SignTx(types.NewTransaction(uint64(i)*3+2, common.Address{}, big.NewInt(0), txCost2, big.NewInt(875000000), nil), signer, testKey) + gen.AddTx(tx) + + // Add two contract creations in block #2 + if i == 1 { + tx, _ = types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 6, + Value: big.NewInt(16), + Gas: 3000000, + GasPrice: big.NewInt(875000000), + Data: code, + }) + gen.AddTx(tx) + + tx, _ = types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 7, + Value: big.NewInt(0), + Gas: 3000000, + GasPrice: big.NewInt(875000000), + Data: codeWithExtCodeCopy, + }) + gen.AddTx(tx) + } + }) + + for i, b := range chain { + fmt.Printf("%d %x\n", i, b.Root()) + } + endnum, err := blockchain.InsertChain(chain) + if err != nil { + t.Fatalf("block %d imported with error: %v", endnum, err) + } + + for i := range 2 { + b := blockchain.GetBlockByNumber(uint64(i) + 1) + if b == nil { + t.Fatalf("expected block %d to be present in chain", i+1) + } + if b.Hash() != chain[i].Hash() { + t.Fatalf("block #%d not found at expected height", b.NumberU64()) + } + if b.GasUsed() != blockGasUsagesExpected[i] { + t.Fatalf("expected block #%d txs to use %d, got %d\n", b.NumberU64(), blockGasUsagesExpected[i], b.GasUsed()) + } + } +} + +func TestProcessParentBlockHash(t *testing.T) { + // This test uses blocks where, + // block 1 parent hash is 0x0100.... + // block 2 parent hash is 0x0200.... + // etc + checkBlockHashes := func(statedb *state.StateDB, isVerkle bool) { + statedb.SetNonce(params.HistoryStorageAddress, 1, tracing.NonceChangeUnspecified) + statedb.SetCode(params.HistoryStorageAddress, params.HistoryStorageCode, tracing.CodeChangeUnspecified) + // Process n blocks, from 1 .. num + var num = 2 + for i := 1; i <= num; i++ { + header := &types.Header{ParentHash: common.Hash{byte(i)}, Number: big.NewInt(int64(i)), Difficulty: new(big.Int)} + chainConfig := params.MergedTestChainConfig + if isVerkle { + chainConfig = testVerkleChainConfig + } + vmContext := NewEVMBlockContext(header, nil, new(common.Address)) + evm := vm.NewEVM(vmContext, statedb, chainConfig, vm.Config{}) + ProcessParentBlockHash(header.ParentHash, evm) + } + // Read block hashes for block 0 .. num-1 + for i := 0; i < num; i++ { + have, want := getContractStoredBlockHash(statedb, uint64(i), isVerkle), common.Hash{byte(i + 1)} + if have != want { + t.Errorf("block %d, verkle=%v, have parent hash %v, want %v", i, isVerkle, have, want) + } + } + } + t.Run("MPT", func(t *testing.T) { + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) + checkBlockHashes(statedb, false) + }) + t.Run("Verkle", func(t *testing.T) { + db := rawdb.NewMemoryDatabase() + cacheConfig := DefaultConfig().WithStateScheme(rawdb.PathScheme) + cacheConfig.SnapshotLimit = 0 + triedb := triedb.NewDatabase(db, cacheConfig.triedbConfig(true)) + statedb, _ := state.New(types.EmptyVerkleHash, state.NewDatabase(triedb, nil)) + checkBlockHashes(statedb, true) + }) +} + +// getContractStoredBlockHash is a utility method which reads the stored parent blockhash for block 'number' +func getContractStoredBlockHash(statedb *state.StateDB, number uint64, isVerkle bool) common.Hash { + ringIndex := number % params.HistoryServeWindow + var key common.Hash + binary.BigEndian.PutUint64(key[24:], ringIndex) + if isVerkle { + return statedb.GetState(params.HistoryStorageAddress, key) + } + return statedb.GetState(params.HistoryStorageAddress, key) +} diff --git a/core/block_validator_test.go b/core/block_validator_test.go index ce026d56f7..f40d503845 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -17,6 +17,7 @@ package core import ( + "context" "math/big" "testing" "time" @@ -210,7 +211,7 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { t.Fatalf("post-block %d: unexpected result returned: %v", i, result) case <-time.After(25 * time.Millisecond): } - chain.InsertBlockWithoutSetHead(postBlocks[i], false) + chain.InsertBlockWithoutSetHead(context.Background(), postBlocks[i], false) } // Verify the blocks with pre-merge blocks and post-merge blocks diff --git a/core/blockchain.go b/core/blockchain.go index a5439b82f6..6adb232ec0 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -18,6 +18,7 @@ package core import ( + "context" "errors" "fmt" "io" @@ -48,6 +49,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/syncx" + "github.com/ethereum/go-ethereum/internal/telemetry" "github.com/ethereum/go-ethereum/internal/version" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" @@ -76,6 +78,8 @@ var ( storageReadTimer = metrics.NewRegisteredResettingTimer("chain/storage/reads", nil) storageUpdateTimer = metrics.NewRegisteredResettingTimer("chain/storage/updates", nil) storageCommitTimer = metrics.NewRegisteredResettingTimer("chain/storage/commits", nil) + codeReadTimer = metrics.NewRegisteredResettingTimer("chain/code/reads", nil) + codeReadBytesTimer = metrics.NewRegisteredResettingTimer("chain/code/readbytes", nil) accountCacheHitMeter = metrics.NewRegisteredMeter("chain/account/reads/cache/process/hit", nil) accountCacheMissMeter = metrics.NewRegisteredMeter("chain/account/reads/cache/process/miss", nil) @@ -89,9 +93,8 @@ var ( accountReadSingleTimer = metrics.NewRegisteredResettingTimer("chain/account/single/reads", nil) storageReadSingleTimer = metrics.NewRegisteredResettingTimer("chain/storage/single/reads", nil) - - snapshotCommitTimer = metrics.NewRegisteredResettingTimer("chain/snapshot/commits", nil) - triedbCommitTimer = metrics.NewRegisteredResettingTimer("chain/triedb/commits", nil) + codeReadSingleTimer = metrics.NewRegisteredResettingTimer("chain/code/single/reads", nil) + triedbCommitTimer = metrics.NewRegisteredResettingTimer("chain/triedb/commits", nil) triedbSizeGauge = metrics.NewRegisteredGauge("chain/triedb/size", nil) triedbGCProcGauge = metrics.NewRegisteredGauge("chain/triedb/gcproc", nil) @@ -181,6 +184,17 @@ type BlockChainConfig struct { // If set to 0, all state histories across the entire chain will be retained; StateHistory uint64 + // Number of blocks from the chain head for which trienode histories are retained. + // If set to 0, all trienode histories across the entire chain will be retained; + // If set to -1, no trienode history will be retained; + TrienodeHistory int64 + + // The frequency of full-value encoding. For example, a value of 16 means + // that, on average, for a given trie node across its 16 consecutive historical + // versions, only one version is stored in full format, while the others + // are stored in diff mode for storage compression. + NodeFullValueCheckpoint uint32 + // State snapshot related options SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory SnapshotNoBuild bool // Whether the background generation is allowed @@ -202,9 +216,8 @@ type BlockChainConfig struct { MaxNumberOfBlocksToSkipStateSaving uint32 MaxAmountOfGasToSkipStateSaving uint64 - // This defines the cutoff block for history expiry. - // Blocks before this number may be unavailable in the chain database. - ChainHistoryMode history.HistoryMode + // HistoryPolicy defines the chain history pruning intent. + HistoryPolicy history.HistoryPolicy // Misc options NoPrefetch bool // Whether to disable heuristic state prefetching when processing blocks @@ -215,13 +228,21 @@ type BlockChainConfig struct { // StateSizeTracking indicates whether the state size tracking is enabled. StateSizeTracking bool + + // SlowBlockThreshold is the block execution time threshold beyond which + // detailed statistics will be logged. Negative value means disabled (default), + // zero logs all blocks, positive value filters blocks by execution time. + SlowBlockThreshold time.Duration + + // Execution configs + StatelessSelfValidation bool // Generate execution witnesses and self-check against them (testing purpose) + EnableWitnessStats bool // Whether trie access statistics collection is enabled } // DefaultConfig returns the default config. // Note the returned object is safe to modify! func DefaultConfig() *BlockChainConfig { return &BlockChainConfig{ - // Arbitrum Config Options // note: some of the defaults are overwritten by nitro side config defaults @@ -235,13 +256,13 @@ func DefaultConfig() *BlockChainConfig { MaxAmountOfGasToSkipStateSaving: 0, MaxDiffLayers: pathdb.DefaultMaxDiffLayers, - TrieCleanLimit: 256, - TrieDirtyLimit: 256, - TrieTimeLimit: 5 * time.Minute, - StateScheme: rawdb.HashScheme, - SnapshotLimit: 256, - SnapshotWait: true, - ChainHistoryMode: history.KeepAll, + TrieCleanLimit: 256, + TrieDirtyLimit: 256, + TrieTimeLimit: 5 * time.Minute, + StateScheme: rawdb.HashScheme, + SnapshotLimit: 256, + SnapshotWait: true, + HistoryPolicy: history.HistoryPolicy{Mode: history.KeepAll}, // Transaction indexing is disabled by default. // This is appropriate for most unit tests. TxIndexer: nil, @@ -288,18 +309,28 @@ func (cfg *BlockChainConfig) triedbConfig(isVerkle bool) *triedb.Config { } if cfg.StateScheme == rawdb.PathScheme { config.PathDB = &pathdb.Config{ - StateHistory: cfg.StateHistory, - EnableStateIndexing: cfg.ArchiveMode, - TrieCleanSize: cfg.TrieCleanLimit * 1024 * 1024, - StateCleanSize: cfg.SnapshotLimit * 1024 * 1024, - JournalDirectory: cfg.TrieJournalDirectory, - MaxDiffLayers: cfg.MaxDiffLayers, - + TrieCleanSize: cfg.TrieCleanLimit * 1024 * 1024, + StateCleanSize: cfg.SnapshotLimit * 1024 * 1024, // TODO(rjl493456442): The write buffer represents the memory limit used // for flushing both trie data and state data to disk. The config name // should be updated to eliminate the confusion. - WriteBufferSize: cfg.TrieDirtyLimit * 1024 * 1024, - NoAsyncFlush: cfg.TrieNoAsyncFlush, + WriteBufferSize: cfg.TrieDirtyLimit * 1024 * 1024, + JournalDirectory: cfg.TrieJournalDirectory, + MaxDiffLayers: cfg.MaxDiffLayers, + + // Historical state configurations + StateHistory: cfg.StateHistory, + TrienodeHistory: cfg.TrienodeHistory, + EnableStateIndexing: cfg.ArchiveMode, + // Archive mode's whole purpose is to serve historical state, so + // the indexer should start its initial scan immediately rather + // than waiting for the "synced within 6h" heuristic that gates + // it on chains with non-realtime block timestamps (e.g. tests). + NoHistoryIndexDelay: cfg.ArchiveMode, + FullValueCheckpoint: cfg.NodeFullValueCheckpoint, + + // Testing configurations + NoAsyncFlush: cfg.TrieNoAsyncFlush, } } return config @@ -337,7 +368,7 @@ type BlockChain struct { lastWrite uint64 // Last block when the state was flushed flushInterval atomic.Int64 // Time interval (processing time) after which to flush a state triedb *triedb.Database // The database handler for maintaining trie nodes. - statedb *state.CachingDB // State database to reuse between imports (contains state cache) + codedb *state.CodeDB // The database handler for maintaining contract codes. txIndexer *txIndexer // Transaction indexer, might be nil if not enabled hc *HeaderChain @@ -346,6 +377,7 @@ type BlockChain struct { chainHeadFeed event.Feed logsFeed event.Feed blockProcFeed event.Feed + newPayloadFeed event.Feed // Feed for engine API newPayload events blockProcCounter int32 scope event.SubscriptionScope genesisBlock *types.Block @@ -378,7 +410,8 @@ type BlockChain struct { logger *tracing.Hooks stateSizer *state.SizeTracker // State size tracking - lastForkReadyAlert time.Time // Last time there was a fork readiness print out + lastForkReadyAlert time.Time // Last time there was a fork readiness print out + slowBlockThreshold time.Duration // Block execution time threshold beyond which detailed statistics will be logged // Arbitrum: numberOfBlocksToSkipStateSaving uint32 @@ -419,7 +452,7 @@ func NewBlockChain(db ethdb.Database, chainConfig *params.ChainConfig, genesis * // yet. The corresponding chain config will be returned, either from the // provided genesis or from the locally stored configuration if the genesis // has already been initialized. - chainConfig, genesisHash, compatErr, err = SetupGenesisBlockWithOverride(db, triedb, genesis, cfg.Overrides) + chainConfig, genesisHash, compatErr, err = SetupGenesisBlockWithOverride(db, triedb, genesis, cfg.Overrides, cfg.VmConfig.Tracer) if err != nil { return nil, err } @@ -433,19 +466,21 @@ func NewBlockChain(db ethdb.Database, chainConfig *params.ChainConfig, genesis * log.Info("") bc := &BlockChain{ - chainConfig: chainConfig, - cfg: cfg, - db: db, - triedb: triedb, - triegc: prque.New[int64, trieGcEntry](nil), - chainmu: syncx.NewClosableMutex(), - bodyCache: lru.NewCache[common.Hash, *types.Body](bodyCacheLimit), - bodyRLPCache: lru.NewCache[common.Hash, rlp.RawValue](bodyCacheLimit), - receiptsCache: lru.NewCache[common.Hash, []*types.Receipt](receiptsCacheLimit), - blockCache: lru.NewCache[common.Hash, *types.Block](blockCacheLimit), - txLookupCache: lru.NewCache[common.Hash, txLookup](txLookupCacheLimit), - engine: engine, - logger: cfg.VmConfig.Tracer, + chainConfig: chainConfig, + cfg: cfg, + db: db, + triedb: triedb, + triegc: prque.New[int64, trieGcEntry](nil), + codedb: state.NewCodeDB(db), + chainmu: syncx.NewClosableMutex(), + bodyCache: lru.NewCache[common.Hash, *types.Body](bodyCacheLimit), + bodyRLPCache: lru.NewCache[common.Hash, rlp.RawValue](bodyCacheLimit), + receiptsCache: lru.NewCache[common.Hash, []*types.Receipt](receiptsCacheLimit), + blockCache: lru.NewCache[common.Hash, *types.Block](blockCacheLimit), + txLookupCache: lru.NewCache[common.Hash, txLookup](txLookupCacheLimit), + engine: engine, + logger: cfg.VmConfig.Tracer, + slowBlockThreshold: cfg.SlowBlockThreshold, } bc.hc, err = NewHeaderChain(db, chainConfig, engine, bc.insertStopped) if err != nil { @@ -457,7 +492,6 @@ func NewBlockChain(db ethdb.Database, chainConfig *params.ChainConfig, genesis * bc.genesisBlock = bc.GetBlockByNumber(0) } bc.flushInterval.Store(int64(cfg.TrieTimeLimit)) - bc.statedb = state.NewDatabase(bc.triedb, nil) bc.validator = NewBlockValidator(chainConfig, bc) bc.prefetcher = newStatePrefetcher(chainConfig, bc.hc) bc.processor = NewStateProcessor(bc.hc) @@ -639,9 +673,6 @@ func (bc *BlockChain) setupSnapshot() { AsyncBuild: !bc.cfg.SnapshotWait, } bc.snaps, _ = snapshot.New(snapconfig, bc.db, bc.triedb, head.Root) - - // Re-initialize the state database with snapshot - bc.statedb = state.NewDatabase(bc.triedb, bc.snaps) } } @@ -759,45 +790,43 @@ func (bc *BlockChain) loadLastState() error { // initializeHistoryPruning sets bc.historyPrunePoint. func (bc *BlockChain) initializeHistoryPruning(latest uint64) error { freezerTail, _ := bc.db.Tail() + policy := bc.cfg.HistoryPolicy - switch bc.cfg.ChainHistoryMode { + switch policy.Mode { case history.KeepAll: - if freezerTail == 0 { - return nil - } - // The database was pruned somehow, so we need to figure out if it's a known - // configuration or an error. - predefinedPoint := history.PrunePoints[bc.genesisBlock.Hash()] - if predefinedPoint == nil || freezerTail != predefinedPoint.BlockNumber { - log.Error("Chain history database is pruned with unknown configuration", "tail", freezerTail) - return errors.New("unexpected database tail") + if freezerTail > 0 { + // Database was pruned externally. Record the actual state. + log.Warn("Chain history database is pruned", "tail", freezerTail, "mode", policy.Mode) + bc.historyPrunePoint.Store(&history.PrunePoint{ + BlockNumber: freezerTail, + BlockHash: bc.GetCanonicalHash(freezerTail), + }) } - bc.historyPrunePoint.Store(predefinedPoint) return nil - case history.KeepPostMerge: - if freezerTail == 0 && latest != 0 { - // This is the case where a user is trying to run with --history.chain - // postmerge directly on an existing DB. We could just trigger the pruning - // here, but it'd be a bit dangerous since they may not have intended this - // action to happen. So just tell them how to do it. - log.Error(fmt.Sprintf("Chain history mode is configured as %q, but database is not pruned.", bc.cfg.ChainHistoryMode.String())) - log.Error(fmt.Sprintf("Run 'geth prune-history' to prune pre-merge history.")) - return errors.New("history pruning requested via configuration") - } - predefinedPoint := history.PrunePoints[bc.genesisBlock.Hash()] - if predefinedPoint == nil { - log.Error("Chain history pruning is not supported for this network", "genesis", bc.genesisBlock.Hash()) - return errors.New("history pruning requested for unknown network") - } else if freezerTail > 0 && freezerTail != predefinedPoint.BlockNumber { - log.Error("Chain history database is pruned to unknown block", "tail", freezerTail) - return errors.New("unexpected database tail") - } - bc.historyPrunePoint.Store(predefinedPoint) + case history.KeepPostMerge, history.KeepPostPrague: + target := policy.Target + // Already at the target. + if freezerTail == target.BlockNumber { + bc.historyPrunePoint.Store(target) + return nil + } + // Database is pruned beyond the target. + if freezerTail > target.BlockNumber { + return fmt.Errorf("database pruned beyond requested history (tail=%d, target=%d)", freezerTail, target.BlockNumber) + } + // Database needs pruning (freezerTail < target). + if latest != 0 { + log.Error(fmt.Sprintf("Chain history mode is configured as %q, but database is not pruned to the target block.", policy.Mode.String())) + log.Error(fmt.Sprintf("Run 'geth prune-history --history.chain %s' to prune history.", policy.Mode.String())) + return errors.New("history pruning required") + } + // Fresh database (latest == 0), will sync from target point. + bc.historyPrunePoint.Store(target) return nil default: - return fmt.Errorf("invalid history mode: %d", bc.cfg.ChainHistoryMode) + return fmt.Errorf("invalid history mode: %d", policy.Mode) } } @@ -808,21 +837,7 @@ func (bc *BlockChain) SetHead(head uint64) error { if _, _, err := bc.setHeadBeyondRoot(head, 0, common.Hash{}, false, 0); err != nil { return err } - // Send chain head event to update the transaction pool - header := bc.CurrentBlock() - if block := bc.GetBlock(header.Hash(), header.Number.Uint64()); block == nil { - // In a pruned node the genesis block will not exist in the freezer. - // It should not happen that we set head to any other pruned block. - if header.Number.Uint64() > 0 { - // This should never happen. In practice, previously currentBlock - // contained the entire block whereas now only a "marker", so there - // is an ever so slight chance for a race we should handle. - log.Error("Current block not found in database", "block", header.Number, "hash", header.Hash()) - return fmt.Errorf("current block missing: #%d [%x..]", header.Number, header.Hash().Bytes()[:4]) - } - } - bc.chainHeadFeed.Send(ChainHeadEvent{Header: header}) - return nil + return bc.sendChainHeadEvent() } // SetHeadWithTimestamp rewinds the local chain to a new head that has at max @@ -833,7 +848,12 @@ func (bc *BlockChain) SetHeadWithTimestamp(timestamp uint64) error { if _, _, err := bc.setHeadBeyondRoot(0, timestamp, common.Hash{}, false, 0); err != nil { return err } - // Send chain head event to update the transaction pool + return bc.sendChainHeadEvent() +} + +// sendChainHeadEvent notifies all subscribers about the new chain head, +// checking first that the current block is actually available. +func (bc *BlockChain) sendChainHeadEvent() error { header := bc.CurrentBlock() if block := bc.GetBlock(header.Hash(), header.Number.Uint64()); block == nil { // In a pruned node the genesis block will not exist in the freezer. @@ -1046,7 +1066,8 @@ func (bc *BlockChain) rewindPathHead(head *types.Header, root common.Hash) (*typ // Recover if the target state if it's not available yet. if !bc.HasState(head.Root) { if err := bc.triedb.Recover(head.Root); err != nil { - log.Crit("Failed to rollback state", "err", err) + log.Error("Failed to rollback state, resetting to genesis", "err", err) + return bc.genesisBlock.Header(), rootNumber } } log.Info("Rewound to block with state", "number", head.Number, "hash", head.Hash()) @@ -1200,11 +1221,12 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha bc.txLookupCache.Purge() // Clear safe block, finalized block if needed - if safe := bc.CurrentSafeBlock(); safe != nil && head < safe.Number.Uint64() { + headBlock := bc.CurrentBlock() + if safe := bc.CurrentSafeBlock(); safe != nil && headBlock.Number.Uint64() < safe.Number.Uint64() { log.Warn("SetHead invalidated safe block") bc.SetSafe(nil) } - if finalized := bc.CurrentFinalBlock(); finalized != nil && head < finalized.Number.Uint64() { + if finalized := bc.CurrentFinalBlock(); finalized != nil && headBlock.Number.Uint64() < finalized.Number.Uint64() { log.Error("SetHead invalidated finalized block") bc.SetFinalized(nil) } @@ -1212,14 +1234,48 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha return blockNumber, rootFound, bc.loadLastState() } -// SnapSyncCommitHead sets the current head block to the one defined by the hash -// irrelevant what the chain contents were prior. -func (bc *BlockChain) SnapSyncCommitHead(hash common.Hash) error { +// SnapSyncStart disables the underlying databases (such as the trie DB and the +// optional state snapshot) to prevent potential concurrent mutations between +// snap sync and other chain operations. +func (bc *BlockChain) SnapSyncStart() error { + if !bc.chainmu.TryLock() { + return errChainStopped + } + defer bc.chainmu.Unlock() + + // Snap sync will directly modify the persistent state, making the entire + // trie database unusable until the state is fully synced. To prevent any + // subsequent state reads, explicitly disable the trie database and state + // syncer is responsible to address and correct any state missing. + if bc.TrieDB().Scheme() == rawdb.PathScheme { + if err := bc.TrieDB().Disable(); err != nil { + return err + } + } + // Snap sync uses the snapshot namespace to store potentially flaky data until + // sync completely heals and finishes. Pause snapshot maintenance in the mean- + // time to prevent access. + if snapshots := bc.Snapshots(); snapshots != nil { // Only nil in tests + snapshots.Disable() + } + return nil +} + +// SnapSyncComplete sets the current head block to the block identified by the +// given hash, regardless of the chain contents prior to snap sync. It is +// invoked once snap sync completes and assumes that SnapSyncStart was called +// previously. +func (bc *BlockChain) SnapSyncComplete(hash common.Hash) error { // Make sure that both the block as well at its state trie exists block := bc.GetBlockByHash(hash) if block == nil { return fmt.Errorf("non existent block [%x..]", hash[:4]) } + if !bc.chainmu.TryLock() { + return errChainStopped + } + defer bc.chainmu.Unlock() + // Reset the trie database with the fresh snap synced state. root := block.Root() if bc.triedb.Scheme() == rawdb.PathScheme { @@ -1230,19 +1286,16 @@ func (bc *BlockChain) SnapSyncCommitHead(hash common.Hash) error { if !bc.HasState(root) { return fmt.Errorf("non existent state [%x..]", root[:4]) } - // If all checks out, manually set the head block. - if !bc.chainmu.TryLock() { - return errChainStopped - } - bc.currentBlock.Store(block.Header()) - headBlockGauge.Update(int64(block.NumberU64())) - bc.chainmu.Unlock() - // Destroy any existing state snapshot and regenerate it in the background, // also resuming the normal maintenance of any previously paused snapshot. if bc.snaps != nil { bc.snaps.Rebuild(root) } + + // If all checks out, manually set the head block. + bc.currentBlock.Store(block.Header()) + headBlockGauge.Update(int64(block.NumberU64())) + log.Info("Committed new head block", "number", block.Number(), "hash", hash) return nil } @@ -1331,6 +1384,8 @@ func (bc *BlockChain) ExportN(w io.Writer, first uint64, last uint64) error { func (bc *BlockChain) writeHeadBlock(block *types.Block) { // Add the block to the canonical chain number scheme and mark as the head batch := bc.db.NewBatch() + defer batch.Close() + rawdb.WriteHeadHeaderHash(batch, block.Hash()) rawdb.WriteHeadFastBlockHash(batch, block.Hash()) rawdb.WriteCanonicalHash(batch, block.Hash(), block.NumberU64()) @@ -1721,21 +1776,49 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. // // Note all the components of block(hash->number map, header, body, receipts) // should be written atomically. BlockBatch is used for containing all components. - blockBatch := bc.db.NewBatch() - rawdb.WriteBlock(blockBatch, block) - rawdb.WriteReceipts(blockBatch, block.Hash(), block.NumberU64(), receipts) - rawdb.WritePreimages(blockBatch, statedb.Preimages()) - if err := blockBatch.Write(); err != nil { + var ( + batch = bc.db.NewBatch() + start = time.Now() + ) + defer batch.Close() + + rawdb.WriteBlock(batch, block) + rawdb.WriteReceipts(batch, block.Hash(), block.NumberU64(), receipts) + rawdb.WritePreimages(batch, statedb.Preimages()) + if err := batch.Write(); err != nil { log.Crit("Failed to write block into disk", "err", err) } - // Commit all cached state changes into underlying memory database. - root, stateUpdate, err := statedb.CommitWithUpdate(block.NumberU64(), bc.chainConfig.IsEIP158(block.Number()), bc.chainConfig.IsCancun(block.Number(), block.Time(), types.DeserializeHeaderExtraInformation(block.Header()).ArbOSFormatVersion)) - if err != nil { - return err - } - // Emit the state update to the state sizestats if it's active - if bc.stateSizer != nil { - bc.stateSizer.Notify(stateUpdate) + log.Debug("Committed block data", "size", common.StorageSize(batch.ValueSize()), "elapsed", common.PrettyDuration(time.Since(start))) + + var ( + err error + root common.Hash + isEIP158 = bc.chainConfig.IsEIP158(block.Number()) + isCancun = bc.chainConfig.IsCancun(block.Number(), block.Time(), types.DeserializeHeaderExtraInformation(block.Header()).ArbOSFormatVersion) + hasStateHook = bc.logger != nil && bc.logger.OnStateUpdate != nil + hasStateSizer = bc.stateSizer != nil + ) + if hasStateHook || hasStateSizer { + r, update, err := statedb.CommitWithUpdate(block.NumberU64(), isEIP158, isCancun) + if err != nil { + return err + } + if hasStateHook { + trUpdate, err := update.ToTracingUpdate() + if err != nil { + return err + } + bc.logger.OnStateUpdate(trUpdate) + } + if hasStateSizer { + bc.stateSizer.Notify(update) + } + root = r + } else { + root, err = statedb.Commit(block.NumberU64(), isEIP158, isCancun) + if err != nil { + return err + } } // If node is running in path mode, skip explicit gc operation // which is unnecessary in this mode. @@ -1908,7 +1991,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { } defer bc.chainmu.Unlock() - _, n, err := bc.insertChain(chain, true, false) // No witness collection for mass inserts (would get super large) + _, n, err := bc.insertChain(context.Background(), chain, true, false) // No witness collection for mass inserts (would get super large) return n, err } @@ -1920,7 +2003,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { // racey behaviour. If a sidechain import is in progress, and the historic state // is imported, but then new canon-head is added before the actual sidechain // completes, then the historic state could be pruned again -func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness bool) (*stateless.Witness, int, error) { +func (bc *BlockChain) insertChain(ctx context.Context, chain types.Blocks, setHead bool, makeWitness bool) (*stateless.Witness, int, error) { // If the chain is terminating, don't even bother starting up. if bc.insertStopped() { return nil, 0, nil @@ -2003,11 +2086,11 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness if setHead { // First block is pruned, insert as sidechain and reorg only if TD grows enough log.Debug("Pruned ancestor, inserting as sidechain", "number", block.Number(), "hash", block.Hash()) - return bc.insertSideChain(block, it, makeWitness) + return bc.insertSideChain(ctx, block, it, makeWitness) } else { // We're post-merge and the parent is pruned, try to recover the parent state log.Debug("Pruned ancestor", "number", block.Number(), "hash", block.Hash()) - _, err := bc.recoverAncestors(block, makeWitness) + _, err := bc.recoverAncestors(ctx, block, makeWitness) return nil, it.index, err } // Some other error(except ErrKnownBlock) occurred, abort. @@ -2015,7 +2098,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness // still need re-execution to generate snapshots that are missing case err != nil && !errors.Is(err, ErrKnownBlock): stats.ignored += len(it.chain) - bc.reportBlock(block, nil, err) + bc.reportBadBlock(block, nil, err) return nil, it.index, err } // Track the singleton witness from this chain insertion (if any) @@ -2079,10 +2162,26 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness } // The traced section of block import. start := time.Now() - res, err := bc.ProcessBlock(parent.Root, block, setHead, makeWitness && len(chain) == 1) + config := ExecuteConfig{ + WriteState: true, + WriteHead: setHead, + EnableTracer: true, + MakeWitness: makeWitness && len(chain) == 1, + StatelessSelfValidation: bc.cfg.StatelessSelfValidation, + EnableWitnessStats: bc.cfg.EnableWitnessStats, + } + res, err := bc.ProcessBlock(ctx, parent.Root, block, config) if err != nil { return nil, it.index, err } + res.stats.reportMetrics() + + // Log slow block only if a single block is inserted (usually after the + // initial sync) to not overwhelm the users. + if len(chain) == 1 { + res.stats.logSlow(block, bc.slowBlockThreshold) + } + // Report the import stats before returning the various results stats.processed++ stats.usedGas += res.usedGas @@ -2143,25 +2242,58 @@ type blockProcessingResult struct { procTime time.Duration status WriteStatus witness *stateless.Witness + stats *ExecuteStats } func (bpr *blockProcessingResult) Witness() *stateless.Witness { return bpr.witness } +func (bpr *blockProcessingResult) Stats() *ExecuteStats { + return bpr.stats +} + +// ExecuteConfig defines optional behaviors during execution. +type ExecuteConfig struct { + // WriteState controls whether the computed state changes are persisted to + // the underlying storage. If false, execution is performed in-memory only. + WriteState bool + + // WriteHead indicates whether the execution result should update the canonical + // chain head. It's only relevant with WriteState == True. + WriteHead bool + + // EnableTracer enables execution tracing. This is typically used for debugging + // or analysis and may significantly impact performance. + EnableTracer bool + + // MakeWitness indicates whether to generate execution witness data during + // execution. Enabling this may introduce additional memory and CPU overhead. + MakeWitness bool + + // StatelessSelfValidation indicates whether the execution witnesses generation + // and self-validation (testing purpose) is enabled. + StatelessSelfValidation bool + + // EnableWitnessStats indicates whether to enable collection of witness trie + // access statistics + EnableWitnessStats bool +} + // ProcessBlock executes and validates the given block. If there was no error // it writes the block and associated state to database. -func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, setHead bool, makeWitness bool) (_ *blockProcessingResult, blockEndErr error) { +func (bc *BlockChain) ProcessBlock(ctx context.Context, parentRoot common.Hash, block *types.Block, config ExecuteConfig) (result *blockProcessingResult, blockEndErr error) { var ( err error startTime = time.Now() statedb *state.StateDB interrupt atomic.Bool + sdb = state.NewDatabase(bc.triedb, bc.codedb).WithSnapshot(bc.snaps) ) defer interrupt.Store(true) // terminate the prefetch at the end if bc.cfg.NoPrefetch { - statedb, err = state.New(parentRoot, bc.statedb) + statedb, err = state.New(parentRoot, sdb) if err != nil { return nil, err } @@ -2171,32 +2303,29 @@ func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, s // // Note: the main processor and prefetcher share the same reader with a local // cache for mitigating the overhead of state access. - prefetch, process, err := bc.statedb.ReadersWithCacheStats(parentRoot) + prefetch, process, err := sdb.ReadersWithCacheStats(parentRoot) if err != nil { return nil, err } - throwaway, err := state.NewWithReader(parentRoot, bc.statedb, prefetch) + throwaway, err := state.NewWithReader(parentRoot, sdb, prefetch) if err != nil { return nil, err } - statedb, err = state.NewWithReader(parentRoot, bc.statedb, process) + statedb, err = state.NewWithReader(parentRoot, sdb, process) if err != nil { return nil, err } // Upload the statistics of reader at the end defer func() { - stats := prefetch.GetStats() - accountCacheHitPrefetchMeter.Mark(stats.AccountHit) - accountCacheMissPrefetchMeter.Mark(stats.AccountMiss) - storageCacheHitPrefetchMeter.Mark(stats.StorageHit) - storageCacheMissPrefetchMeter.Mark(stats.StorageMiss) - stats = process.GetStats() - accountCacheHitMeter.Mark(stats.AccountHit) - accountCacheMissMeter.Mark(stats.AccountMiss) - storageCacheHitMeter.Mark(stats.StorageHit) - storageCacheMissMeter.Mark(stats.StorageMiss) + if result != nil { + if stater, ok := prefetch.(state.ReaderStater); ok { + result.stats.StatePrefetchCacheStats = stater.GetStats() + } + if stater, ok := process.(state.ReaderStater); ok { + result.stats.StateReadCacheStats = stater.GetStats() + } + } }() - go func(start time.Time, throwaway *state.StateDB, block *types.Block) { // Disable tracing for prefetcher executions. vmCfg := bc.cfg.VmConfig @@ -2213,52 +2342,54 @@ func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, s // If we are past Byzantium, enable prefetching to pull in trie node paths // while processing transactions. Before Byzantium the prefetcher is mostly // useless due to the intermediate root hashing after each transaction. - var ( - witness *stateless.Witness - witnessStats *stateless.WitnessStats - ) + var witness *stateless.Witness if bc.chainConfig.IsByzantium(block.Number()) { // Generate witnesses either if we're self-testing, or if it's the // only block being inserted. A bit crude, but witnesses are huge, // so we refuse to make an entire chain of them. - if bc.cfg.VmConfig.StatelessSelfValidation || makeWitness { - witness, err = stateless.NewWitness(block.Header(), bc) + if config.StatelessSelfValidation || config.MakeWitness { + witness, err = stateless.NewWitness(block.Header(), bc, config.EnableWitnessStats) if err != nil { return nil, err } - if bc.cfg.VmConfig.EnableWitnessStats { - witnessStats = stateless.NewWitnessStats() - } } - statedb.StartPrefetcher("chain", witness, witnessStats) + statedb.StartPrefetcher("chain", witness) defer statedb.StopPrefetcher() } - if bc.logger != nil && bc.logger.OnBlockStart != nil { - bc.logger.OnBlockStart(tracing.BlockEvent{ - Block: block, - Finalized: bc.CurrentFinalBlock(), - Safe: bc.CurrentSafeBlock(), - }) - } - if bc.logger != nil && bc.logger.OnBlockEnd != nil { - defer func() { - bc.logger.OnBlockEnd(blockEndErr) - }() + // Instrument the blockchain tracing + if config.EnableTracer { + if bc.logger != nil && bc.logger.OnBlockStart != nil { + bc.logger.OnBlockStart(tracing.BlockEvent{ + Block: block, + Finalized: bc.CurrentFinalBlock(), + Safe: bc.CurrentSafeBlock(), + }) + } + if bc.logger != nil && bc.logger.OnBlockEnd != nil { + defer func() { + bc.logger.OnBlockEnd(blockEndErr) + }() + } } // Process block using the parent state as reference point pstart := time.Now() - res, err := bc.processor.Process(block, statedb, bc.cfg.VmConfig) + pctx, _, spanEnd := telemetry.StartSpan(ctx, "bc.processor.Process") + res, err := bc.processor.Process(pctx, block, statedb, bc.cfg.VmConfig) + spanEnd(&err) if err != nil { - bc.reportBlock(block, res, err) + bc.reportBadBlock(block, res, err) return nil, err } ptime := time.Since(pstart) vstart := time.Now() - if err := bc.validator.ValidateState(block, statedb, res, false); err != nil { - bc.reportBlock(block, res, err) + _, _, spanEnd = telemetry.StartSpan(ctx, "bc.validator.ValidateState") + err = bc.validator.ValidateState(block, statedb, res, false) + spanEnd(&err) + if err != nil { + bc.reportBadBlock(block, res, err) return nil, err } vtime := time.Since(vstart) @@ -2269,7 +2400,7 @@ func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, s // witness builder/runner, which would otherwise be impossible due to the // various invalid chain states/behaviors being contained in those tests. xvstart := time.Now() - if witness := statedb.Witness(); witness != nil && bc.cfg.VmConfig.StatelessSelfValidation { + if witness := statedb.Witness(); witness != nil && config.StatelessSelfValidation { log.Warn("Running stateless self-validation", "block", block.Number(), "hash", block.Hash()) // Remove critical computed fields from the block to force true recalculation @@ -2280,7 +2411,7 @@ func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, s task := types.NewBlockWithHeader(context).WithBody(*block.Body()) // Run the stateless self-cross-validation - crossStateRoot, crossReceiptRoot, err := ExecuteStateless(bc.chainConfig, bc.cfg.VmConfig, task, witness) + crossStateRoot, crossReceiptRoot, err := ExecuteStateless(ctx, bc.chainConfig, bc.cfg.VmConfig, task, witness) if err != nil { return nil, fmt.Errorf("stateless self-validation failed: %v", err) } @@ -2292,68 +2423,72 @@ func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, s } } - xvtime := time.Since(xvstart) - proctime := time.Since(startTime) // processing + validation + cross validation - + var ( + xvtime = time.Since(xvstart) + proctime = time.Since(startTime) // processing + validation + cross validation + stats = &ExecuteStats{} + ) // Update the metrics touched during block processing and validation - accountReadTimer.Update(statedb.AccountReads) // Account reads are complete(in processing) - storageReadTimer.Update(statedb.StorageReads) // Storage reads are complete(in processing) - if statedb.AccountLoaded != 0 { - accountReadSingleTimer.Update(statedb.AccountReads / time.Duration(statedb.AccountLoaded)) - } - if statedb.StorageLoaded != 0 { - storageReadSingleTimer.Update(statedb.StorageReads / time.Duration(statedb.StorageLoaded)) - } - accountUpdateTimer.Update(statedb.AccountUpdates) // Account updates are complete(in validation) - storageUpdateTimer.Update(statedb.StorageUpdates) // Storage updates are complete(in validation) - accountHashTimer.Update(statedb.AccountHashes) // Account hashes are complete(in validation) - triehash := statedb.AccountHashes // The time spent on tries hashing - trieUpdate := statedb.AccountUpdates + statedb.StorageUpdates // The time spent on tries update - blockExecutionTimer.Update(ptime - (statedb.AccountReads + statedb.StorageReads)) // The time spent on EVM processing - blockValidationTimer.Update(vtime - (triehash + trieUpdate)) // The time spent on block validation - blockCrossValidationTimer.Update(xvtime) // The time spent on stateless cross validation + stats.AccountReads = statedb.AccountReads // Account reads are complete(in processing) + stats.StorageReads = statedb.StorageReads // Storage reads are complete(in processing) + stats.AccountUpdates = statedb.AccountUpdates // Account updates are complete(in validation) + stats.StorageUpdates = statedb.StorageUpdates // Storage updates are complete(in validation) + stats.AccountHashes = statedb.AccountHashes // Account hashes are complete(in validation) + stats.CodeReads = statedb.CodeReads + + stats.AccountLoaded = statedb.AccountLoaded + stats.AccountUpdated = statedb.AccountUpdated + stats.AccountDeleted = statedb.AccountDeleted + stats.StorageLoaded = statedb.StorageLoaded + stats.StorageUpdated = int(statedb.StorageUpdated.Load()) + stats.StorageDeleted = int(statedb.StorageDeleted.Load()) + + stats.CodeLoaded = statedb.CodeLoaded + stats.CodeLoadBytes = statedb.CodeLoadBytes + stats.CodeUpdated = statedb.CodeUpdated + stats.CodeUpdateBytes = statedb.CodeUpdateBytes + + stats.Execution = ptime - (statedb.AccountReads + statedb.StorageReads + statedb.CodeReads) // The time spent on EVM processing + stats.Validation = vtime - (statedb.AccountHashes + statedb.AccountUpdates + statedb.StorageUpdates) // The time spent on block validation + stats.CrossValidation = xvtime // The time spent on stateless cross validation // Write the block to the chain and get the status. - var ( - wstart = time.Now() - status WriteStatus - ) - if !setHead { - // Don't set the head, only insert the block - err = bc.writeBlockWithState(block, res.Receipts, statedb) - } else { - status, err = bc.writeBlockAndSetHead(block, res.Receipts, res.Logs, statedb, false) - } - if err != nil { - return nil, err + var status WriteStatus + if config.WriteState { + wstart := time.Now() + if !config.WriteHead { + // Don't set the head, only insert the block + err = bc.writeBlockWithState(block, res.Receipts, statedb) + } else { + status, err = bc.writeBlockAndSetHead(block, res.Receipts, res.Logs, statedb, false) + } + if err != nil { + return nil, err + } + // Update the metrics touched during block commit + stats.AccountCommits = statedb.AccountCommits // Account commits are complete, we can mark them + stats.StorageCommits = statedb.StorageCommits // Storage commits are complete, we can mark them + stats.DatabaseCommit = statedb.DatabaseCommits // Database commits are complete, we can mark them + stats.BlockWrite = time.Since(wstart) - max(statedb.AccountCommits, statedb.StorageCommits) /* concurrent */ - statedb.DatabaseCommits } // Report the collected witness statistics - if witnessStats != nil { - witnessStats.ReportMetrics(block.NumberU64()) + if witness != nil { + witness.ReportMetrics(block.NumberU64()) } - - // Update the metrics touched during block commit - accountCommitTimer.Update(statedb.AccountCommits) // Account commits are complete, we can mark them - storageCommitTimer.Update(statedb.StorageCommits) // Storage commits are complete, we can mark them - snapshotCommitTimer.Update(statedb.SnapshotCommits) // Snapshot commits are complete, we can mark them - triedbCommitTimer.Update(statedb.TrieDBCommits) // Trie database commits are complete, we can mark them - - blockWriteTimer.Update(time.Since(wstart) - max(statedb.AccountCommits, statedb.StorageCommits) /* concurrent */ - statedb.SnapshotCommits - statedb.TrieDBCommits) elapsed := time.Since(startTime) + 1 // prevent zero division - blockInsertTimer.Update(elapsed) + stats.TotalTime = elapsed + stats.MgasPerSecond = float64(res.GasUsed) * 1000 / float64(elapsed) + if bc.logger != nil && bc.logger.OnBlockEndMetrics != nil { bc.logger.OnBlockEndMetrics(block.NumberU64(), elapsed) } - // TODO(rjl493456442) generalize the ResettingTimer - mgasps := float64(res.GasUsed) * 1000 / float64(elapsed) - chainMgaspsMeter.Update(time.Duration(mgasps)) - return &blockProcessingResult{ usedGas: res.GasUsed, procTime: proctime, status: status, witness: witness, + stats: stats, }, nil } @@ -2364,7 +2499,7 @@ func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, s // The method writes all (header-and-body-valid) blocks to disk, then tries to // switch over to the new chain if the TD exceeded the current chain. // insertSideChain is only used pre-merge. -func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator, makeWitness bool) (*stateless.Witness, int, error) { +func (bc *BlockChain) insertSideChain(ctx context.Context, block *types.Block, it *insertIterator, makeWitness bool) (*stateless.Witness, int, error) { var current = bc.CurrentBlock() // The first sidechain block error is already verified to be ErrPrunedAncestor. @@ -2445,7 +2580,7 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator, ma // memory here. if len(blocks) >= 2048 || memory > 64*1024*1024 { log.Info("Importing heavy sidechain segment", "blocks", len(blocks), "start", blocks[0].NumberU64(), "end", block.NumberU64()) - if _, _, err := bc.insertChain(blocks, true, false); err != nil { + if _, _, err := bc.insertChain(ctx, blocks, true, false); err != nil { return nil, 0, err } blocks, memory = blocks[:0], 0 @@ -2459,7 +2594,7 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator, ma } if len(blocks) > 0 { log.Info("Importing sidechain segment", "start", blocks[0].NumberU64(), "end", blocks[len(blocks)-1].NumberU64()) - return bc.insertChain(blocks, true, makeWitness) + return bc.insertChain(ctx, blocks, true, makeWitness) } return nil, 0, nil } @@ -2468,7 +2603,7 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator, ma // all the ancestor blocks since that. // recoverAncestors is only used post-merge. // We return the hash of the latest block that we could correctly validate. -func (bc *BlockChain) recoverAncestors(block *types.Block, makeWitness bool) (common.Hash, error) { +func (bc *BlockChain) recoverAncestors(ctx context.Context, block *types.Block, makeWitness bool) (common.Hash, error) { // Gather all the sidechain hashes (full blocks may be memory heavy) var ( hashes []common.Hash @@ -2508,7 +2643,7 @@ func (bc *BlockChain) recoverAncestors(block *types.Block, makeWitness bool) (co } else { b = bc.GetBlock(hashes[i], numbers[i]) } - if _, _, err := bc.insertChain(types.Blocks{b}, false, makeWitness && i == 0); err != nil { + if _, _, err := bc.insertChain(ctx, types.Blocks{b}, false, makeWitness && i == 0); err != nil { return b.ParentHash(), err } } @@ -2632,6 +2767,7 @@ func (bc *BlockChain) reorg(oldHead *types.Header, newHead *types.Header) error // as the txlookups should be changed atomically, and all subsequent // reads should be blocked until the mutation is complete. bc.txLookupLock.Lock() + defer bc.txLookupLock.Unlock() // Reorg can be executed, start reducing the chain's old blocks and appending // the new blocks @@ -2709,6 +2845,8 @@ func (bc *BlockChain) reorg(oldHead *types.Header, newHead *types.Header) error // Delete useless indexes right now which includes the non-canonical // transaction indexes, canonical chain indexes which above the head. batch := bc.db.NewBatch() + defer batch.Close() + for _, tx := range types.HashDifference(deletedTxs, rebirthTxs) { rawdb.DeleteTxLookupEntry(batch, tx) } @@ -2732,9 +2870,6 @@ func (bc *BlockChain) reorg(oldHead *types.Header, newHead *types.Header) error // Reset the tx lookup cache to clear stale txlookup cache. bc.txLookupCache.Purge() - // Release the tx-lookup lock after mutation. - bc.txLookupLock.Unlock() - return nil } @@ -2743,14 +2878,16 @@ func (bc *BlockChain) reorg(oldHead *types.Header, newHead *types.Header) error // The key difference between the InsertChain is it won't do the canonical chain // updating. It relies on the additional SetCanonical call to finalize the entire // procedure. -func (bc *BlockChain) InsertBlockWithoutSetHead(block *types.Block, makeWitness bool) (*stateless.Witness, error) { +func (bc *BlockChain) InsertBlockWithoutSetHead(ctx context.Context, block *types.Block, makeWitness bool) (witness *stateless.Witness, err error) { + _, _, spanEnd := telemetry.StartSpan(ctx, "core.blockchain.InsertBlockWithoutSetHead") + defer spanEnd(&err) if !bc.chainmu.TryLock() { return nil, errChainStopped } defer bc.chainmu.Unlock() - witness, _, err := bc.insertChain(types.Blocks{block}, false, makeWitness) - return witness, err + witness, _, err = bc.insertChain(ctx, types.Blocks{block}, false, makeWitness) + return } // SetCanonical rewinds the chain to set the new head block as the specified @@ -2764,7 +2901,7 @@ func (bc *BlockChain) SetCanonical(head *types.Block) (common.Hash, error) { // Re-execute the reorged chain in case the head state is missing. if !bc.HasState(head.Root()) { - if latestValidHash, err := bc.recoverAncestors(head, false); err != nil { + if latestValidHash, err := bc.recoverAncestors(context.Background(), head, false); err != nil { return latestValidHash, err } log.Info("Recovered head state", "number", head.Number(), "hash", head.Hash()) @@ -2846,8 +2983,8 @@ func (bc *BlockChain) skipBlock(err error, it *insertIterator) bool { return false } -// reportBlock logs a bad block error. -func (bc *BlockChain) reportBlock(block *types.Block, res *ProcessResult, err error) { +// reportBadBlock logs a bad block error. +func (bc *BlockChain) reportBadBlock(block *types.Block, res *ProcessResult, err error) { var receipts types.Receipts if res != nil { receipts = res.Receipts diff --git a/core/blockchain_arbitrum.go b/core/blockchain_arbitrum.go index 0bb075c239..25f45146ae 100644 --- a/core/blockchain_arbitrum.go +++ b/core/blockchain_arbitrum.go @@ -18,6 +18,7 @@ package core import ( + "context" "fmt" "math/rand/v2" "time" @@ -106,11 +107,11 @@ func (bc *BlockChain) ClipToPostNitroGenesis(blockNum rpc.BlockNumber) (rpc.Bloc return blockNum, currentBlock } -func (bc *BlockChain) RecoverState(block *types.Block) error { +func (bc *BlockChain) RecoverState(ctx context.Context, block *types.Block) error { if bc.HasState(block.Root()) { return nil } log.Warn("recovering block state", "num", block.Number(), "hash", block.Hash(), "root", block.Root()) - _, err := bc.recoverAncestors(block, false) + _, err := bc.recoverAncestors(ctx, block, false) return err } diff --git a/core/blockchain_insert.go b/core/blockchain_insert.go index ac6a156d3e..07a250a1bb 100644 --- a/core/blockchain_insert.go +++ b/core/blockchain_insert.go @@ -131,28 +131,6 @@ func (it *insertIterator) next() (*types.Block, error) { return it.chain[it.index], it.validator.ValidateBody(it.chain[it.index]) } -// peek returns the next block in the iterator, along with any potential validation -// error for that block, but does **not** advance the iterator. -// -// Both header and body validation errors (nil too) is cached into the iterator -// to avoid duplicating work on the following next() call. -// nolint:unused -func (it *insertIterator) peek() (*types.Block, error) { - // If we reached the end of the chain, abort - if it.index+1 >= len(it.chain) { - return nil, nil - } - // Wait for verification result if not yet done - if len(it.errors) <= it.index+1 { - it.errors = append(it.errors, <-it.results) - } - if it.errors[it.index+1] != nil { - return it.chain[it.index+1], it.errors[it.index+1] - } - // Block header valid, ignore body validation since we don't have a parent anyway - return it.chain[it.index+1], nil -} - // previous returns the previous header that was being processed, or nil. func (it *insertIterator) previous() *types.Header { if it.index < 1 { diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index 8160e5364c..5d9b13fcf1 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" @@ -379,7 +380,7 @@ func (bc *BlockChain) TxIndexDone() bool { // HasState checks if state trie is fully present in the database or not. func (bc *BlockChain) HasState(hash common.Hash) bool { - _, err := bc.statedb.OpenTrie(hash) + _, err := bc.triedb.NodeReader(hash) return err == nil } @@ -411,7 +412,7 @@ func (bc *BlockChain) stateRecoverable(root common.Hash) bool { func (bc *BlockChain) ContractCodeWithPrefix(hash common.Hash) []byte { // TODO(rjl493456442) The associated account address is also required // in Verkle scheme. Fix it once snap-sync is supported for Verkle. - return bc.statedb.ContractCodeWithPrefix(common.Address{}, hash) + return bc.codedb.Reader().CodeWithPrefix(common.Address{}, hash) } // State returns a new mutable state based on the current HEAD block. @@ -421,14 +422,14 @@ func (bc *BlockChain) State() (*state.StateDB, error) { // StateAt returns a new mutable state based on a particular point in time. func (bc *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) { - return state.New(root, bc.statedb) + return state.New(root, state.NewDatabase(bc.triedb, bc.codedb).WithSnapshot(bc.snaps)) } // HistoricState returns a historic state specified by the given root. // Live states are not available and won't be served, please use `State` // or `StateAt` instead. func (bc *BlockChain) HistoricState(root common.Hash) (*state.StateDB, error) { - return state.New(root, state.NewHistoricDatabase(bc.db, bc.triedb)) + return state.New(root, state.NewHistoricDatabase(bc.triedb, bc.codedb)) } // Config retrieves the chain's fork configuration. @@ -452,11 +453,6 @@ func (bc *BlockChain) Processor() Processor { return bc.processor } -// StateCache returns the caching database underpinning the blockchain instance. -func (bc *BlockChain) StateCache() state.Database { - return bc.statedb -} - // GasLimit returns the gas limit of the current HEAD block. func (bc *BlockChain) GasLimit() uint64 { return bc.CurrentBlock().GasLimit @@ -472,6 +468,24 @@ func (bc *BlockChain) GetVMConfig() *vm.Config { return &bc.cfg.VmConfig } +// Arbitrum: StatelessSelfValidation reports whether execution-witness self-validation +// is enabled on the chain config. +func (bc *BlockChain) StatelessSelfValidation() bool { + return bc.cfg.StatelessSelfValidation +} + +// Arbitrum: EnableWitnessStats reports whether witness trie access statistics +// collection is enabled on the chain config. +func (bc *BlockChain) EnableWitnessStats() bool { + return bc.cfg.EnableWitnessStats +} + +// Arbitrum: WasmStore returns the underlying key-value store used for Stylus +// activated-program data. +func (bc *BlockChain) WasmStore() ethdb.KeyValueStore { + return bc.triedb.Disk().WasmDataBase() +} + // TxIndexProgress returns the transaction indexing progress. func (bc *BlockChain) TxIndexProgress() (TxIndexProgress, error) { if bc.txIndexer == nil { @@ -500,6 +514,11 @@ func (bc *BlockChain) TrieDB() *triedb.Database { return bc.triedb } +// CodeDB retrieves the low level contract code database used for data storage. +func (bc *BlockChain) CodeDB() *state.CodeDB { + return bc.codedb +} + // HeaderChain returns the underlying header chain. func (bc *BlockChain) HeaderChain() *HeaderChain { return bc.hc @@ -530,3 +549,13 @@ func (bc *BlockChain) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscript func (bc *BlockChain) SubscribeBlockProcessingEvent(ch chan<- bool) event.Subscription { return bc.scope.Track(bc.blockProcFeed.Subscribe(ch)) } + +// SubscribeNewPayloadEvent registers a subscription for NewPayloadEvent. +func (bc *BlockChain) SubscribeNewPayloadEvent(ch chan<- NewPayloadEvent) event.Subscription { + return bc.scope.Track(bc.newPayloadFeed.Subscribe(ch)) +} + +// SendNewPayloadEvent sends a NewPayloadEvent to subscribers. +func (bc *BlockChain) SendNewPayloadEvent(ev NewPayloadEvent) { + bc.newPayloadFeed.Send(ev) +} diff --git a/core/blockchain_sethead_test.go b/core/blockchain_sethead_test.go index b7b1ebdbd4..54761966ad 100644 --- a/core/blockchain_sethead_test.go +++ b/core/blockchain_sethead_test.go @@ -30,7 +30,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb/pebble" "github.com/ethereum/go-ethereum/params" @@ -2044,7 +2043,6 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme dbconfig.HashDB = hashdb.Defaults } chain.triedb = triedb.NewDatabase(chain.db, dbconfig) - chain.statedb = state.NewDatabase(chain.triedb, chain.snaps) // Force run a freeze cycle type freezer interface { diff --git a/core/blockchain_stats.go b/core/blockchain_stats.go new file mode 100644 index 0000000000..d753b0b700 --- /dev/null +++ b/core/blockchain_stats.go @@ -0,0 +1,255 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package core + +import ( + "encoding/json" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" +) + +// ExecuteStats includes all the statistics of a block execution in details. +type ExecuteStats struct { + // State read times + AccountReads time.Duration // Time spent on the account reads + StorageReads time.Duration // Time spent on the storage reads + AccountHashes time.Duration // Time spent on the account trie hash + AccountUpdates time.Duration // Time spent on the account trie update + AccountCommits time.Duration // Time spent on the account trie commit + StorageUpdates time.Duration // Time spent on the storage trie update + StorageCommits time.Duration // Time spent on the storage trie commit + CodeReads time.Duration // Time spent on the contract code read + + AccountLoaded int // Number of accounts loaded + AccountUpdated int // Number of accounts updated + AccountDeleted int // Number of accounts deleted + StorageLoaded int // Number of storage slots loaded + StorageUpdated int // Number of storage slots updated + StorageDeleted int // Number of storage slots deleted + CodeLoaded int // Number of contract code loaded + CodeLoadBytes int // Number of bytes read from contract code + CodeUpdated int // Number of contract code written (CREATE/CREATE2 + EIP-7702) + CodeUpdateBytes int // Total bytes of code written + + Execution time.Duration // Time spent on the EVM execution + Validation time.Duration // Time spent on the block validation + CrossValidation time.Duration // Optional, time spent on the block cross validation + DatabaseCommit time.Duration // Time spent on database commit + BlockWrite time.Duration // Time spent on block write + TotalTime time.Duration // The total time spent on block execution + MgasPerSecond float64 // The million gas processed per second + + // Cache hit rates + StateReadCacheStats state.ReaderStats + StatePrefetchCacheStats state.ReaderStats +} + +// reportMetrics uploads execution statistics to the metrics system. +func (s *ExecuteStats) reportMetrics() { + if s.AccountLoaded != 0 { + accountReadTimer.Update(s.AccountReads) + accountReadSingleTimer.Update(s.AccountReads / time.Duration(s.AccountLoaded)) + } + if s.StorageLoaded != 0 { + storageReadTimer.Update(s.StorageReads) + storageReadSingleTimer.Update(s.StorageReads / time.Duration(s.StorageLoaded)) + } + if s.CodeLoaded != 0 { + codeReadTimer.Update(s.CodeReads) + codeReadSingleTimer.Update(s.CodeReads / time.Duration(s.CodeLoaded)) + codeReadBytesTimer.Update(time.Duration(s.CodeLoadBytes)) + } + accountUpdateTimer.Update(s.AccountUpdates) // Account updates are complete(in validation) + storageUpdateTimer.Update(s.StorageUpdates) // Storage updates are complete(in validation) + accountHashTimer.Update(s.AccountHashes) // Account hashes are complete(in validation) + accountCommitTimer.Update(s.AccountCommits) // Account commits are complete, we can mark them + storageCommitTimer.Update(s.StorageCommits) // Storage commits are complete, we can mark them + + blockExecutionTimer.Update(s.Execution) // The time spent on EVM processing + blockValidationTimer.Update(s.Validation) // The time spent on block validation + blockCrossValidationTimer.Update(s.CrossValidation) // The time spent on stateless cross validation + triedbCommitTimer.Update(s.DatabaseCommit) // Trie database commits are complete, we can mark them + blockWriteTimer.Update(s.BlockWrite) // The time spent on block write + blockInsertTimer.Update(s.TotalTime) // The total time spent on block execution + chainMgaspsMeter.Update(time.Duration(s.MgasPerSecond)) // TODO(rjl493456442) generalize the ResettingTimer + + // Cache hit rates + accountCacheHitPrefetchMeter.Mark(s.StatePrefetchCacheStats.StateStats.AccountCacheHit) + accountCacheMissPrefetchMeter.Mark(s.StatePrefetchCacheStats.StateStats.AccountCacheMiss) + storageCacheHitPrefetchMeter.Mark(s.StatePrefetchCacheStats.StateStats.StorageCacheHit) + storageCacheMissPrefetchMeter.Mark(s.StatePrefetchCacheStats.StateStats.StorageCacheMiss) + + accountCacheHitMeter.Mark(s.StateReadCacheStats.StateStats.AccountCacheHit) + accountCacheMissMeter.Mark(s.StateReadCacheStats.StateStats.AccountCacheMiss) + storageCacheHitMeter.Mark(s.StateReadCacheStats.StateStats.StorageCacheHit) + storageCacheMissMeter.Mark(s.StateReadCacheStats.StateStats.StorageCacheMiss) +} + +// slowBlockLog represents the JSON structure for slow block logging. +// This format is designed for cross-client compatibility with other +// Ethereum execution clients (reth, Besu, Nethermind). +type slowBlockLog struct { + Level string `json:"level"` + Msg string `json:"msg"` + Block slowBlockInfo `json:"block"` + Timing slowBlockTime `json:"timing"` + Throughput slowBlockThru `json:"throughput"` + StateReads slowBlockReads `json:"state_reads"` + StateWrites slowBlockWrites `json:"state_writes"` + Cache slowBlockCache `json:"cache"` +} + +type slowBlockInfo struct { + Number uint64 `json:"number"` + Hash common.Hash `json:"hash"` + GasUsed uint64 `json:"gas_used"` + TxCount int `json:"tx_count"` +} + +type slowBlockTime struct { + ExecutionMs float64 `json:"execution_ms"` + StateReadMs float64 `json:"state_read_ms"` + StateHashMs float64 `json:"state_hash_ms"` + CommitMs float64 `json:"commit_ms"` + TotalMs float64 `json:"total_ms"` +} + +type slowBlockThru struct { + MgasPerSec float64 `json:"mgas_per_sec"` +} + +type slowBlockReads struct { + Accounts int `json:"accounts"` + StorageSlots int `json:"storage_slots"` + Code int `json:"code"` + CodeBytes int `json:"code_bytes"` +} + +type slowBlockWrites struct { + Accounts int `json:"accounts"` + AccountsDeleted int `json:"accounts_deleted"` + StorageSlots int `json:"storage_slots"` + StorageSlotsDeleted int `json:"storage_slots_deleted"` + Code int `json:"code"` + CodeBytes int `json:"code_bytes"` +} + +// slowBlockCache represents cache hit/miss statistics for cross-client analysis. +type slowBlockCache struct { + Account slowBlockCacheEntry `json:"account"` + Storage slowBlockCacheEntry `json:"storage"` + Code slowBlockCodeCacheEntry `json:"code"` +} + +// slowBlockCacheEntry represents cache statistics for account/storage caches. +type slowBlockCacheEntry struct { + Hits int64 `json:"hits"` + Misses int64 `json:"misses"` + HitRate float64 `json:"hit_rate"` +} + +// slowBlockCodeCacheEntry represents cache statistics for code cache with byte-level granularity. +type slowBlockCodeCacheEntry struct { + Hits int64 `json:"hits"` + Misses int64 `json:"misses"` + HitRate float64 `json:"hit_rate"` + HitBytes int64 `json:"hit_bytes"` + MissBytes int64 `json:"miss_bytes"` +} + +// durationToMs converts a time.Duration to milliseconds as a float64 +// with sub-millisecond precision for accurate cross-client metrics. +func durationToMs(d time.Duration) float64 { + return float64(d.Nanoseconds()) / 1e6 +} + +// logSlow prints the detailed execution statistics in JSON format if the block +// is regarded as slow. The JSON format is designed for cross-client compatibility +// with other Ethereum execution clients. +func (s *ExecuteStats) logSlow(block *types.Block, slowBlockThreshold time.Duration) { + // Negative threshold means disabled (default when flag not set) + if slowBlockThreshold < 0 { + return + } + // Threshold of 0 logs all blocks; positive threshold filters + if slowBlockThreshold > 0 && s.TotalTime < slowBlockThreshold { + return + } + logEntry := slowBlockLog{ + Level: "warn", + Msg: "Slow block", + Block: slowBlockInfo{ + Number: block.NumberU64(), + Hash: block.Hash(), + GasUsed: block.GasUsed(), + TxCount: len(block.Transactions()), + }, + Timing: slowBlockTime{ + ExecutionMs: durationToMs(s.Execution), + StateReadMs: durationToMs(s.AccountReads + s.StorageReads + s.CodeReads), + StateHashMs: durationToMs(s.AccountHashes + s.AccountUpdates + s.StorageUpdates), + CommitMs: durationToMs(max(s.AccountCommits, s.StorageCommits) + s.DatabaseCommit + s.BlockWrite), + TotalMs: durationToMs(s.TotalTime), + }, + Throughput: slowBlockThru{ + MgasPerSec: s.MgasPerSecond, + }, + StateReads: slowBlockReads{ + Accounts: s.AccountLoaded, + StorageSlots: s.StorageLoaded, + Code: s.CodeLoaded, + CodeBytes: s.CodeLoadBytes, + }, + StateWrites: slowBlockWrites{ + Accounts: s.AccountUpdated, + AccountsDeleted: s.AccountDeleted, + StorageSlots: s.StorageUpdated, + StorageSlotsDeleted: s.StorageDeleted, + Code: s.CodeUpdated, + CodeBytes: s.CodeUpdateBytes, + }, + Cache: slowBlockCache{ + Account: slowBlockCacheEntry{ + Hits: s.StateReadCacheStats.StateStats.AccountCacheHit, + Misses: s.StateReadCacheStats.StateStats.AccountCacheMiss, + HitRate: s.StateReadCacheStats.StateStats.AccountCacheHitRate(), + }, + Storage: slowBlockCacheEntry{ + Hits: s.StateReadCacheStats.StateStats.StorageCacheHit, + Misses: s.StateReadCacheStats.StateStats.StorageCacheMiss, + HitRate: s.StateReadCacheStats.StateStats.StorageCacheHitRate(), + }, + Code: slowBlockCodeCacheEntry{ + Hits: s.StateReadCacheStats.CodeStats.CacheHit, + Misses: s.StateReadCacheStats.CodeStats.CacheMiss, + HitRate: s.StateReadCacheStats.CodeStats.HitRate(), + HitBytes: s.StateReadCacheStats.CodeStats.CacheHitBytes, + MissBytes: s.StateReadCacheStats.CodeStats.CacheMissBytes, + }, + }, + } + jsonBytes, err := json.Marshal(logEntry) + if err != nil { + log.Error("Failed to marshal slow block log", "error", err) + return + } + log.Warn(string(jsonBytes)) +} diff --git a/core/blockchain_test.go b/core/blockchain_test.go index b1c71b2f81..cd44f063a3 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -18,6 +18,7 @@ package core import ( "bytes" + "context" "errors" "fmt" gomath "math" @@ -35,7 +36,6 @@ import ( "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/beacon" "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" @@ -156,18 +156,18 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error { } return err } - statedb, err := state.New(blockchain.GetBlockByHash(block.ParentHash()).Root(), blockchain.statedb) + statedb, err := state.New(blockchain.GetBlockByHash(block.ParentHash()).Root(), state.NewDatabase(blockchain.triedb, blockchain.codedb)) if err != nil { return err } - res, err := blockchain.processor.Process(block, statedb, vm.Config{}) + res, err := blockchain.processor.Process(context.Background(), block, statedb, vm.Config{}) if err != nil { - blockchain.reportBlock(block, res, err) + blockchain.reportBadBlock(block, res, err) return err } err = blockchain.validator.ValidateState(block, statedb, res, false) if err != nil { - blockchain.reportBlock(block, res, err) + blockchain.reportBadBlock(block, res, err) return err } @@ -3456,7 +3456,7 @@ func testSetCanonical(t *testing.T, scheme string) { gen.AddTx(tx) }) for _, block := range side { - _, err := chain.InsertBlockWithoutSetHead(block, false) + _, err := chain.InsertBlockWithoutSetHead(context.Background(), block, false) if err != nil { t.Fatalf("Failed to insert into chain: %v", err) } @@ -4336,20 +4336,8 @@ func TestInsertChainWithCutoff(t *testing.T) { func testInsertChainWithCutoff(t *testing.T, cutoff uint64, ancientLimit uint64, genesis *Genesis, blocks []*types.Block, receipts []types.Receipts) { // log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true))) - // Add a known pruning point for the duration of the test. ghash := genesis.ToBlock().Hash() cutoffBlock := blocks[cutoff-1] - history.PrunePoints[ghash] = &history.PrunePoint{ - BlockNumber: cutoffBlock.NumberU64(), - BlockHash: cutoffBlock.Hash(), - } - defer func() { - delete(history.PrunePoints, ghash) - }() - - // Enable pruning in cache config. - config := DefaultConfig().WithStateScheme(rawdb.PathScheme) - config.ChainHistoryMode = history.KeepPostMerge db, _ := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{}) defer db.Close() @@ -4515,3 +4503,45 @@ func TestGetCanonicalReceipt(t *testing.T) { } } } + +// TestSetHeadBeyondRootFinalizedBug tests the issue where the finalized block +// is not cleared when rewinding past it using setHeadBeyondRoot. +func TestSetHeadBeyondRootFinalizedBug(t *testing.T) { + // Create a clean blockchain with 100 blocks using PathScheme (PBSS) + _, _, blockchain, err := newCanonical(ethash.NewFaker(), 100, true, rawdb.PathScheme) + if err != nil { + t.Fatalf("failed to create pristine chain: %v", err) + } + defer blockchain.Stop() + + // Set the "Finalized" marker to the current Head (Block 100) + headBlock := blockchain.CurrentBlock() + if headBlock.Number.Uint64() != 100 { + t.Fatalf("Setup failed: expected head 100, got %d", headBlock.Number.Uint64()) + } + blockchain.SetFinalized(headBlock) + + // Verify setup + if blockchain.CurrentFinalBlock().Number.Uint64() != 100 { + t.Fatalf("Setup failed: Finalized block should be 100") + } + targetBlock := blockchain.GetBlockByNumber(50) + + // Call setHeadBeyondRoot with: + // head = 100 + // repair = true + if _, _, err := blockchain.setHeadBeyondRoot(100, 0, targetBlock.Root(), true, 0); err != nil { + t.Fatalf("Failed to rewind: %v", err) + } + + currentFinal := blockchain.CurrentFinalBlock() + currentHead := blockchain.CurrentBlock().Number.Uint64() + + // The previous finalized block (100) is now invalid because we rewound to 50. + // The function should have cleared the finalized marker (set to nil). + if currentFinal != nil && currentFinal.Number.Uint64() > currentHead { + t.Errorf("Chain Head: %d , Finalized Block: %d , Finalized block was >= head block.", + currentHead, + currentFinal.Number.Uint64()) + } +} diff --git a/core/chain_makers.go b/core/chain_makers.go index ea91ee7970..515c623b81 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -17,6 +17,7 @@ package core import ( + "context" "fmt" "math/big" @@ -32,7 +33,6 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/triedb" - "github.com/ethereum/go-verkle" "github.com/holiman/uint256" ) @@ -64,7 +64,7 @@ func (b *BlockGen) SetCoinbase(addr common.Address) { panic("coinbase can only be set once") } b.header.Coinbase = addr - b.gasPool = new(GasPool).AddGas(b.header.GasLimit) + b.gasPool = NewGasPool(b.header.GasLimit) } // SetExtra sets the extra data field of the generated block. @@ -118,10 +118,12 @@ func (b *BlockGen) addTx(bc *BlockChain, vmConfig vm.Config, tx *types.Transacti evm = vm.NewEVM(blockContext, b.statedb, b.cm.config, vmConfig) ) b.statedb.SetTxContext(tx.Hash(), len(b.txs)) - receipt, _, err := ApplyTransaction(evm, b.gasPool, b.statedb, b.header, tx, &b.header.GasUsed) + receipt, _, err := ApplyTransaction(evm, b.gasPool, b.statedb, b.header, tx) if err != nil { panic(err) } + b.header.GasUsed = b.gasPool.Used() + // Merge the tx-local access event into the "block-local" one, in order to collect // all values, so that the witness can be built. if b.statedb.Database().TrieDB().IsVerkle() { @@ -413,7 +415,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse } body := types.Body{Transactions: b.txs, Uncles: b.uncles, Withdrawals: b.withdrawals} - block, err := b.engine.FinalizeAndAssemble(cm, b.header, statedb, &body, b.receipts) + block, err := b.engine.FinalizeAndAssemble(context.Background(), cm, b.header, statedb, &body, b.receipts) if err != nil { panic(err) } @@ -430,7 +432,11 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse } // Forcibly use hash-based state scheme for retaining all nodes in disk. - triedb := triedb.NewDatabase(db, triedb.HashDefaults) + var triedbConfig *triedb.Config = triedb.HashDefaults + if config.IsVerkle(config.ChainID, 0) { + triedbConfig = triedb.VerkleDefaults + } + triedb := triedb.NewDatabase(db, triedbConfig) defer triedb.Close() for i := 0; i < n; i++ { @@ -475,125 +481,19 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse // then generate chain on top. func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, gen func(int, *BlockGen)) (ethdb.Database, []*types.Block, []types.Receipts) { db := rawdb.NewMemoryDatabase() - triedb := triedb.NewDatabase(db, triedb.HashDefaults) - defer triedb.Close() - _, err := genesis.Commit(db, triedb) - if err != nil { - panic(err) - } - blocks, receipts := GenerateChain(genesis.Config, genesis.ToBlock(), engine, db, n, gen) - return db, blocks, receipts -} - -func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine consensus.Engine, db ethdb.Database, trdb *triedb.Database, n int, gen func(int, *BlockGen)) ([]*types.Block, []types.Receipts, []*verkle.VerkleProof, []verkle.StateDiff) { - if config == nil { - config = params.TestChainConfig + var triedbConfig *triedb.Config = triedb.HashDefaults + if genesis.Config != nil && genesis.Config.IsVerkle(genesis.Config.ChainID, 0) { + triedbConfig = triedb.VerkleDefaults } - proofs := make([]*verkle.VerkleProof, 0, n) - keyvals := make([]verkle.StateDiff, 0, n) - cm := newChainMaker(parent, config, engine) - - genblock := func(i int, parent *types.Block, triedb *triedb.Database, statedb *state.StateDB) (*types.Block, types.Receipts) { - b := &BlockGen{i: i, cm: cm, parent: parent, statedb: statedb, engine: engine} - b.header = cm.makeHeader(parent, statedb, b.engine) - - // TODO uncomment when proof generation is merged - // Save pre state for proof generation - // preState := statedb.Copy() - - // EIP-2935 / 7709 - blockContext := NewEVMBlockContext(b.header, cm, &b.header.Coinbase) - blockContext.Random = &common.Hash{} // enable post-merge instruction set - evm := vm.NewEVM(blockContext, statedb, cm.config, vm.Config{}) - ProcessParentBlockHash(b.header.ParentHash, evm) - - // Execute any user modifications to the block. - if gen != nil { - gen(i, b) - } - - requests := b.collectRequests(false) - if requests != nil { - reqHash := types.CalcRequestsHash(requests) - b.header.RequestsHash = &reqHash - } - - body := &types.Body{ - Transactions: b.txs, - Uncles: b.uncles, - Withdrawals: b.withdrawals, - } - block, err := b.engine.FinalizeAndAssemble(cm, b.header, statedb, body, b.receipts) - if err != nil { - panic(err) - } - - // Write state changes to DB. - root, err := statedb.Commit(b.header.Number.Uint64(), config.IsEIP158(b.header.Number), config.IsCancun(b.header.Number, b.header.Time, types.DeserializeHeaderExtraInformation(b.header).ArbOSFormatVersion)) - if err != nil { - panic(fmt.Sprintf("state write error: %v", err)) - } - if err = triedb.Commit(root, false); err != nil { - panic(fmt.Sprintf("trie write error: %v", err)) - } - - proofs = append(proofs, block.ExecutionWitness().VerkleProof) - keyvals = append(keyvals, block.ExecutionWitness().StateDiff) - - return block, b.receipts - } - - sdb := state.NewDatabase(trdb, nil) - - for i := 0; i < n; i++ { - statedb, err := state.New(parent.Root(), sdb) - if err != nil { - panic(err) - } - block, receipts := genblock(i, parent, trdb, statedb) - - // Post-process the receipts. - // Here we assign the final block hash and other info into the receipt. - // In order for DeriveFields to work, the transaction and receipt lists need to be - // of equal length. If AddUncheckedTx or AddUncheckedReceipt are used, there will be - // extra ones, so we just trim the lists here. - receiptsCount := len(receipts) - txs := block.Transactions() - if len(receipts) > len(txs) { - receipts = receipts[:len(txs)] - } else if len(receipts) < len(txs) { - txs = txs[:len(receipts)] - } - var blobGasPrice *big.Int - if block.ExcessBlobGas() != nil { - blobGasPrice = eip4844.CalcBlobFee(cm.config, block.Header()) - } - if err := receipts.DeriveFields(config, block.Hash(), block.NumberU64(), block.Time(), block.BaseFee(), blobGasPrice, txs, true); err != nil { - panic(err) - } - - // Re-expand to ensure all receipts are returned. - receipts = receipts[:receiptsCount] - - // Advance the chain. - cm.add(block, receipts) - parent = block - } - return cm.chain, cm.receipts, proofs, keyvals -} - -func GenerateVerkleChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, gen func(int, *BlockGen)) (common.Hash, ethdb.Database, []*types.Block, []types.Receipts, []*verkle.VerkleProof, []verkle.StateDiff) { - db := rawdb.NewMemoryDatabase() - cacheConfig := DefaultConfig().WithStateScheme(rawdb.PathScheme) - cacheConfig.SnapshotLimit = 0 - triedb := triedb.NewDatabase(db, cacheConfig.triedbConfig(true)) - defer triedb.Close() - genesisBlock, err := genesis.Commit(db, triedb) + genesisTriedb := triedb.NewDatabase(db, triedbConfig) + block, err := genesis.Commit(db, genesisTriedb, nil) if err != nil { + genesisTriedb.Close() panic(err) } - blocks, receipts, proofs, keyvals := GenerateVerkleChain(genesis.Config, genesisBlock, engine, db, triedb, n, gen) - return genesisBlock.Hash(), db, blocks, receipts, proofs, keyvals + genesisTriedb.Close() + blocks, receipts := GenerateChain(genesis.Config, block, engine, db, n, gen) + return db, blocks, receipts } func (cm *chainMaker) makeHeader(parent *types.Block, state *state.StateDB, engine consensus.Engine) *types.Header { diff --git a/core/error.go b/core/error.go index 635d802863..7dd5b8a432 100644 --- a/core/error.go +++ b/core/error.go @@ -58,14 +58,14 @@ var ( // by a transaction is higher than what's left in the block. ErrGasLimitReached = errors.New("gas limit reached") + // ErrGasLimitOverflow is returned by the gas pool if the remaining gas + // exceeds the maximum value of uint64. + ErrGasLimitOverflow = errors.New("gas limit overflow") + // ErrInsufficientFundsForTransfer is returned if the transaction sender doesn't // have enough funds for transfer(topmost call only). ErrInsufficientFundsForTransfer = errors.New("insufficient funds for transfer") - // ErrMaxInitCodeSizeExceeded is returned if creation transaction provides the init code bigger - // than init code size limit. - ErrMaxInitCodeSizeExceeded = errors.New("max initcode size exceeded") - // ErrInsufficientBalanceWitness is returned if the transaction sender has enough // funds to cover the transfer, but not enough to pay for witness access/modification // costs for the transaction diff --git a/core/eth_transfer_logs_test.go b/core/eth_transfer_logs_test.go new file mode 100644 index 0000000000..815b56b588 --- /dev/null +++ b/core/eth_transfer_logs_test.go @@ -0,0 +1,169 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package core + +import ( + "encoding/binary" + "math/big" + "reflect" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/beacon" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/params" +) + +var ethTransferTestCode = common.FromHex("6080604052600436106100345760003560e01c8063574ffc311461003957806366e41cb714610090578063f8a8fd6d1461009a575b600080fd5b34801561004557600080fd5b5061004e6100a4565b604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390f35b6100986100ac565b005b6100a26100f5565b005b63deadbeef81565b7f38e80b5c85ba49b7280ccc8f22548faa62ae30d5a008a1b168fba5f47f5d1ee560405160405180910390a1631234567873ffffffffffffffffffffffffffffffffffffffff16ff5b7f24ec1d3ff24c2f6ff210738839dbc339cd45a5294d85c79361016243157aae7b60405160405180910390a163deadbeef73ffffffffffffffffffffffffffffffffffffffff166002348161014657fe5b046040516024016040516020818303038152906040527f66e41cb7000000000000000000000000000000000000000000000000000000007bffffffffffffffffffffffffffffffffffffffffffffffffffffffff19166020820180517bffffffffffffffffffffffffffffffffffffffffffffffffffffffff83818316178352505050506040518082805190602001908083835b602083106101fd57805182526020820191506020810190506020830392506101da565b6001836020036101000a03801982511681845116808217855250505050505090500191505060006040518083038185875af1925050503d806000811461025f576040519150601f19603f3d011682016040523d82523d6000602084013e610264565b606091505b50505056fea265627a7a723158202cce817a434785d8560c200762f972d453ccd30694481be7545f9035a512826364736f6c63430005100032") + +/* +pragma solidity >=0.4.22 <0.6.0; + +contract TestLogs { + + address public constant target_contract = 0x00000000000000000000000000000000DeaDBeef; + address payable constant selfdestruct_addr = 0x0000000000000000000000000000000012345678; + + event Response(bool success, bytes data); + event TestEvent(); + event TestEvent2(); + + function test() public payable { + emit TestEvent(); + target_contract.call.value(msg.value/2)(abi.encodeWithSignature("test2()")); + } + function test2() public payable { + emit TestEvent2(); + selfdestruct(selfdestruct_addr); + } +} +*/ + +// TestEthTransferLogs tests EIP-7708 ETH transfer log output by simulating a +// scenario including transaction, CALL and SELFDESTRUCT value transfers, and +// also "ordinary" logs emitted. The same scenario is also tested with no value +// transferred. +func TestEthTransferLogs(t *testing.T) { + testEthTransferLogs(t, 1_000_000_000) + testEthTransferLogs(t, 0) +} + +func testEthTransferLogs(t *testing.T, value uint64) { + var ( + key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + addr1 = crypto.PubkeyToAddress(key1.PublicKey) + addr2 = common.HexToAddress("cafebabe") // caller + addr3 = common.HexToAddress("deadbeef") // callee + addr4 = common.HexToAddress("12345678") // selfdestruct target + testEvent = crypto.Keccak256Hash([]byte("TestEvent()")) + testEvent2 = crypto.Keccak256Hash([]byte("TestEvent2()")) + config = *params.MergedTestChainConfig + signer = types.LatestSigner(&config) + engine = beacon.New(ethash.NewFaker()) + ) + + //TODO remove this hacky config initialization when final Amsterdam config is available + config.AmsterdamTime = new(uint64) + blobConfig := *config.BlobScheduleConfig + blobConfig.Amsterdam = blobConfig.Osaka + config.BlobScheduleConfig = &blobConfig + + gspec := &Genesis{ + Config: &config, + Alloc: types.GenesisAlloc{ + addr1: {Balance: newGwei(1000000000)}, + addr2: {Code: ethTransferTestCode}, + addr3: {Code: ethTransferTestCode}, + }, + } + _, blocks, receipts := GenerateChainWithGenesis(gspec, engine, 1, func(i int, b *BlockGen) { + tx := types.MustSignNewTx(key1, signer, &types.DynamicFeeTx{ + ChainID: gspec.Config.ChainID, + Nonce: 0, + To: &addr2, + Gas: 500_000, + GasFeeCap: newGwei(5), + GasTipCap: newGwei(5), + Value: big.NewInt(int64(value)), + Data: common.FromHex("f8a8fd6d"), + }) + b.AddTx(tx) + }) + + blockHash := blocks[0].Hash() + txHash := blocks[0].Transactions()[0].Hash() + addr2hash := func(addr common.Address) (hash common.Hash) { + copy(hash[12:], addr[:]) + return + } + u256 := func(amount uint64) []byte { + data := make([]byte, 32) + binary.BigEndian.PutUint64(data[24:], amount) + return data + } + + var expLogs = []*types.Log{ + { + Address: params.SystemAddress, + Topics: []common.Hash{params.EthTransferLogEvent, addr2hash(addr1), addr2hash(addr2)}, + Data: u256(value), + }, + { + Address: addr2, + Topics: []common.Hash{testEvent}, + Data: nil, + }, + { + Address: params.SystemAddress, + Topics: []common.Hash{params.EthTransferLogEvent, addr2hash(addr2), addr2hash(addr3)}, + Data: u256(value / 2), + }, + { + Address: addr3, + Topics: []common.Hash{testEvent2}, + Data: nil, + }, + { + Address: params.SystemAddress, + Topics: []common.Hash{params.EthTransferLogEvent, addr2hash(addr3), addr2hash(addr4)}, + Data: u256(value / 2), + }, + } + if value == 0 { + // no ETH transfer logs expected with zero value + expLogs = []*types.Log{expLogs[1], expLogs[3]} + } + for i, log := range expLogs { + log.BlockNumber = 1 + log.BlockHash = blockHash + log.BlockTimestamp = 10 + log.TxIndex = 0 + log.TxHash = txHash + log.Index = uint(i) + } + + if len(expLogs) != len(receipts[0][0].Logs) { + t.Fatalf("Incorrect number of logs (expected: %d, got: %d)", len(expLogs), len(receipts[0][0].Logs)) + } + for i, log := range receipts[0][0].Logs { + if !reflect.DeepEqual(expLogs[i], log) { + t.Fatalf("Incorrect log at index %d (expected: %v, got: %v)", i, expLogs[i], log) + } + } +} diff --git a/core/events.go b/core/events.go index ef0de32426..ed853f1790 100644 --- a/core/events.go +++ b/core/events.go @@ -17,6 +17,9 @@ package core import ( + "time" + + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" ) @@ -35,3 +38,10 @@ type ChainEvent struct { type ChainHeadEvent struct { Header *types.Header } + +// NewPayloadEvent is posted when engine_newPayloadVX processes a block. +type NewPayloadEvent struct { + Hash common.Hash + Number uint64 + ProcessingTime time.Duration +} diff --git a/core/evm.go b/core/evm.go index 0a827aa56b..49cefc7b66 100644 --- a/core/evm.go +++ b/core/evm.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/params" "github.com/holiman/uint256" ) @@ -44,6 +45,7 @@ func NewEVMBlockContext(header *types.Header, chain ChainContext, author *common baseFee *big.Int blobBaseFee *big.Int random *common.Hash + slotNum uint64 ) // If we don't have an explicit author (i.e. not mining), extract from the header @@ -61,11 +63,15 @@ func NewEVMBlockContext(header *types.Header, chain ChainContext, author *common if header.Difficulty.Sign() == 0 { random = &header.MixDigest } + if header.SlotNumber != nil { + slotNum = *header.SlotNumber + } arbOsVersion := types.DeserializeHeaderExtraInformation(header).ArbOSFormatVersion if arbOsVersion > 0 { difficultyHash := common.BigToHash(header.Difficulty) random = &difficultyHash } + return vm.BlockContext{ CanTransfer: CanTransfer, Transfer: Transfer, @@ -78,6 +84,7 @@ func NewEVMBlockContext(header *types.Header, chain ChainContext, author *common BlobBaseFee: blobBaseFee, GasLimit: header.GasLimit, Random: random, + SlotNum: slotNum, ArbOSVersion: arbOsVersion, } } @@ -86,12 +93,9 @@ func NewEVMBlockContext(header *types.Header, chain ChainContext, author *common func NewEVMTxContext(msg *Message) vm.TxContext { ctx := vm.TxContext{ Origin: msg.From, - GasPrice: new(big.Int).Set(msg.GasPrice), + GasPrice: uint256.MustFromBig(msg.GasPrice), BlobHashes: msg.BlobHashes, } - if msg.BlobGasFeeCap != nil { - ctx.BlobFeeCap = new(big.Int).Set(msg.BlobGasFeeCap) - } return ctx } @@ -141,7 +145,10 @@ func CanTransfer(db vm.StateDB, addr common.Address, amount *uint256.Int) bool { } // Transfer subtracts amount from sender and adds amount to recipient using the given Db -func Transfer(db vm.StateDB, sender, recipient common.Address, amount *uint256.Int) { +func Transfer(db vm.StateDB, sender, recipient common.Address, amount *uint256.Int, rules *params.Rules) { db.SubBalance(sender, amount, tracing.BalanceChangeTransfer) db.AddBalance(recipient, amount, tracing.BalanceChangeTransfer) + if rules.IsAmsterdam && !amount.IsZero() && sender != recipient { + db.AddLog(types.EthTransferLog(sender, recipient, amount)) + } } diff --git a/core/filtermaps/checkpoints_mainnet.json b/core/filtermaps/checkpoints_mainnet.json index 2ea065ddb7..795967405d 100644 --- a/core/filtermaps/checkpoints_mainnet.json +++ b/core/filtermaps/checkpoints_mainnet.json @@ -288,5 +288,32 @@ {"blockNumber": 22958100, "blockId": "0xe38e0ff7b0c4065ca42ea577bc32f2566ca46f2ddeedcc4bc1f8fb00e7f26329", "firstIndex": 19260242424}, {"blockNumber": 22988600, "blockId": "0x04ca74758b22e0ea54b8c992022ff21c16a2af9c45144c3b0f80de921a7eee82", "firstIndex": 19327351273}, {"blockNumber": 23018392, "blockId": "0x61cc979b00bc97b48356f986a5b9ec997d674bc904c2a2e4b0f17de08e50b3bb", "firstIndex": 19394459627}, -{"blockNumber": 23048524, "blockId": "0x489de15d95739ede4ab15e8b5151d80d4dc85ae10e7be800b1a4723094a678df", "firstIndex": 19461570073} +{"blockNumber": 23048524, "blockId": "0x489de15d95739ede4ab15e8b5151d80d4dc85ae10e7be800b1a4723094a678df", "firstIndex": 19461570073}, +{"blockNumber": 23078983, "blockId": "0xd64da4fe45a0a349101b8ce1a6336fb099d8b00cc274d0eb59356e134190b8f2", "firstIndex": 19528679292}, +{"blockNumber": 23109224, "blockId": "0xbb6c29f91820fcf6caef881bdfe61eb690f9796f8f139c56eaf27aa601fe5ed2", "firstIndex": 19595787850}, +{"blockNumber": 23136690, "blockId": "0xc1915739edff731d469ec1500ad05102c0d68cc1ef062411d2e9741b8ebdc571", "firstIndex": 19662896969}, +{"blockNumber": 23164324, "blockId": "0x97d0078f2a22a8fbde4660d5f11846d00a669c606e42291df55e863190914a9f", "firstIndex": 19730004997}, +{"blockNumber": 23192975, "blockId": "0x8463749ec09f55fccdd962498f93d6468d744faedc35da8097642cd4609e08f2", "firstIndex": 19797111604}, +{"blockNumber": 23219805, "blockId": "0x76ca51d01b5724f1b0f6c3acb50dfcdb00e1648a475dd7599174b301c25e5517", "firstIndex": 19864222312}, +{"blockNumber": 23236331, "blockId": "0x388f9e36b3ec2120d2b4adfdcdb0c0b8de139eef107e327670dd77fc581c2c5f", "firstIndex": 19931331831}, +{"blockNumber": 23260849, "blockId": "0xe4a467164dbc8f9beebf0c8478a9e7ee16dfce65e2da423b1048601831222ba7", "firstIndex": 19998441286}, +{"blockNumber": 23282795, "blockId": "0x74ad210aa1bfdd4bcf61298ebff622da758c36c38d62d001f2440d09e73ef6c7", "firstIndex": 20065548083}, +{"blockNumber": 23300759, "blockId": "0xa405f5ea21a5207d3cde718a1e3fb7f0ce3dd87ac6040a0db52da0e9488e63f6", "firstIndex": 20132623334}, +{"blockNumber": 23319772, "blockId": "0xafae645dd057af450eddf69c7604bde0136524abc5b7d6697426427ef2d30724", "firstIndex": 20199767615}, +{"blockNumber": 23342113, "blockId": "0x8482c4be13294cfd862d500051c8b4efb95b50f4a79717da6aeabb2a6ff3e199", "firstIndex": 20266874986}, +{"blockNumber": 23366974, "blockId": "0xf6047cafea5da7aaf20691df700d759fe84f5aa2176d9dd42c6ae138899a29ea", "firstIndex": 20333983351}, +{"blockNumber": 23397579, "blockId": "0xe94815fe0278659a26e15b023c0c3877abf6f1265710c5dfddf161ee8af01b40", "firstIndex": 20401094335}, +{"blockNumber": 23425940, "blockId": "0xa98d6d48b93d9ef848c340b47cf9d191f36e76375dd747d8adcb39d72251822d", "firstIndex": 20468200506}, +{"blockNumber": 23452402, "blockId": "0x5f26ff308c0883a04f4a92411bcd05e50538d95a96532452f43b71fde4af908d", "firstIndex": 20535311178}, +{"blockNumber": 23478884, "blockId": "0x23a81186c21218f7034ee81dcd72db71edcbc0e97da26703b8026b6d4a81f693", "firstIndex": 20602384106}, +{"blockNumber": 23507136, "blockId": "0xb1d4a2bd836c5d78b678f0bfffd651bdfeae8b7965639b640f8745193b142323", "firstIndex": 20669529935}, +{"blockNumber": 23535463, "blockId": "0x1886a719a6376352753242ec5093c434938b9a90356facdbdaafd2670da97d82", "firstIndex": 20736637374}, +{"blockNumber": 23564278, "blockId": "0xcdc5a1349f45430771fa2080555f20b80b2d84f575728dbb8bd3b633fbb0a00b", "firstIndex": 20803747399}, +{"blockNumber": 23594711, "blockId": "0xbbf41e407367feeb865b305e38aee8bef84ba62b66efbd6fcd3b16a5d50cc055", "firstIndex": 20870854523}, +{"blockNumber": 23627417, "blockId": "0x1b3c6bd39c5aa7c73101e7e946964b3d7afecf3f253efda8de5610dd99cbee82", "firstIndex": 20937963176}, +{"blockNumber": 23660347, "blockId": "0xadc8ac88c281f50c3aaf1d08149100e8f91a16bbe09b28ac86665bcea681d41b", "firstIndex": 21005072865}, +{"blockNumber": 23692228, "blockId": "0xeb4040468161d9d5b6863eb62e4f21f72a1944f910ecfa95ee6a9dbc39e92ef0", "firstIndex": 21072181753}, +{"blockNumber": 23722331, "blockId": "0xdfc9be1b43488148868da0c8ac56e99e0ffde18bcc418755c4765e745fe74024", "firstIndex": 21139291342}, +{"blockNumber": 23752866, "blockId": "0xb9cf0dfee429a1450f5fb7a237729cf7d0c49e7e35c00dc3709f197c091f8b39", "firstIndex": 21206399901}, +{"blockNumber": 23784485, "blockId": "0x92f5f119078b4ef7ad99273e2ae6f874cfb663e80d741a821e0bb7c25c0369c7", "firstIndex": 21273505141} ] diff --git a/core/filtermaps/filtermaps.go b/core/filtermaps/filtermaps.go index fede54df57..f6b1ef26d0 100644 --- a/core/filtermaps/filtermaps.go +++ b/core/filtermaps/filtermaps.go @@ -434,7 +434,7 @@ func (f *FilterMaps) safeDeleteWithLogs(deleteFn func(db ethdb.KeyValueStore, ha lastLogPrinted = start ) switch err := deleteFn(f.db, f.hashScheme, func(deleted bool) bool { - if deleted && !logPrinted || time.Since(lastLogPrinted) > time.Second*10 { + if deleted && (!logPrinted || time.Since(lastLogPrinted) > time.Second*10) { log.Info(action+" in progress...", "elapsed", common.PrettyDuration(time.Since(start))) logPrinted, lastLogPrinted = true, time.Now() } diff --git a/core/gaspool.go b/core/gaspool.go index 767222674f..14f5abd93c 100644 --- a/core/gaspool.go +++ b/core/gaspool.go @@ -21,39 +21,87 @@ import ( "math" ) -// GasPool tracks the amount of gas available during execution of the transactions -// in a block. The zero value is a pool with zero gas available. -type GasPool uint64 - -// AddGas makes gas available for execution. -func (gp *GasPool) AddGas(amount uint64) *GasPool { - if uint64(*gp) > math.MaxUint64-amount { - panic("gas pool pushed above uint64") +// GasPool tracks the amount of gas available for transaction execution +// within a block, along with the cumulative gas consumed. +type GasPool struct { + remaining uint64 + initial uint64 + cumulativeUsed uint64 +} + +// NewGasPool initializes the gasPool with the given amount. +func NewGasPool(amount uint64) *GasPool { + return &GasPool{ + remaining: amount, + initial: amount, } - *(*uint64)(gp) += amount - return gp } // SubGas deducts the given amount from the pool if enough gas is // available and returns an error otherwise. func (gp *GasPool) SubGas(amount uint64) error { - if uint64(*gp) < amount { + if gp.remaining < amount { return ErrGasLimitReached } - *(*uint64)(gp) -= amount + gp.remaining -= amount + return nil +} + +// ReturnGas adds the refunded gas back to the pool and updates +// the cumulative gas usage accordingly. +func (gp *GasPool) ReturnGas(returned uint64, gasUsed uint64) error { + if gp.remaining > math.MaxUint64-returned { + return fmt.Errorf("%w: remaining: %d, returned: %d", ErrGasLimitOverflow, gp.remaining, returned) + } + // The returned gas calculation differs across forks. + // + // - Pre-Amsterdam: + // returned = purchased - remaining (refund included) + // + // - Post-Amsterdam: + // returned = purchased - gasUsed (refund excluded) + gp.remaining += returned + + // gasUsed = max(txGasUsed - gasRefund, calldataFloorGasCost) + // regardless of Amsterdam is activated or not. + gp.cumulativeUsed += gasUsed return nil } // Gas returns the amount of gas remaining in the pool. func (gp *GasPool) Gas() uint64 { - return uint64(*gp) + return gp.remaining +} + +// CumulativeUsed returns the amount of cumulative consumed gas (refunded included). +func (gp *GasPool) CumulativeUsed() uint64 { + return gp.cumulativeUsed +} + +// Used returns the amount of consumed gas. +func (gp *GasPool) Used() uint64 { + if gp.initial < gp.remaining { + panic("gas used underflow") + } + return gp.initial - gp.remaining +} + +// Snapshot returns the deep-copied object as the snapshot. +func (gp *GasPool) Snapshot() *GasPool { + return &GasPool{ + initial: gp.initial, + remaining: gp.remaining, + cumulativeUsed: gp.cumulativeUsed, + } } -// SetGas sets the amount of gas with the provided number. -func (gp *GasPool) SetGas(gas uint64) { - *(*uint64)(gp) = gas +// Set sets the content of gasPool with the provided one. +func (gp *GasPool) Set(other *GasPool) { + gp.initial = other.initial + gp.remaining = other.remaining + gp.cumulativeUsed = other.cumulativeUsed } func (gp *GasPool) String() string { - return fmt.Sprintf("%d", *gp) + return fmt.Sprintf("initial: %d, remaining: %d, cumulative used: %d", gp.initial, gp.remaining, gp.cumulativeUsed) } diff --git a/core/gen_genesis.go b/core/gen_genesis.go index df5dcb4ae3..8630412a25 100644 --- a/core/gen_genesis.go +++ b/core/gen_genesis.go @@ -34,6 +34,7 @@ func (g Genesis) MarshalJSON() ([]byte, error) { BaseFee *math.HexOrDecimal256 `json:"baseFeePerGas"` ExcessBlobGas *math.HexOrDecimal64 `json:"excessBlobGas"` BlobGasUsed *math.HexOrDecimal64 `json:"blobGasUsed"` + SlotNumber *uint64 `json:"slotNumber"` SerializedChainConfig string `json:"serializedChainConfig,omitempty"` ArbOSInit *params.ArbOSInit `json:"arbOSInit,omitempty"` } @@ -58,6 +59,7 @@ func (g Genesis) MarshalJSON() ([]byte, error) { enc.BaseFee = (*math.HexOrDecimal256)(g.BaseFee) enc.ExcessBlobGas = (*math.HexOrDecimal64)(g.ExcessBlobGas) enc.BlobGasUsed = (*math.HexOrDecimal64)(g.BlobGasUsed) + enc.SlotNumber = g.SlotNumber enc.SerializedChainConfig = g.SerializedChainConfig enc.ArbOSInit = g.ArbOSInit return json.Marshal(&enc) @@ -81,6 +83,7 @@ func (g *Genesis) UnmarshalJSON(input []byte) error { BaseFee *math.HexOrDecimal256 `json:"baseFeePerGas"` ExcessBlobGas *math.HexOrDecimal64 `json:"excessBlobGas"` BlobGasUsed *math.HexOrDecimal64 `json:"blobGasUsed"` + SlotNumber *uint64 `json:"slotNumber"` SerializedChainConfig *string `json:"serializedChainConfig,omitempty"` ArbOSInit *params.ArbOSInit `json:"arbOSInit,omitempty"` } @@ -139,6 +142,9 @@ func (g *Genesis) UnmarshalJSON(input []byte) error { if dec.BlobGasUsed != nil { g.BlobGasUsed = (*uint64)(dec.BlobGasUsed) } + if dec.SlotNumber != nil { + g.SlotNumber = dec.SlotNumber + } if dec.SerializedChainConfig != nil { g.SerializedChainConfig = *dec.SerializedChainConfig } diff --git a/core/genesis.go b/core/genesis.go index a31d2eae56..52969c8325 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -74,6 +74,7 @@ type Genesis struct { BaseFee *big.Int `json:"baseFeePerGas"` // EIP-1559 ExcessBlobGas *uint64 `json:"excessBlobGas"` // EIP-4844 BlobGasUsed *uint64 `json:"blobGasUsed"` // EIP-4844 + SlotNumber *uint64 `json:"slotNumber"` // EIP-7843 // Arbitrum SerializedChainConfig string `json:"serializedChainConfig,omitempty"` @@ -141,6 +142,7 @@ func ReadGenesis(db ethdb.Database) (*Genesis, error) { genesis.BaseFee = genesisHeader.BaseFee genesis.ExcessBlobGas = genesisHeader.ExcessBlobGas genesis.BlobGasUsed = genesisHeader.BlobGasUsed + genesis.SlotNumber = genesisHeader.SlotNumber return &genesis, nil } @@ -183,7 +185,7 @@ func hashAlloc(ga *types.GenesisAlloc, isVerkle bool) (common.Hash, error) { // flushAlloc is very similar with hash, but the main difference is all the // generated states will be persisted into the given database. -func flushAlloc(ga *types.GenesisAlloc, triedb *triedb.Database) (common.Hash, error) { +func flushAlloc(ga *types.GenesisAlloc, triedb *triedb.Database, tracer *tracing.Hooks) (common.Hash, error) { emptyRoot := types.EmptyRootHash if triedb.IsVerkle() { emptyRoot = types.EmptyVerkleHash @@ -204,12 +206,28 @@ func flushAlloc(ga *types.GenesisAlloc, triedb *triedb.Database) (common.Hash, e statedb.SetState(addr, key, value) } } - root, err := statedb.Commit(0, false, false) - if err != nil { - return common.Hash{}, err + + var root common.Hash + if tracer != nil && tracer.OnStateUpdate != nil { + r, update, err := statedb.CommitWithUpdate(0, false, false) + if err != nil { + return common.Hash{}, err + } + trUpdate, err := update.ToTracingUpdate() + if err != nil { + return common.Hash{}, err + } + tracer.OnStateUpdate(trUpdate) + root = r + } else { + root, err = statedb.Commit(0, false, false) + if err != nil { + return common.Hash{}, err + } } + // Commit newly generated states into disk if it's not empty. - if root != types.EmptyRootHash { + if root != emptyRoot { if err := triedb.Commit(root, true); err != nil { return common.Hash{}, err } @@ -315,10 +333,10 @@ func (o *ChainOverrides) apply(cfg *params.ChainConfig) error { // specify a fork block below the local head block). In case of a conflict, the // error is a *params.ConfigCompatError and the new, unwritten config is returned. func SetupGenesisBlock(db ethdb.Database, triedb *triedb.Database, genesis *Genesis) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) { - return SetupGenesisBlockWithOverride(db, triedb, genesis, nil) + return SetupGenesisBlockWithOverride(db, triedb, genesis, nil, nil) } -func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, genesis *Genesis, overrides *ChainOverrides) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) { +func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, genesis *Genesis, overrides *ChainOverrides, tracer *tracing.Hooks) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) { // Copy the genesis, so we can operate on a copy. genesis = genesis.copy() // Sanitize the supplied genesis, ensuring it has the associated chain @@ -339,7 +357,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, g return nil, common.Hash{}, nil, err } - block, err := genesis.Commit(db, triedb) + block, err := genesis.Commit(db, triedb, tracer) if err != nil { return nil, common.Hash{}, nil, err } @@ -367,7 +385,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, g if hash := genesis.ToBlock().Hash(); hash != ghash { return nil, common.Hash{}, nil, &GenesisMismatchError{ghash, hash} } - block, err := genesis.Commit(db, triedb) + block, err := genesis.Commit(db, triedb, tracer) if err != nil { return nil, common.Hash{}, nil, err } @@ -551,13 +569,19 @@ func (g *Genesis) toBlockWithRoot(root common.Hash) *types.Block { if conf.IsPrague(num, g.Timestamp, arbosVersion) { head.RequestsHash = &types.EmptyRequestsHash } + if conf.IsAmsterdam(num, g.Timestamp) { + head.SlotNumber = g.SlotNumber + if head.SlotNumber == nil { + head.SlotNumber = new(uint64) + } + } } return types.NewBlock(head, &types.Body{Withdrawals: withdrawals}, nil, trie.NewStackTrie(nil)) } // Commit writes the block and state of a genesis specification to the database. // The block is committed as the canonical head block. -func (g *Genesis) Commit(db ethdb.Database, triedb *triedb.Database) (*types.Block, error) { +func (g *Genesis) Commit(db ethdb.Database, triedb *triedb.Database, tracer *tracing.Hooks) (*types.Block, error) { if g.Number != 0 { return nil, errors.New("can't commit genesis block with number > 0") } @@ -572,7 +596,7 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *triedb.Database) (*types.Blo return nil, errors.New("can't start clique chain without signers") } // flush the data to disk and compute the state root - root, err := flushAlloc(&g.Alloc, triedb) + root, err := flushAlloc(&g.Alloc, triedb, tracer) if err != nil { return nil, err } @@ -602,7 +626,7 @@ func WriteHeadBlock(batch ethdb.Batch, block *types.Block) { // MustCommit writes the genesis block and state to db, panicking on error. // The block is committed as the canonical head block. func (g *Genesis) MustCommit(db ethdb.Database, triedb *triedb.Database) *types.Block { - block, err := g.Commit(db, triedb) + block, err := g.Commit(db, triedb, nil) if err != nil { panic(err) } diff --git a/core/genesis_test.go b/core/genesis_test.go index abc623d00f..d87cd3c226 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -88,7 +88,7 @@ func testSetupGenesis(t *testing.T, scheme string) { name: "custom block in DB, genesis == nil", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) { tdb := triedb.NewDatabase(db, newDbConfig(scheme)) - customg.Commit(db, tdb) + customg.Commit(db, tdb, nil) return SetupGenesisBlock(db, tdb, nil) }, wantHash: customghash, @@ -98,7 +98,7 @@ func testSetupGenesis(t *testing.T, scheme string) { name: "custom block in DB, genesis == sepolia", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) { tdb := triedb.NewDatabase(db, newDbConfig(scheme)) - customg.Commit(db, tdb) + customg.Commit(db, tdb, nil) return SetupGenesisBlock(db, tdb, DefaultSepoliaGenesisBlock()) }, wantErr: &GenesisMismatchError{Stored: customghash, New: params.SepoliaGenesisHash}, @@ -107,7 +107,7 @@ func testSetupGenesis(t *testing.T, scheme string) { name: "custom block in DB, genesis == hoodi", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) { tdb := triedb.NewDatabase(db, newDbConfig(scheme)) - customg.Commit(db, tdb) + customg.Commit(db, tdb, nil) return SetupGenesisBlock(db, tdb, DefaultHoodiGenesisBlock()) }, wantErr: &GenesisMismatchError{Stored: customghash, New: params.HoodiGenesisHash}, @@ -116,7 +116,7 @@ func testSetupGenesis(t *testing.T, scheme string) { name: "compatible config in DB", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) { tdb := triedb.NewDatabase(db, newDbConfig(scheme)) - oldcustomg.Commit(db, tdb) + oldcustomg.Commit(db, tdb, nil) return SetupGenesisBlock(db, tdb, &customg) }, wantHash: customghash, @@ -128,7 +128,7 @@ func testSetupGenesis(t *testing.T, scheme string) { // Commit the 'old' genesis block with Homestead transition at #2. // Advance to block #4, past the homestead transition block of customg. tdb := triedb.NewDatabase(db, newDbConfig(scheme)) - oldcustomg.Commit(db, tdb) + oldcustomg.Commit(db, tdb, nil) bc, _ := NewBlockChain(db, nil, &oldcustomg, ethash.NewFullFaker(), DefaultConfig().WithStateScheme(scheme)) defer bc.Stop() @@ -308,7 +308,7 @@ func TestVerkleGenesisCommit(t *testing.T) { }, } - expected := common.FromHex("018d20eebb130b5e2b796465fe36aafab650650729a92435aec071bf2386f080") + expected := common.FromHex("1fd154971d9a386c4ec75fe7138c17efb569bfc2962e46e94a376ba997e3fadc") got := genesis.ToBlock().Root().Bytes() if !bytes.Equal(got, expected) { t.Fatalf("invalid genesis state root, expected %x, got %x", expected, got) diff --git a/core/headerchain_test.go b/core/headerchain_test.go index b51fb8f226..dba04e2cf2 100644 --- a/core/headerchain_test.go +++ b/core/headerchain_test.go @@ -69,7 +69,7 @@ func TestHeaderInsertion(t *testing.T) { db = rawdb.NewMemoryDatabase() gspec = &Genesis{BaseFee: big.NewInt(params.InitialBaseFee), Config: params.AllEthashProtocolChanges} ) - gspec.Commit(db, triedb.NewDatabase(db, nil)) + gspec.Commit(db, triedb.NewDatabase(db, nil), nil) hc, err := NewHeaderChain(db, gspec.Config, ethash.NewFaker(), func() bool { return false }) if err != nil { t.Fatal(err) diff --git a/core/history/historymode.go b/core/history/historymode.go index e735222d37..1adfe014b2 100644 --- a/core/history/historymode.go +++ b/core/history/historymode.go @@ -32,10 +32,13 @@ const ( // KeepPostMerge sets the history pruning point to the merge activation block. KeepPostMerge + + // KeepPostPrague sets the history pruning point to the Prague (Pectra) activation block. + KeepPostPrague ) func (m HistoryMode) IsValid() bool { - return m <= KeepPostMerge + return m <= KeepPostPrague } func (m HistoryMode) String() string { @@ -44,6 +47,8 @@ func (m HistoryMode) String() string { return "all" case KeepPostMerge: return "postmerge" + case KeepPostPrague: + return "postprague" default: return fmt.Sprintf("invalid HistoryMode(%d)", m) } @@ -64,33 +69,73 @@ func (m *HistoryMode) UnmarshalText(text []byte) error { *m = KeepAll case "postmerge": *m = KeepPostMerge + case "postprague": + *m = KeepPostPrague default: - return fmt.Errorf(`unknown sync mode %q, want "all" or "postmerge"`, text) + return fmt.Errorf(`unknown history mode %q, want "all", "postmerge", or "postprague"`, text) } return nil } +// PrunePoint identifies a specific block for history pruning. type PrunePoint struct { BlockNumber uint64 BlockHash common.Hash } -// PrunePoints the pre-defined history pruning cutoff blocks for known networks. -// They point to the first post-merge block. Any pruning should truncate *up to* but excluding -// given block. -var PrunePoints = map[common.Hash]*PrunePoint{ - // mainnet - params.MainnetGenesisHash: { - BlockNumber: 15537393, - BlockHash: common.HexToHash("0x55b11b918355b1ef9c5db810302ebad0bf2544255b530cdce90674d5887bb286"), +// staticPrunePoints contains the pre-defined history pruning cutoff blocks for +// known networks, keyed by history mode and genesis hash. They point to the first +// block after the respective fork. Any pruning should truncate *up to* but +// excluding the given block. +var staticPrunePoints = map[HistoryMode]map[common.Hash]*PrunePoint{ + KeepPostMerge: { + params.MainnetGenesisHash: { + BlockNumber: 15537393, + BlockHash: common.HexToHash("0x55b11b918355b1ef9c5db810302ebad0bf2544255b530cdce90674d5887bb286"), + }, + params.SepoliaGenesisHash: { + BlockNumber: 1450409, + BlockHash: common.HexToHash("0x229f6b18ca1552f1d5146deceb5387333f40dc6275aebee3f2c5c4ece07d02db"), + }, }, - // sepolia - params.SepoliaGenesisHash: { - BlockNumber: 1450409, - BlockHash: common.HexToHash("0x229f6b18ca1552f1d5146deceb5387333f40dc6275aebee3f2c5c4ece07d02db"), + KeepPostPrague: { + params.MainnetGenesisHash: { + BlockNumber: 22431084, + BlockHash: common.HexToHash("0x50c8cab760b2948349c590461b166773c45d8f4858cccf5a43025ab2960152e8"), + }, + params.SepoliaGenesisHash: { + BlockNumber: 7836331, + BlockHash: common.HexToHash("0xe6571beb68bf24dbd8a6ba354518996920c55a3f8d8fdca423e391b8ad071f22"), + }, }, } +// HistoryPolicy describes the configured history pruning strategy. It captures +// user intent as opposed to the actual DB state. +type HistoryPolicy struct { + Mode HistoryMode + // Static prune point for PostMerge/PostPrague, nil otherwise. + Target *PrunePoint +} + +// NewPolicy constructs a HistoryPolicy from the given mode and genesis hash. +func NewPolicy(mode HistoryMode, genesisHash common.Hash) (HistoryPolicy, error) { + switch mode { + case KeepAll: + return HistoryPolicy{Mode: KeepAll}, nil + + case KeepPostMerge, KeepPostPrague: + point := staticPrunePoints[mode][genesisHash] + if point == nil { + return HistoryPolicy{}, fmt.Errorf("%s history pruning not available for network %s", mode, genesisHash.Hex()) + } + return HistoryPolicy{Mode: mode, Target: point}, nil + + default: + return HistoryPolicy{}, fmt.Errorf("invalid history mode: %d", mode) + } +} + // PrunedHistoryError is returned by APIs when the requested history is pruned. type PrunedHistoryError struct{} diff --git a/core/history/historymode_test.go b/core/history/historymode_test.go new file mode 100644 index 0000000000..87eae188dd --- /dev/null +++ b/core/history/historymode_test.go @@ -0,0 +1,58 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package history + +import ( + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/params" +) + +func TestNewPolicy(t *testing.T) { + // KeepAll: no target. + p, err := NewPolicy(KeepAll, params.MainnetGenesisHash) + if err != nil { + t.Fatalf("KeepAll: %v", err) + } + if p.Mode != KeepAll || p.Target != nil { + t.Errorf("KeepAll: unexpected policy %+v", p) + } + + // PostMerge: resolves known mainnet prune point. + p, err = NewPolicy(KeepPostMerge, params.MainnetGenesisHash) + if err != nil { + t.Fatalf("PostMerge: %v", err) + } + if p.Target == nil || p.Target.BlockNumber != 15537393 { + t.Errorf("PostMerge: unexpected target %+v", p.Target) + } + + // PostPrague: resolves known mainnet prune point. + p, err = NewPolicy(KeepPostPrague, params.MainnetGenesisHash) + if err != nil { + t.Fatalf("PostPrague: %v", err) + } + if p.Target == nil || p.Target.BlockNumber != 22431084 { + t.Errorf("PostPrague: unexpected target %+v", p.Target) + } + + // PostMerge on unknown network: error. + if _, err = NewPolicy(KeepPostMerge, common.HexToHash("0xdeadbeef")); err == nil { + t.Fatal("PostMerge unknown network: expected error") + } +} diff --git a/core/overlay/state_transition.go b/core/overlay/state_transition.go index 67ca0f9671..a52d9139c9 100644 --- a/core/overlay/state_transition.go +++ b/core/overlay/state_transition.go @@ -97,7 +97,7 @@ func LoadTransitionState(db ethdb.KeyValueReader, root common.Hash, isVerkle boo // Initialize the first transition state, with the "ended" // field set to true if the database was created // as a verkle database. - log.Debug("no transition state found, starting fresh", "is verkle", db) + log.Debug("no transition state found, starting fresh", "verkle", isVerkle) // Start with a fresh state ts = &TransitionState{Ended: isVerkle} diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 5235d01bc3..f80b9b2bd2 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -83,65 +83,6 @@ type NumberHash struct { Hash common.Hash } -// ReadAllHashesInRange retrieves all the hashes assigned to blocks at certain -// heights, both canonical and reorged forks included. -// This method considers both limits to be _inclusive_. -func ReadAllHashesInRange(db ethdb.Iteratee, first, last uint64) []*NumberHash { - var ( - start = encodeBlockNumber(first) - keyLength = len(headerPrefix) + 8 + 32 - hashes = make([]*NumberHash, 0, 1+last-first) - it = db.NewIterator(headerPrefix, start) - ) - defer it.Release() - for it.Next() { - key := it.Key() - if len(key) != keyLength { - continue - } - num := binary.BigEndian.Uint64(key[len(headerPrefix) : len(headerPrefix)+8]) - if num > last { - break - } - hash := common.BytesToHash(key[len(key)-32:]) - hashes = append(hashes, &NumberHash{num, hash}) - } - return hashes -} - -// ReadAllCanonicalHashes retrieves all canonical number and hash mappings at the -// certain chain range. If the accumulated entries reaches the given threshold, -// abort the iteration and return the semi-finish result. -func ReadAllCanonicalHashes(db ethdb.Iteratee, from uint64, to uint64, limit int) ([]uint64, []common.Hash) { - // Short circuit if the limit is 0. - if limit == 0 { - return nil, nil - } - var ( - numbers []uint64 - hashes []common.Hash - ) - // Construct the key prefix of start point. - start, end := headerHashKey(from), headerHashKey(to) - it := db.NewIterator(nil, start) - defer it.Release() - - for it.Next() { - if bytes.Compare(it.Key(), end) >= 0 { - break - } - if key := it.Key(); len(key) == len(headerPrefix)+8+1 && bytes.Equal(key[len(key)-1:], headerHashSuffix) { - numbers = append(numbers, binary.BigEndian.Uint64(key[len(headerPrefix):len(headerPrefix)+8])) - hashes = append(hashes, common.BytesToHash(it.Value())) - // If the accumulated entries reaches the limit threshold, return. - if len(numbers) >= limit { - break - } - } - } - return numbers, hashes -} - // ReadHeaderNumber returns the header number assigned to a hash. func ReadHeaderNumber(db ethdb.KeyValueReader, hash common.Hash) (uint64, bool) { data, _ := db.Get(headerNumberKey(hash)) @@ -921,40 +862,6 @@ func WriteBadBlock(db ethdb.KeyValueStore, block *types.Block) { } } -// DeleteBadBlocks deletes all the bad blocks from the database -func DeleteBadBlocks(db ethdb.KeyValueWriter) { - if err := db.Delete(badBlockKey); err != nil { - log.Crit("Failed to delete bad blocks", "err", err) - } -} - -// FindCommonAncestor returns the last common ancestor of two block headers -func FindCommonAncestor(db ethdb.Reader, a, b *types.Header) *types.Header { - for bn := b.Number.Uint64(); a.Number.Uint64() > bn; { - a = ReadHeader(db, a.ParentHash, a.Number.Uint64()-1) - if a == nil { - return nil - } - } - for an := a.Number.Uint64(); an < b.Number.Uint64(); { - b = ReadHeader(db, b.ParentHash, b.Number.Uint64()-1) - if b == nil { - return nil - } - } - for a.Hash() != b.Hash() { - a = ReadHeader(db, a.ParentHash, a.Number.Uint64()-1) - if a == nil { - return nil - } - b = ReadHeader(db, b.ParentHash, b.Number.Uint64()-1) - if b == nil { - return nil - } - } - return a -} - // ReadHeadHeader returns the current canonical head header. func ReadHeadHeader(db ethdb.Reader) *types.Header { headHeaderHash := ReadHeadHeaderHash(db) diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index cc2431ca0d..da6e618141 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -23,7 +23,6 @@ import ( "math/big" "math/rand" "os" - "reflect" "testing" "github.com/ethereum/go-ethereum/arbcrypto" @@ -53,10 +52,7 @@ func TestHeaderStorage(t *testing.T) { if entry := ReadHeaderRLP(db, header.Hash(), header.Number.Uint64()); entry == nil { t.Fatalf("Stored header RLP not found") } else { - hasher := arbcrypto.NewLegacyKeccak256() - hasher.Write(entry) - - if hash := common.BytesToHash(hasher.Sum(nil)); hash != header.Hash() { + if hash := crypto.Keccak256Hash(entry); hash != header.Hash() { t.Fatalf("Retrieved RLP header mismatch: have %v, want %v", entry, header) } } @@ -91,10 +87,7 @@ func TestBodyStorage(t *testing.T) { if entry := ReadBodyRLP(db, hash, 0); entry == nil { t.Fatalf("Stored body RLP not found") } else { - hasher := arbcrypto.NewLegacyKeccak256() - hasher.Write(entry) - - if calc := common.BytesToHash(hasher.Sum(nil)); calc != hash { + if calc := crypto.Keccak256Hash(entry); calc != hash { t.Fatalf("Retrieved RLP body mismatch: have %v, want %v", entry, body) } } @@ -249,13 +242,6 @@ func TestBadBlockStorage(t *testing.T) { t.Fatalf("The bad blocks are not sorted #[%d](%d) < #[%d](%d)", i, badBlocks[i].NumberU64(), i+1, badBlocks[i+1].NumberU64()) } } - - // Delete all bad blocks - DeleteBadBlocks(db) - badBlocks = ReadAllBadBlocks(db) - if len(badBlocks) != 0 { - t.Fatalf("Failed to delete bad blocks") - } } // Tests that canonical numbers can be mapped to hashes and retrieved. @@ -516,37 +502,6 @@ func TestWriteAncientHeaderChain(t *testing.T) { } } -func TestCanonicalHashIteration(t *testing.T) { - var cases = []struct { - from, to uint64 - limit int - expect []uint64 - }{ - {1, 8, 0, nil}, - {1, 8, 1, []uint64{1}}, - {1, 8, 10, []uint64{1, 2, 3, 4, 5, 6, 7}}, - {1, 9, 10, []uint64{1, 2, 3, 4, 5, 6, 7, 8}}, - {2, 9, 10, []uint64{2, 3, 4, 5, 6, 7, 8}}, - {9, 10, 10, nil}, - } - // Test empty db iteration - db := NewMemoryDatabase() - numbers, _ := ReadAllCanonicalHashes(db, 0, 10, 10) - if len(numbers) != 0 { - t.Fatalf("No entry should be returned to iterate an empty db") - } - // Fill database with testing data. - for i := uint64(1); i <= 8; i++ { - WriteCanonicalHash(db, common.Hash{}, i) - } - for i, c := range cases { - numbers, _ := ReadAllCanonicalHashes(db, c.from, c.to, c.limit) - if !reflect.DeepEqual(numbers, c.expect) { - t.Fatalf("Case %d failed, want %v, got %v", i, c.expect, numbers) - } - } -} - func TestHashesInRange(t *testing.T) { mkHeader := func(number, seq int) *types.Header { h := types.Header{ @@ -565,18 +520,6 @@ func TestHashesInRange(t *testing.T) { total++ } } - if have, want := len(ReadAllHashesInRange(db, 10, 10)), 10; have != want { - t.Fatalf("Wrong number of hashes read, want %d, got %d", want, have) - } - if have, want := len(ReadAllHashesInRange(db, 10, 9)), 0; have != want { - t.Fatalf("Wrong number of hashes read, want %d, got %d", want, have) - } - if have, want := len(ReadAllHashesInRange(db, 0, 100)), total; have != want { - t.Fatalf("Wrong number of hashes read, want %d, got %d", want, have) - } - if have, want := len(ReadAllHashesInRange(db, 9, 10)), 9+10; have != want { - t.Fatalf("Wrong number of hashes read, want %d, got %d", want, have) - } if have, want := len(ReadAllHashes(db, 10)), 10; have != want { t.Fatalf("Wrong number of hashes read, want %d, got %d", want, have) } diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index 535178628b..827708ec9c 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -147,9 +147,6 @@ func findTxInBlockBody(blockbody rlp.RawValue, target common.Hash) (*types.Trans } txIndex := uint64(0) for iter.Next() { - if iter.Err() != nil { - return nil, 0, err - } // The preimage for the hash calculation of legacy transactions // is just their RLP encoding. For typed (EIP-2718) transactions, // which are encoded as byte arrays, the preimage is the content of @@ -177,6 +174,9 @@ func findTxInBlockBody(blockbody rlp.RawValue, target common.Hash) (*types.Trans } txIndex++ } + if iter.Err() != nil { + return nil, 0, iter.Err() + } return nil, 0, errors.New("transaction not found") } diff --git a/core/rawdb/ancient_utils.go b/core/rawdb/ancient_utils.go index b940d91040..8c6b18df08 100644 --- a/core/rawdb/ancient_utils.go +++ b/core/rawdb/ancient_utils.go @@ -34,14 +34,10 @@ type freezerInfo struct { name string // The identifier of freezer head uint64 // The number of last stored item in the freezer tail uint64 // The number of first stored item in the freezer + count uint64 // The number of stored items in the freezer sizes []tableSize // The storage size per table } -// count returns the number of stored items in the freezer. -func (info *freezerInfo) count() uint64 { - return info.head - info.tail + 1 -} - // size returns the storage size of the entire freezer. func (info *freezerInfo) size() common.StorageSize { var total common.StorageSize @@ -65,7 +61,11 @@ func inspect(name string, order map[string]freezerTableConfig, reader ethdb.Anci if err != nil { return freezerInfo{}, err } - info.head = ancients - 1 + if ancients > 0 { + info.head = ancients - 1 + } else { + info.head = 0 + } // Retrieve the number of first stored item tail, err := reader.Tail() @@ -73,6 +73,12 @@ func inspect(name string, order map[string]freezerTableConfig, reader ethdb.Anci return freezerInfo{}, err } info.tail = tail + + if ancients == 0 { + info.count = 0 + } else { + info.count = info.head - info.tail + 1 + } return info, nil } @@ -143,6 +149,8 @@ func InspectFreezerTable(ancient string, freezerName string, tableName string, s path, tables = resolveChainFreezerDir(ancient), chainFreezerTableConfigs case MerkleStateFreezerName, VerkleStateFreezerName: path, tables = filepath.Join(ancient, freezerName), stateFreezerTableConfigs + case MerkleTrienodeFreezerName, VerkleTrienodeFreezerName: + path, tables = filepath.Join(ancient, freezerName), trienodeFreezerTableConfigs default: return fmt.Errorf("unknown freezer, supported ones: %v", freezers) } @@ -158,6 +166,7 @@ func InspectFreezerTable(ancient string, freezerName string, tableName string, s if err != nil { return err } + defer table.Close() table.dumpIndexStdout(start, end) return nil } diff --git a/core/rawdb/ancienttest/testsuite.go b/core/rawdb/ancienttest/testsuite.go index 7512c1f44b..eb66645a3a 100644 --- a/core/rawdb/ancienttest/testsuite.go +++ b/core/rawdb/ancienttest/testsuite.go @@ -260,6 +260,46 @@ func basicWrite(t *testing.T, newFn func(kinds []string) ethdb.AncientStore) { if err != nil { t.Fatalf("Failed to write ancient data %v", err) } + + // Write should work after truncating from tail but over the head + db.TruncateTail(200) + head, err := db.Ancients() + if err != nil { + t.Fatalf("Failed to retrieve head ancients %v", err) + } + tail, err := db.Tail() + if err != nil { + t.Fatalf("Failed to retrieve tail ancients %v", err) + } + if head != 200 || tail != 200 { + t.Fatalf("Ancient head and tail are not expected") + } + _, err = db.ModifyAncients(func(op ethdb.AncientWriteOp) error { + offset := uint64(200) + for i := 0; i < 100; i++ { + if err := op.AppendRaw("a", offset+uint64(i), dataA[i]); err != nil { + return err + } + if err := op.AppendRaw("b", offset+uint64(i), dataB[i]); err != nil { + return err + } + } + return nil + }) + if err != nil { + t.Fatalf("Failed to write ancient data %v", err) + } + head, err = db.Ancients() + if err != nil { + t.Fatalf("Failed to retrieve head ancients %v", err) + } + tail, err = db.Tail() + if err != nil { + t.Fatalf("Failed to retrieve tail ancients %v", err) + } + if head != 300 || tail != 200 { + t.Fatalf("Ancient head and tail are not expected") + } } func nonMutable(t *testing.T, newFn func(kinds []string) ethdb.AncientStore) { diff --git a/core/rawdb/chain_iterator.go b/core/rawdb/chain_iterator.go index 3eb5500f2f..ad7ba5fc3d 100644 --- a/core/rawdb/chain_iterator.go +++ b/core/rawdb/chain_iterator.go @@ -94,6 +94,7 @@ func InitDatabaseFromFreezer(db ethdb.Database) { type blockTxHashes struct { number uint64 hashes []common.Hash + err error } // iterateTransactions iterates over all transactions in the (canon) block @@ -157,17 +158,22 @@ func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool }() for data := range rlpCh { var body types.Body + var result *blockTxHashes if err := rlp.DecodeBytes(data.rlp, &body); err != nil { log.Warn("Failed to decode block body", "block", data.number, "error", err) - return - } - var hashes []common.Hash - for _, tx := range body.Transactions { - hashes = append(hashes, tx.Hash()) - } - result := &blockTxHashes{ - hashes: hashes, - number: data.number, + result = &blockTxHashes{ + number: data.number, + err: err, + } + } else { + hashes := make([]common.Hash, len(body.Transactions)) + for i, tx := range body.Transactions { + hashes[i] = tx.Hash() + } + result = &blockTxHashes{ + hashes: hashes, + number: data.number, + } } // Feed the block to the aggregator, or abort on interrupt select { @@ -227,6 +233,10 @@ func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan // Next block available, pop it off and index it delivery := queue.PopItem() lastNum = delivery.number + if delivery.err != nil { + log.Warn("Skipping tx indexing for block with missing/corrupt body", "block", delivery.number, "error", delivery.err) + continue + } WriteTxLookupEntries(batch, delivery.number, delivery.hashes) blocks++ txs += len(delivery.hashes) @@ -322,6 +332,10 @@ func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt ch } delivery := queue.PopItem() nextNum = delivery.number + 1 + if delivery.err != nil { + log.Warn("Skipping tx unindexing for block with missing/corrupt body", "block", delivery.number, "error", delivery.err) + continue + } DeleteTxLookupEntries(batch, delivery.hashes) txs += len(delivery.hashes) blocks++ @@ -360,9 +374,9 @@ func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt ch } select { case <-interrupt: - logger("Transaction unindexing interrupted", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) + logger("Transaction unindexing interrupted", "blocks", blocks, "txs", txs, "tail", nextNum, "elapsed", common.PrettyDuration(time.Since(start))) default: - logger("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) + logger("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", nextNum, "elapsed", common.PrettyDuration(time.Since(start))) } txIndexerUnindexedBlocksHistogram.Update(int64(blocks)) txIndexerUnindexedTransactionsHistogram.Update(int64(txs)) diff --git a/core/rawdb/chain_iterator_test.go b/core/rawdb/chain_iterator_test.go index c1adfd3e80..6774842e32 100644 --- a/core/rawdb/chain_iterator_test.go +++ b/core/rawdb/chain_iterator_test.go @@ -218,6 +218,36 @@ func TestIndexTransactions(t *testing.T) { verify(0, 8, false, 8) } +func TestUnindexTransactionsMissingBody(t *testing.T) { + // Construct test chain db + chainDB := NewMemoryDatabase() + blocks, _ := initDatabaseWithTransactions(chainDB) + + // Index the entire chain. + lastBlock := blocks[len(blocks)-1].NumberU64() + IndexTransactions(chainDB, 0, lastBlock+1, nil, false, 0) + + // Prove that block 2 body exists in the database. + if raw := ReadCanonicalBodyRLP(chainDB, 2, nil); len(raw) == 0 { + t.Fatalf("Block 2 body does not exist in the database.") + } + + // Delete body for block 2. This simulates a corrupted database. + key := blockBodyKey(2, blocks[2].Hash()) + if err := chainDB.Delete(key); err != nil { + t.Fatalf("Failed to delete block body %v", err) + } + + // Unindex blocks [0, 3) + UnindexTransactions(chainDB, 0, 3, nil, false, 0) + + // Verify that tx index tail is updated to 3. + tail := ReadTxIndexTail(chainDB) + if tail == nil || *tail != 3 { + t.Fatalf("The tx index tail is wrong: got %v want %d", *tail, 3) + } +} + func TestPruneTransactionIndex(t *testing.T) { chainDB := NewMemoryDatabase() blocks, _ := initDatabaseWithTransactions(chainDB) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 78a30d01b0..7133f03633 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -35,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb/memorydb" + "github.com/ethereum/go-ethereum/internal/tablewriter" "github.com/ethereum/go-ethereum/log" "golang.org/x/sync/errgroup" ) @@ -368,6 +369,7 @@ func Open(db ethdb.KeyValueStore, opts OpenOptions) (ethdb.Database, error) { }() } return &freezerdb{ + readOnly: opts.ReadOnly, ancientRoot: opts.Ancient, KeyValueStore: db, chainFreezer: frdb, @@ -466,7 +468,8 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { filterMapBlockLV stat // Path-mode archive data - stateIndex stat + stateIndex stat + trienodeIndex stat // Verkle statistics verkleTries stat @@ -513,9 +516,9 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { bodies.add(size) case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength): receipts.add(size) - case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix): + case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix) && len(key) == (len(headerPrefix)+8+common.HashLength+len(headerTDSuffix)): tds.add(size) - case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix): + case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix) && len(key) == (len(headerPrefix)+8+len(headerHashSuffix)): numHashPairings.add(size) case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength): hashNumPairings.add(size) @@ -561,8 +564,19 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { bloomBits.add(size) // Path-based historic state indexes - case bytes.HasPrefix(key, StateHistoryIndexPrefix) && len(key) >= len(StateHistoryIndexPrefix)+common.HashLength: + case bytes.HasPrefix(key, StateHistoryAccountMetadataPrefix) && len(key) == len(StateHistoryAccountMetadataPrefix)+common.HashLength: stateIndex.add(size) + case bytes.HasPrefix(key, StateHistoryStorageMetadataPrefix) && len(key) == len(StateHistoryStorageMetadataPrefix)+2*common.HashLength: + stateIndex.add(size) + case bytes.HasPrefix(key, StateHistoryAccountBlockPrefix) && len(key) == len(StateHistoryAccountBlockPrefix)+common.HashLength+4: + stateIndex.add(size) + case bytes.HasPrefix(key, StateHistoryStorageBlockPrefix) && len(key) == len(StateHistoryStorageBlockPrefix)+2*common.HashLength+4: + stateIndex.add(size) + + case bytes.HasPrefix(key, TrienodeHistoryMetadataPrefix) && len(key) >= len(TrienodeHistoryMetadataPrefix)+common.HashLength: + trienodeIndex.add(size) + case bytes.HasPrefix(key, TrienodeHistoryBlockPrefix) && len(key) >= len(TrienodeHistoryBlockPrefix)+common.HashLength+4: + trienodeIndex.add(size) // Verkle trie data is detected, determine the sub-category case bytes.HasPrefix(key, VerklePrefix): @@ -659,12 +673,13 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { {"Key-Value store", "Path trie state lookups", stateLookups.sizeString(), stateLookups.countString()}, {"Key-Value store", "Path trie account nodes", accountTries.sizeString(), accountTries.countString()}, {"Key-Value store", "Path trie storage nodes", storageTries.sizeString(), storageTries.countString()}, - {"Key-Value store", "Path state history indexes", stateIndex.sizeString(), stateIndex.countString()}, {"Key-Value store", "Verkle trie nodes", verkleTries.sizeString(), verkleTries.countString()}, {"Key-Value store", "Verkle trie state lookups", verkleStateLookups.sizeString(), verkleStateLookups.countString()}, {"Key-Value store", "Trie preimages", preimages.sizeString(), preimages.countString()}, {"Key-Value store", "Account snapshot", accountSnaps.sizeString(), accountSnaps.countString()}, {"Key-Value store", "Storage snapshot", storageSnaps.sizeString(), storageSnaps.countString()}, + {"Key-Value store", "Historical state index", stateIndex.sizeString(), stateIndex.countString()}, + {"Key-Value store", "Historical trie index", trienodeIndex.sizeString(), trienodeIndex.countString()}, {"Key-Value store", "Beacon sync headers", beaconHeaders.sizeString(), beaconHeaders.countString()}, {"Key-Value store", "Clique snapshots", cliqueSnaps.sizeString(), cliqueSnaps.countString()}, {"Key-Value store", "Singleton metadata", metadata.sizeString(), metadata.countString()}, @@ -681,13 +696,13 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { fmt.Sprintf("Ancient store (%s)", strings.Title(ancient.name)), strings.Title(table.name), table.size.String(), - fmt.Sprintf("%d", ancient.count()), + fmt.Sprintf("%d", ancient.count), }) } total.Add(uint64(ancient.size())) } - table := newTableWriter(os.Stdout) + table := tablewriter.NewWriter(os.Stdout) table.SetHeader([]string{"Database", "Category", "Size", "Items"}) table.SetFooter([]string{"", "Total", common.StorageSize(total.Load()).String(), fmt.Sprintf("%d", count.Load())}) table.AppendBulk(stats) @@ -709,7 +724,7 @@ var knownMetadataKeys = [][]byte{ snapshotGeneratorKey, snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey, uncleanShutdownKey, badBlockKey, transitionStatusKey, skeletonSyncStatusKey, persistentStateIDKey, trieJournalKey, snapshotSyncStatusKey, snapSyncStatusFlagKey, - filterMapsRangeKey, headStateHistoryIndexKey, VerkleTransitionStatePrefix, + filterMapsRangeKey, headStateHistoryIndexKey, headTrienodeHistoryIndexKey, VerkleTransitionStatePrefix, } // printChainMetadata prints out chain metadata to stderr. diff --git a/core/rawdb/eradb/eradb.go b/core/rawdb/eradb/eradb.go index 29e658798e..d715c824ed 100644 --- a/core/rawdb/eradb/eradb.go +++ b/core/rawdb/eradb/eradb.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/era/onedb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" ) @@ -51,7 +52,7 @@ type Store struct { type fileCacheEntry struct { refcount int // reference count. This is protected by Store.mu! opened chan struct{} // signals opening of file has completed - file *era.Era // the file + file *onedb.Era // the file err error // error from opening the file } @@ -102,7 +103,7 @@ func (db *Store) Close() { // GetRawBody returns the raw body for a given block number. func (db *Store) GetRawBody(number uint64) ([]byte, error) { - epoch := number / uint64(era.MaxEra1Size) + epoch := number / uint64(era.MaxSize) entry := db.getEraByEpoch(epoch) if entry.err != nil { if errors.Is(entry.err, fs.ErrNotExist) { @@ -117,7 +118,7 @@ func (db *Store) GetRawBody(number uint64) ([]byte, error) { // GetRawReceipts returns the raw receipts for a given block number. func (db *Store) GetRawReceipts(number uint64) ([]byte, error) { - epoch := number / uint64(era.MaxEra1Size) + epoch := number / uint64(era.MaxSize) entry := db.getEraByEpoch(epoch) if entry.err != nil { if errors.Is(entry.err, fs.ErrNotExist) { @@ -249,7 +250,7 @@ func (db *Store) getCacheEntry(epoch uint64) (stat fileCacheStatus, entry *fileC } // fileOpened is called after an era file has been successfully opened. -func (db *Store) fileOpened(epoch uint64, entry *fileCacheEntry, file *era.Era) { +func (db *Store) fileOpened(epoch uint64, entry *fileCacheEntry, file *onedb.Era) { db.mu.Lock() defer db.mu.Unlock() @@ -282,7 +283,7 @@ func (db *Store) fileFailedToOpen(epoch uint64, entry *fileCacheEntry, err error entry.err = err } -func (db *Store) openEraFile(epoch uint64) (*era.Era, error) { +func (db *Store) openEraFile(epoch uint64) (*onedb.Era, error) { // File name scheme is --. glob := fmt.Sprintf("*-%05d-*.era1", epoch) matches, err := filepath.Glob(filepath.Join(db.datadir, glob)) @@ -297,16 +298,17 @@ func (db *Store) openEraFile(epoch uint64) (*era.Era, error) { } filename := matches[0] - e, err := era.Open(filename) + e, err := onedb.Open(filename) if err != nil { return nil, err } // Sanity-check start block. - if e.Start()%uint64(era.MaxEra1Size) != 0 { - return nil, fmt.Errorf("pre-merge era1 file has invalid boundary. %d %% %d != 0", e.Start(), era.MaxEra1Size) + if e.Start()%uint64(era.MaxSize) != 0 { + e.Close() + return nil, fmt.Errorf("pre-merge era1 file has invalid boundary. %d %% %d != 0", e.Start(), era.MaxSize) } log.Debug("Opened era1 file", "epoch", epoch) - return e, nil + return e.(*onedb.Era), nil } // doneWithFile signals that the caller has finished using a file. diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 67a4d47575..e00ec79cf2 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -58,7 +58,7 @@ const freezerTableSize = 2 * 1000 * 1000 * 1000 // - The in-order data ensures that disk reads are always optimized. type Freezer struct { datadir string - frozen atomic.Uint64 // Number of items already frozen + head atomic.Uint64 // Number of items stored (including items removed from tail) tail atomic.Uint64 // Number of the first stored item in the freezer // This lock synchronizes writers and the truncate operation, as well as @@ -96,12 +96,12 @@ func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize ui return nil, errSymlinkDatadir } } + // Leveldb/Pebble uses LOCK as the filelock filename. To prevent the + // name collision, we use FLOCK as the lock name. flockFile := filepath.Join(datadir, "FLOCK") if err := os.MkdirAll(filepath.Dir(flockFile), 0755); err != nil { return nil, err } - // Leveldb uses LOCK as the filelock filename. To prevent the - // name collision, we use FLOCK as the lock name. lock := NewFileLock(flockFile) tryLock := lock.TryLock if readonly { @@ -212,7 +212,7 @@ func (f *Freezer) AncientBytes(kind string, id, offset, length uint64) ([]byte, // Ancients returns the length of the frozen items. func (f *Freezer) Ancients() (uint64, error) { - return f.frozen.Load(), nil + return f.head.Load(), nil } // Tail returns the number of first stored item in the freezer. @@ -251,7 +251,7 @@ func (f *Freezer) ModifyAncients(fn func(ethdb.AncientWriteOp) error) (writeSize defer f.writeLock.Unlock() // Roll back all tables to the starting position in case of error. - prevItem := f.frozen.Load() + prevItem := f.head.Load() defer func() { if err != nil { // The write operation has failed. Go back to the previous item position. @@ -272,7 +272,7 @@ func (f *Freezer) ModifyAncients(fn func(ethdb.AncientWriteOp) error) (writeSize if err != nil { return 0, err } - f.frozen.Store(item) + f.head.Store(item) return writeSize, nil } @@ -285,7 +285,7 @@ func (f *Freezer) TruncateHead(items uint64) (uint64, error) { f.writeLock.Lock() defer f.writeLock.Unlock() - oitems := f.frozen.Load() + oitems := f.head.Load() if oitems <= items { return oitems, nil } @@ -294,7 +294,7 @@ func (f *Freezer) TruncateHead(items uint64) (uint64, error) { return 0, err } } - f.frozen.Store(items) + f.head.Store(items) return oitems, nil } @@ -319,6 +319,11 @@ func (f *Freezer) TruncateTail(tail uint64) (uint64, error) { } } f.tail.Store(tail) + + // Update the head if the requested tail exceeds the current head + if f.head.Load() < tail { + f.head.Store(tail) + } return old, nil } @@ -378,7 +383,7 @@ func (f *Freezer) validate() error { prunedTail = &tmp } - f.frozen.Store(head) + f.head.Store(head) f.tail.Store(*prunedTail) return nil } @@ -413,7 +418,7 @@ func (f *Freezer) repair() error { } } - f.frozen.Store(head) + f.head.Store(head) f.tail.Store(prunedTail) return nil } diff --git a/core/rawdb/freezer_batch.go b/core/rawdb/freezer_batch.go index 7e46e49f43..080c0720a1 100644 --- a/core/rawdb/freezer_batch.go +++ b/core/rawdb/freezer_batch.go @@ -51,12 +51,18 @@ func newFreezerBatch(f *Freezer) *freezerBatch { // Append adds an RLP-encoded item of the given kind. func (batch *freezerBatch) Append(kind string, num uint64, item interface{}) error { - return batch.tables[kind].Append(num, item) + if table := batch.tables[kind]; table != nil { + return table.Append(num, item) + } + return errUnknownTable } // AppendRaw adds an item of the given kind. func (batch *freezerBatch) AppendRaw(kind string, num uint64, item []byte) error { - return batch.tables[kind].AppendRaw(num, item) + if table := batch.tables[kind]; table != nil { + return table.AppendRaw(num, item) + } + return errUnknownTable } // reset initializes the batch. diff --git a/core/rawdb/freezer_memory.go b/core/rawdb/freezer_memory.go index 8cb4cc2006..ec6d4b22e2 100644 --- a/core/rawdb/freezer_memory.go +++ b/core/rawdb/freezer_memory.go @@ -91,6 +91,13 @@ func (t *memoryTable) truncateHead(items uint64) error { if items < t.offset { return errors.New("truncation below tail") } + for i := int(items - t.offset); i < len(t.data); i++ { + if t.size > uint64(len(t.data[i])) { + t.size -= uint64(len(t.data[i])) + } else { + t.size = 0 + } + } t.data = t.data[:items-t.offset] t.items = items return nil @@ -106,13 +113,30 @@ func (t *memoryTable) truncateTail(items uint64) error { return nil } if t.items < items { - return errors.New("truncation above head") + return t.reset(items) + } + for i := uint64(0); i < items-t.offset; i++ { + if t.size > uint64(len(t.data[i])) { + t.size -= uint64(len(t.data[i])) + } else { + t.size = 0 + } } t.data = t.data[items-t.offset:] t.offset = items return nil } +// reset clears the entire table and sets both the head and tail to the given +// value. It assumes the caller holds the lock and that tail > t.items. +func (t *memoryTable) reset(offset uint64) error { + t.size = 0 + t.data = nil + t.items = offset + t.offset = offset + return nil +} + // commit merges the given item batch into table. It's presumed that the // batch is ordered and continuous with table. func (t *memoryTable) commit(batch [][]byte) error { @@ -373,6 +397,9 @@ func (f *MemoryFreezer) TruncateTail(tail uint64) (uint64, error) { } } f.tail = tail + if f.items < tail { + f.items = tail + } return old, nil } diff --git a/core/rawdb/freezer_resettable.go b/core/rawdb/freezer_resettable.go index f531e668c3..5494a648c8 100644 --- a/core/rawdb/freezer_resettable.go +++ b/core/rawdb/freezer_resettable.go @@ -221,13 +221,12 @@ func cleanup(path string) error { if err != nil { return err } + defer dir.Close() + names, err := dir.Readdirnames(0) if err != nil { return err } - if cerr := dir.Close(); cerr != nil { - return cerr - } for _, name := range names { if name == filepath.Base(path)+tmpSuffix { log.Info("Removed leftover freezer directory", "name", name) diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index 01a754c5c8..280f6e1aaa 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -707,12 +707,13 @@ func (t *freezerTable) truncateTail(items uint64) error { t.lock.Lock() defer t.lock.Unlock() - // Ensure the given truncate target falls in the correct range + // Short-circuit if the requested tail deletion points to a stale position if t.itemHidden.Load() >= items { return nil } + // If the requested tail exceeds the current head, reset the entire table if t.items.Load() < items { - return errors.New("truncation above head") + return t.resetTo(items) } // Load the new tail index by the given new tail position var ( @@ -822,10 +823,9 @@ func (t *freezerTable) truncateTail(items uint64) error { shorten := indexEntrySize * int64(newDeleted-deleted) if t.metadata.flushOffset <= shorten { return fmt.Errorf("invalid index flush offset: %d, shorten: %d", t.metadata.flushOffset, shorten) - } else { - if err := t.metadata.setFlushOffset(t.metadata.flushOffset-shorten, true); err != nil { - return err - } + } + if err := t.metadata.setFlushOffset(t.metadata.flushOffset-shorten, true); err != nil { + return err } // Retrieve the new size and update the total size counter newSize, err := t.sizeNolock() @@ -836,6 +836,59 @@ func (t *freezerTable) truncateTail(items uint64) error { return nil } +// resetTo clears the entire table and sets both the head and tail to the given +// value. It assumes the caller holds the lock and that tail > t.items. +func (t *freezerTable) resetTo(tail uint64) error { + // Sync the entire table before resetting, eliminating the potential + // data corruption. + err := t.doSync() + if err != nil { + return err + } + // Update the index file to reflect the new offset + if err := t.index.Close(); err != nil { + return err + } + entry := &indexEntry{ + filenum: t.headId + 1, + offset: uint32(tail), + } + if err := reset(t.index.Name(), entry.append(nil)); err != nil { + return err + } + if err := t.metadata.setVirtualTail(tail, true); err != nil { + return err + } + if err := t.metadata.setFlushOffset(indexEntrySize, true); err != nil { + return err + } + t.index, err = openFreezerFileForAppend(t.index.Name()) + if err != nil { + return err + } + + // Purge all the existing data file + if err := t.head.Close(); err != nil { + return err + } + t.headId = t.headId + 1 + t.tailId = t.headId + t.headBytes = 0 + + t.head, err = t.openFile(t.headId, openFreezerFileTruncated) + if err != nil { + return err + } + t.releaseFilesBefore(t.headId, true) + + t.items.Store(tail) + t.itemOffset.Store(tail) + t.itemHidden.Store(tail) + t.sizeGauge.Update(0) + + return nil +} + // Close closes all opened files and finalizes the freezer table for use. // This operation must be completed before shutdown to prevent the loss of // recent writes. @@ -1196,8 +1249,7 @@ func (t *freezerTable) sizeNolock() (uint64, error) { } // advanceHead should be called when the current head file would outgrow the file limits, -// and a new file must be opened. The caller of this method must hold the write-lock -// before calling this method. +// and a new file must be opened. This method acquires the write-lock internally. func (t *freezerTable) advanceHead() error { t.lock.Lock() defer t.lock.Unlock() @@ -1218,7 +1270,9 @@ func (t *freezerTable) advanceHead() error { return err } t.releaseFile(t.headId) - t.openFile(t.headId, openFreezerFileForReadOnly) + if _, err := t.openFile(t.headId, openFreezerFileForReadOnly); err != nil { + return err + } // Swap out the current head. t.head = newHead @@ -1246,25 +1300,20 @@ func (t *freezerTable) doSync() error { if t.index == nil || t.head == nil || t.metadata.file == nil { return errClosed } - var err error - trackError := func(e error) { - if e != nil && err == nil { - err = e - } + if err := t.index.Sync(); err != nil { + return err + } + if err := t.head.Sync(); err != nil { + return err } - trackError(t.index.Sync()) - trackError(t.head.Sync()) - // A crash may occur before the offset is updated, leaving the offset - // points to a old position. If so, the extra items above the offset + // points to an old position. If so, the extra items above the offset // will be truncated during the next run. stat, err := t.index.Stat() if err != nil { return err } - offset := stat.Size() - trackError(t.metadata.setFlushOffset(offset, true)) - return err + return t.metadata.setFlushOffset(stat.Size(), true) } func (t *freezerTable) dumpIndexStdout(start, stop int64) { diff --git a/core/rawdb/freezer_table_test.go b/core/rawdb/freezer_table_test.go index fc21ea6c63..3393f88e1a 100644 --- a/core/rawdb/freezer_table_test.go +++ b/core/rawdb/freezer_table_test.go @@ -1139,6 +1139,7 @@ const ( opTruncateHeadAll opTruncateTail opTruncateTailAll + opTruncateTailOverHead opCheckAll opMax // boundary value, not an actual op ) @@ -1226,6 +1227,11 @@ func (randTest) Generate(r *rand.Rand, size int) reflect.Value { step.target = deleted + uint64(len(items)) items = items[:0] deleted = step.target + case opTruncateTailOverHead: + newDeleted := deleted + uint64(len(items)) + 10 + step.target = newDeleted + deleted = newDeleted + items = items[:0] } steps = append(steps, step) } @@ -1268,7 +1274,7 @@ func runRandTest(rt randTest) bool { for i := 0; i < len(step.items); i++ { batch.AppendRaw(step.items[i], step.blobs[i]) } - batch.commit() + rt[i].err = batch.commit() values = append(values, step.blobs...) case opRetrieve: @@ -1290,24 +1296,28 @@ func runRandTest(rt randTest) bool { } case opTruncateHead: - f.truncateHead(step.target) + rt[i].err = f.truncateHead(step.target) length := f.items.Load() - f.itemHidden.Load() values = values[:length] case opTruncateHeadAll: - f.truncateHead(step.target) + rt[i].err = f.truncateHead(step.target) values = nil case opTruncateTail: prev := f.itemHidden.Load() - f.truncateTail(step.target) + rt[i].err = f.truncateTail(step.target) truncated := f.itemHidden.Load() - prev values = values[truncated:] case opTruncateTailAll: - f.truncateTail(step.target) + rt[i].err = f.truncateTail(step.target) + values = nil + + case opTruncateTailOverHead: + rt[i].err = f.truncateTail(step.target) values = nil } // Abort the test on error. @@ -1633,3 +1643,43 @@ func TestFreezerAncientBytes(t *testing.T) { }) } } + +func TestTruncateOverHead(t *testing.T) { + t.Parallel() + + fn := fmt.Sprintf("t-%d", rand.Uint64()) + f, err := newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 100, freezerTableConfig{noSnappy: true}, false) + if err != nil { + t.Fatal(err) + } + + // Tail truncation on an empty table + if err := f.truncateTail(10); err != nil { + t.Fatal(err) + } + batch := f.newBatch() + data := getChunk(10, 1) + require.NoError(t, batch.AppendRaw(uint64(10), data)) + require.NoError(t, batch.commit()) + + got, err := f.RetrieveItems(uint64(10), 1, 0) + require.NoError(t, err) + if !bytes.Equal(got[0], data) { + t.Fatalf("Unexpected bytes, want: %v, got: %v", data, got[0]) + } + + // Tail truncation on the non-empty table + if err := f.truncateTail(20); err != nil { + t.Fatal(err) + } + batch = f.newBatch() + data = getChunk(10, 1) + require.NoError(t, batch.AppendRaw(uint64(20), data)) + require.NoError(t, batch.commit()) + + got, err = f.RetrieveItems(uint64(20), 1, 0) + require.NoError(t, err) + if !bytes.Equal(got[0], data) { + t.Fatalf("Unexpected bytes, want: %v, got: %v", data, got[0]) + } +} diff --git a/core/rawdb/freezer_utils.go b/core/rawdb/freezer_utils.go index 752e95ba6a..7f1a978b63 100644 --- a/core/rawdb/freezer_utils.go +++ b/core/rawdb/freezer_utils.go @@ -22,6 +22,13 @@ import ( "path/filepath" ) +func atomicRename(src, dest string) error { + if err := os.Rename(src, dest); err != nil { + return err + } + return syncDir(filepath.Dir(src)) +} + // copyFrom copies data from 'srcPath' at offset 'offset' into 'destPath'. // The 'destPath' is created if it doesn't exist, otherwise it is overwritten. // Before the copy is executed, there is a callback can be registered to @@ -73,13 +80,48 @@ func copyFrom(srcPath, destPath string, offset uint64, before func(f *os.File) e return err } f = nil - return os.Rename(fname, destPath) + + return atomicRename(fname, destPath) +} + +// reset atomically replaces the file at the given path with the provided content. +func reset(path string, content []byte) error { + // Create a temp file in the same dir where we want it to wind up + f, err := os.CreateTemp(filepath.Dir(path), "*") + if err != nil { + return err + } + fname := f.Name() + + // Clean up the leftover file + defer func() { + if f != nil { + f.Close() + } + os.Remove(fname) + }() + + // Write the content into the temp file + _, err = f.Write(content) + if err != nil { + return err + } + // Permanently persist the content into disk + if err := f.Sync(); err != nil { + return err + } + if err := f.Close(); err != nil { + return err + } + f = nil + + return atomicRename(fname, path) } // openFreezerFileForAppend opens a freezer table file and seeks to the end func openFreezerFileForAppend(filename string) (*os.File, error) { // Open the file without the O_APPEND flag - // because it has differing behaviour during Truncate operations + // because it has differing behavior during Truncate operations // on different OS's file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0644) if err != nil { diff --git a/core/rawdb/freezer_utils_unix.go b/core/rawdb/freezer_utils_unix.go new file mode 100644 index 0000000000..1d26490cab --- /dev/null +++ b/core/rawdb/freezer_utils_unix.go @@ -0,0 +1,49 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +//go:build !windows +// +build !windows + +package rawdb + +import ( + "errors" + "os" + "syscall" +) + +// syncDir ensures that the directory metadata (e.g. newly renamed files) +// is flushed to durable storage. +func syncDir(name string) error { + f, err := os.Open(name) + if err != nil { + return err + } + defer f.Close() + + // Some file systems do not support fsyncing directories (e.g. some FUSE + // mounts). Ignore EINVAL in those cases. + if err := f.Sync(); err != nil { + if errors.Is(err, os.ErrInvalid) { + return nil + } + if patherr, ok := err.(*os.PathError); ok && patherr.Err == syscall.EINVAL { + return nil + } + return err + } + return nil +} diff --git a/eth/protocols/snap/tracker.go b/core/rawdb/freezer_utils_windows.go similarity index 74% rename from eth/protocols/snap/tracker.go rename to core/rawdb/freezer_utils_windows.go index 2cf59cc23a..7b652f7ab5 100644 --- a/eth/protocols/snap/tracker.go +++ b/core/rawdb/freezer_utils_windows.go @@ -1,4 +1,4 @@ -// Copyright 2021 The go-ethereum Authors +// Copyright 2022 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -14,13 +14,13 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package snap +//go:build windows +// +build windows -import ( - "time" +package rawdb - "github.com/ethereum/go-ethereum/p2p/tracker" -) - -// requestTracker is a singleton tracker for request times. -var requestTracker = tracker.New(ProtocolName, time.Minute) +// syncDir is a no-op on Windows. Fsyncing a directory handle is not +// supported and returns "Access is denied". +func syncDir(name string) error { + return nil +} diff --git a/core/rawdb/table.go b/core/rawdb/table.go index 4f533b19ef..63ec3f5c61 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -257,6 +257,11 @@ func (b *tableBatch) Reset() { b.batch.Reset() } +// Close closes the batch and releases all associated resources. +func (b *tableBatch) Close() { + b.batch.Close() +} + // tableReplayer is a wrapper around a batch replayer which truncates // the added prefix. type tableReplayer struct { diff --git a/core/state/access_events.go b/core/state/access_events.go index 0575c9898a..86f44bd623 100644 --- a/core/state/access_events.go +++ b/core/state/access_events.go @@ -23,7 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie/utils" + "github.com/ethereum/go-ethereum/trie/bintrie" "github.com/holiman/uint256" ) @@ -45,15 +45,12 @@ var zeroTreeIndex uint256.Int type AccessEvents struct { branches map[branchAccessKey]mode chunks map[chunkAccessKey]mode - - pointCache *utils.PointCache } -func NewAccessEvents(pointCache *utils.PointCache) *AccessEvents { +func NewAccessEvents() *AccessEvents { return &AccessEvents{ - branches: make(map[branchAccessKey]mode), - chunks: make(map[chunkAccessKey]mode), - pointCache: pointCache, + branches: make(map[branchAccessKey]mode), + chunks: make(map[chunkAccessKey]mode), } } @@ -75,8 +72,11 @@ func (ae *AccessEvents) Keys() [][]byte { // TODO: consider if parallelizing this is worth it, probably depending on len(ae.chunks). keys := make([][]byte, 0, len(ae.chunks)) for chunk := range ae.chunks { - basePoint := ae.pointCache.Get(chunk.addr[:]) - key := utils.GetTreeKeyWithEvaluatedAddress(basePoint, &chunk.treeIndex, chunk.leafKey) + var offset [32]byte + treeIndexBytes := chunk.treeIndex.Bytes32() + copy(offset[:31], treeIndexBytes[1:]) + offset[31] = chunk.leafKey + key := bintrie.GetBinaryTreeKey(chunk.addr, offset[:]) keys = append(keys, key) } return keys @@ -84,9 +84,8 @@ func (ae *AccessEvents) Keys() [][]byte { func (ae *AccessEvents) Copy() *AccessEvents { cpy := &AccessEvents{ - branches: maps.Clone(ae.branches), - chunks: maps.Clone(ae.chunks), - pointCache: ae.pointCache, + branches: maps.Clone(ae.branches), + chunks: maps.Clone(ae.chunks), } return cpy } @@ -95,12 +94,12 @@ func (ae *AccessEvents) Copy() *AccessEvents { // member fields of an account. func (ae *AccessEvents) AddAccount(addr common.Address, isWrite bool, availableGas uint64) uint64 { var gas uint64 // accumulate the consumed gas - consumed, expected := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.BasicDataLeafKey, isWrite, availableGas) + consumed, expected := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.BasicDataLeafKey, isWrite, availableGas) if consumed < expected { return expected } gas += consumed - consumed, expected = ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeHashLeafKey, isWrite, availableGas-consumed) + consumed, expected = ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.CodeHashLeafKey, isWrite, availableGas-consumed) if consumed < expected { return expected + gas } @@ -112,7 +111,7 @@ func (ae *AccessEvents) AddAccount(addr common.Address, isWrite bool, availableG // cold member fields of an account, that need to be touched when making a message // call to that account. func (ae *AccessEvents) MessageCallGas(destination common.Address, availableGas uint64) uint64 { - _, expected := ae.touchAddressAndChargeGas(destination, zeroTreeIndex, utils.BasicDataLeafKey, false, availableGas) + _, expected := ae.touchAddressAndChargeGas(destination, zeroTreeIndex, bintrie.BasicDataLeafKey, false, availableGas) if expected == 0 { expected = params.WarmStorageReadCostEIP2929 } @@ -122,11 +121,11 @@ func (ae *AccessEvents) MessageCallGas(destination common.Address, availableGas // ValueTransferGas returns the gas to be charged for each of the currently // cold balance member fields of the caller and the callee accounts. func (ae *AccessEvents) ValueTransferGas(callerAddr, targetAddr common.Address, availableGas uint64) uint64 { - _, expected1 := ae.touchAddressAndChargeGas(callerAddr, zeroTreeIndex, utils.BasicDataLeafKey, true, availableGas) + _, expected1 := ae.touchAddressAndChargeGas(callerAddr, zeroTreeIndex, bintrie.BasicDataLeafKey, true, availableGas) if expected1 > availableGas { return expected1 } - _, expected2 := ae.touchAddressAndChargeGas(targetAddr, zeroTreeIndex, utils.BasicDataLeafKey, true, availableGas-expected1) + _, expected2 := ae.touchAddressAndChargeGas(targetAddr, zeroTreeIndex, bintrie.BasicDataLeafKey, true, availableGas-expected1) if expected1+expected2 == 0 { return params.WarmStorageReadCostEIP2929 } @@ -138,8 +137,8 @@ func (ae *AccessEvents) ValueTransferGas(callerAddr, targetAddr common.Address, // address collision is done before the transfer, and so no write // are guaranteed to happen at this point. func (ae *AccessEvents) ContractCreatePreCheckGas(addr common.Address, availableGas uint64) uint64 { - consumed, expected1 := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.BasicDataLeafKey, false, availableGas) - _, expected2 := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeHashLeafKey, false, availableGas-consumed) + consumed, expected1 := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.BasicDataLeafKey, false, availableGas) + _, expected2 := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.CodeHashLeafKey, false, availableGas-consumed) return expected1 + expected2 } @@ -147,9 +146,9 @@ func (ae *AccessEvents) ContractCreatePreCheckGas(addr common.Address, available // a contract creation. func (ae *AccessEvents) ContractCreateInitGas(addr common.Address, availableGas uint64) (uint64, uint64) { var gas uint64 - consumed, expected1 := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.BasicDataLeafKey, true, availableGas) + consumed, expected1 := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.BasicDataLeafKey, true, availableGas) gas += consumed - consumed, expected2 := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeHashLeafKey, true, availableGas-consumed) + consumed, expected2 := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.CodeHashLeafKey, true, availableGas-consumed) gas += consumed return gas, expected1 + expected2 } @@ -157,20 +156,20 @@ func (ae *AccessEvents) ContractCreateInitGas(addr common.Address, availableGas // AddTxOrigin adds the member fields of the sender account to the access event list, // so that cold accesses are not charged, since they are covered by the 21000 gas. func (ae *AccessEvents) AddTxOrigin(originAddr common.Address) { - ae.touchAddressAndChargeGas(originAddr, zeroTreeIndex, utils.BasicDataLeafKey, true, gomath.MaxUint64) - ae.touchAddressAndChargeGas(originAddr, zeroTreeIndex, utils.CodeHashLeafKey, false, gomath.MaxUint64) + ae.touchAddressAndChargeGas(originAddr, zeroTreeIndex, bintrie.BasicDataLeafKey, true, gomath.MaxUint64) + ae.touchAddressAndChargeGas(originAddr, zeroTreeIndex, bintrie.CodeHashLeafKey, false, gomath.MaxUint64) } // AddTxDestination adds the member fields of the sender account to the access event list, // so that cold accesses are not charged, since they are covered by the 21000 gas. func (ae *AccessEvents) AddTxDestination(addr common.Address, sendsValue, doesntExist bool) { - ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.BasicDataLeafKey, sendsValue, gomath.MaxUint64) - ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeHashLeafKey, doesntExist, gomath.MaxUint64) + ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.BasicDataLeafKey, sendsValue, gomath.MaxUint64) + ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.CodeHashLeafKey, doesntExist, gomath.MaxUint64) } // SlotGas returns the amount of gas to be charged for a cold storage access. func (ae *AccessEvents) SlotGas(addr common.Address, slot common.Hash, isWrite bool, availableGas uint64, chargeWarmCosts bool) uint64 { - treeIndex, subIndex := utils.StorageIndex(slot.Bytes()) + treeIndex, subIndex := bintrie.StorageIndex(slot.Bytes()) _, expected := ae.touchAddressAndChargeGas(addr, *treeIndex, subIndex, isWrite, availableGas) if expected == 0 && chargeWarmCosts { expected = params.WarmStorageReadCostEIP2929 @@ -313,7 +312,7 @@ func (ae *AccessEvents) CodeChunksRangeGas(contractAddr common.Address, startPC, // Note that an access in write mode implies an access in read mode, whereas an // access in read mode does not imply an access in write mode. func (ae *AccessEvents) BasicDataGas(addr common.Address, isWrite bool, availableGas uint64, chargeWarmCosts bool) uint64 { - _, expected := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.BasicDataLeafKey, isWrite, availableGas) + _, expected := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.BasicDataLeafKey, isWrite, availableGas) if expected == 0 && chargeWarmCosts { if availableGas < params.WarmStorageReadCostEIP2929 { return availableGas @@ -329,7 +328,7 @@ func (ae *AccessEvents) BasicDataGas(addr common.Address, isWrite bool, availabl // Note that an access in write mode implies an access in read mode, whereas an access in // read mode does not imply an access in write mode. func (ae *AccessEvents) CodeHashGas(addr common.Address, isWrite bool, availableGas uint64, chargeWarmCosts bool) uint64 { - _, expected := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeHashLeafKey, isWrite, availableGas) + _, expected := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.CodeHashLeafKey, isWrite, availableGas) if expected == 0 && chargeWarmCosts { if availableGas < params.WarmStorageReadCostEIP2929 { return availableGas diff --git a/core/state/access_events_test.go b/core/state/access_events_test.go index e80859a0b4..0b39130e8d 100644 --- a/core/state/access_events_test.go +++ b/core/state/access_events_test.go @@ -22,7 +22,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie/utils" ) var ( @@ -38,7 +37,7 @@ func init() { } func TestAccountHeaderGas(t *testing.T) { - ae := NewAccessEvents(utils.NewPointCache(1024)) + ae := NewAccessEvents() // Check cold read cost gas := ae.BasicDataGas(testAddr, false, math.MaxUint64, false) @@ -93,7 +92,7 @@ func TestAccountHeaderGas(t *testing.T) { // TestContractCreateInitGas checks that the gas cost of contract creation is correctly // calculated. func TestContractCreateInitGas(t *testing.T) { - ae := NewAccessEvents(utils.NewPointCache(1024)) + ae := NewAccessEvents() var testAddr [20]byte for i := byte(0); i < 20; i++ { @@ -116,7 +115,7 @@ func TestContractCreateInitGas(t *testing.T) { // TestMessageCallGas checks that the gas cost of message calls is correctly // calculated. func TestMessageCallGas(t *testing.T) { - ae := NewAccessEvents(utils.NewPointCache(1024)) + ae := NewAccessEvents() // Check cold read cost, without a value gas := ae.MessageCallGas(testAddr, math.MaxUint64) diff --git a/core/state/database.go b/core/state/database.go index e1236c6571..53bf38df61 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -27,24 +27,17 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/bintrie" + "github.com/ethereum/go-ethereum/trie/transitiontrie" "github.com/ethereum/go-ethereum/trie/trienode" - "github.com/ethereum/go-ethereum/trie/utils" "github.com/ethereum/go-ethereum/triedb" ) const ( // Arbitrum: Cache size granted for caching clean compiled wasm code. activatedWasmCacheSize = 64 * 1024 * 1024 - - // Number of codehash->size associations to keep. - codeSizeCacheSize = 1_000_000 // 4 megabytes in total - - // Cache size granted for caching clean code. - codeCacheSize = 256 * 1024 * 1024 - - // Number of address->curve point associations to keep. - pointCacheSize = 4096 ) // Database wraps access to tries and contract code. @@ -65,14 +58,16 @@ type Database interface { // DiskDB returns the underlying key-value disk database. DiskDB() ethdb.KeyValueStore - // PointCache returns the cache holding points used in verkle tree key computation - PointCache() *utils.PointCache - // TrieDB returns the underlying trie database for managing trie nodes. TrieDB() *triedb.Database // Snapshot returns the underlying state snapshot. Snapshot() *snapshot.Tree + + // Commit flushes all pending writes and finalizes the state transition, + // committing the changes to the underlying storage. It returns an error + // if the commit fails. + Commit(update *stateUpdate) error } // Trie is a Ethereum Merkle Patricia trie. @@ -172,20 +167,21 @@ type CachingDB struct { // Arbitrum activatedAsmCache *lru.SizeConstrainedCache[activatedAsmCacheKey, []byte] - disk ethdb.KeyValueStore - wasmdb ethdb.KeyValueStore - triedb *triedb.Database - snap *snapshot.Tree - codeCache *lru.SizeConstrainedCache[common.Hash, []byte] - codeSizeCache *lru.Cache[common.Hash, int] - pointCache *utils.PointCache + disk ethdb.KeyValueStore + wasmdb ethdb.KeyValueStore + triedb *triedb.Database + codedb *CodeDB + snap *snapshot.Tree // Transition-specific fields TransitionStatePerRoot *lru.Cache[common.Hash, *overlay.TransitionState] } // NewDatabase creates a state database with the provided data sources. -func NewDatabase(triedb *triedb.Database, snap *snapshot.Tree) *CachingDB { +func NewDatabase(triedb *triedb.Database, codedb *CodeDB) *CachingDB { + if codedb == nil { + codedb = NewCodeDB(triedb.Disk()) + } wasmdb := triedb.Disk().WasmDataBase() return &CachingDB{ // Arbitrum only @@ -194,10 +190,7 @@ func NewDatabase(triedb *triedb.Database, snap *snapshot.Tree) *CachingDB { disk: triedb.Disk(), wasmdb: wasmdb, triedb: triedb, - snap: snap, - codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), - codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), - pointCache: utils.NewPointCache(pointCacheSize), + codedb: codedb, TransitionStatePerRoot: lru.NewCache[common.Hash, *overlay.TransitionState](1000), } } @@ -205,11 +198,19 @@ func NewDatabase(triedb *triedb.Database, snap *snapshot.Tree) *CachingDB { // NewDatabaseForTesting is similar to NewDatabase, but it initializes the caching // db by using an ephemeral memory db with default config for testing. func NewDatabaseForTesting() *CachingDB { - return NewDatabase(triedb.NewDatabase(rawdb.NewMemoryDatabase(), nil), nil) + db := rawdb.NewMemoryDatabase() + return NewDatabase(triedb.NewDatabase(db, nil), NewCodeDB(db)) } -// Reader returns a state reader associated with the specified state root. -func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) { +// WithSnapshot configures the provided contract code cache. Note that this +// registration must be performed before the cachingDB is used. +func (db *CachingDB) WithSnapshot(snapshot *snapshot.Tree) *CachingDB { + db.snap = snapshot + return db +} + +// StateReader returns a state reader associated with the specified state root. +func (db *CachingDB) StateReader(stateRoot common.Hash) (StateReader, error) { var readers []StateReader // Configure the state reader using the standalone snapshot in hash mode. @@ -233,29 +234,37 @@ func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) { } // Configure the trie reader, which is expected to be available as the // gatekeeper unless the state is corrupted. - tr, err := newTrieReader(stateRoot, db.triedb, db.pointCache) + tr, err := newTrieReader(stateRoot, db.triedb) if err != nil { return nil, err } readers = append(readers, tr) - combined, err := newMultiStateReader(readers...) + return newMultiStateReader(readers...) +} + +// Reader implements Database, returning a reader associated with the specified +// state root. +func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) { + sr, err := db.StateReader(stateRoot) if err != nil { return nil, err } - return newReader(newCachingCodeReader(db.disk, db.codeCache, db.codeSizeCache), combined), nil + return newReader(db.codedb.Reader(), sr), nil } -// ReadersWithCacheStats creates a pair of state readers sharing the same internal cache and -// same backing Reader, but exposing separate statistics. -// and statistics. -func (db *CachingDB) ReadersWithCacheStats(stateRoot common.Hash) (ReaderWithStats, ReaderWithStats, error) { - reader, err := db.Reader(stateRoot) +// ReadersWithCacheStats creates a pair of state readers that share the same +// underlying state reader and internal state cache, while maintaining separate +// statistics respectively. +func (db *CachingDB) ReadersWithCacheStats(stateRoot common.Hash) (Reader, Reader, error) { + r, err := db.StateReader(stateRoot) if err != nil { return nil, nil, err } - shared := newReaderWithCache(reader) - return newReaderWithCacheStats(shared), newReaderWithCacheStats(shared), nil + sr := newStateReaderWithCache(r) + ra := newReader(db.codedb.Reader(), newStateReaderWithStats(sr)) + rb := newReader(db.codedb.Reader(), newStateReaderWithStats(sr)) + return ra, rb, nil } // OpenTrie opens the main account trie at a specific root hash. @@ -263,10 +272,12 @@ func (db *CachingDB) OpenTrie(root common.Hash) (Trie, error) { if db.triedb.IsVerkle() { ts := overlay.LoadTransitionState(db.TrieDB().Disk(), root, db.triedb.IsVerkle()) if ts.InTransition() { - panic("transition isn't supported yet") + panic("state tree transition isn't supported yet") } if ts.Transitioned() { - return trie.NewVerkleTrie(root, db.triedb, db.pointCache) + // Use BinaryTrie instead of VerkleTrie when IsVerkle is set + // (IsVerkle actually means Binary Trie mode in this codebase) + return bintrie.NewBinaryTrie(root, db.triedb) } } tr, err := trie.NewStateTrie(trie.StateTrieID(root), db.triedb) @@ -288,45 +299,56 @@ func (db *CachingDB) OpenStorageTrie(stateRoot common.Hash, address common.Addre return tr, nil } -// ContractCodeWithPrefix retrieves a particular contract's code. If the -// code can't be found in the cache, then check the existence with **new** -// db scheme. -func (db *CachingDB) ContractCodeWithPrefix(address common.Address, codeHash common.Hash) []byte { - code, _ := db.codeCache.Get(codeHash) - if len(code) > 0 { - return code - } - code = rawdb.ReadCodeWithPrefix(db.disk, codeHash) - if len(code) > 0 { - db.codeCache.Add(codeHash, code) - db.codeSizeCache.Add(codeHash, len(code)) - } - return code -} - // TrieDB retrieves any intermediate trie-node caching layer. func (db *CachingDB) TrieDB() *triedb.Database { return db.triedb } -// PointCache returns the cache of evaluated curve points. -func (db *CachingDB) PointCache() *utils.PointCache { - return db.pointCache -} - // Snapshot returns the underlying state snapshot. func (db *CachingDB) Snapshot() *snapshot.Tree { return db.snap } +// Commit flushes all pending writes and finalizes the state transition, +// committing the changes to the underlying storage. It returns an error +// if the commit fails. +func (db *CachingDB) Commit(update *stateUpdate) error { + // Short circuit if nothing to commit + if update.empty() { + return nil + } + // Commit dirty contract code if any exists + if len(update.codes) > 0 { + batch := db.codedb.NewBatchWithSize(len(update.codes)) + for _, code := range update.codes { + batch.Put(code.hash, code.blob) + } + if err := batch.Commit(); err != nil { + return err + } + } + // If snapshotting is enabled, update the snapshot tree with this new version + if db.snap != nil && db.snap.Snapshot(update.originRoot) != nil { + if err := db.snap.Update(update.root, update.originRoot, update.accounts, update.storages); err != nil { + log.Warn("Failed to update snapshot tree", "from", update.originRoot, "to", update.root, "err", err) + } + // Keep 128 diff layers in the memory, persistent layer is 129th. + // - head layer is paired with HEAD state + // - head-1 layer is paired with HEAD-1 state + // - head-127 layer(bottom-most diff layer) is paired with HEAD-127 state + if err := db.snap.Cap(update.root, DefaultTriesInMemory); err != nil { + log.Warn("Failed to cap snapshot tree", "root", update.root, "layers", DefaultTriesInMemory, "err", err) + } + } + return db.triedb.Update(update.root, update.originRoot, update.blockNumber, update.nodes, update.stateSet()) +} + // mustCopyTrie returns a deep-copied trie. func mustCopyTrie(t Trie) Trie { switch t := t.(type) { case *trie.StateTrie: return t.Copy() - case *trie.VerkleTrie: - return t.Copy() - case *trie.TransitionTrie: + case *transitiontrie.TransitionTrie: return t.Copy() default: panic(fmt.Errorf("unknown trie type %T", t)) diff --git a/core/state/database_code.go b/core/state/database_code.go new file mode 100644 index 0000000000..820c9c1168 --- /dev/null +++ b/core/state/database_code.go @@ -0,0 +1,231 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import ( + "sync/atomic" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/lru" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" +) + +const ( + // Number of codeHash->size associations to keep. + codeSizeCacheSize = 1_000_000 + + // Cache size granted for caching clean code. + codeCacheSize = 256 * 1024 * 1024 +) + +// CodeCache maintains cached contract code that is shared across blocks, enabling +// fast access for external calls such as RPCs and state transitions. +// +// It is thread-safe and has a bounded size. +type codeCache struct { + codeCache *lru.SizeConstrainedCache[common.Hash, []byte] + codeSizeCache *lru.Cache[common.Hash, int] +} + +// newCodeCache initializes the contract code cache with the predefined capacity. +func newCodeCache() *codeCache { + return &codeCache{ + codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), + codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), + } +} + +// Get returns the contract code associated with the provided code hash. +func (c *codeCache) Get(hash common.Hash) ([]byte, bool) { + return c.codeCache.Get(hash) +} + +// GetSize returns the contract code size associated with the provided code hash. +func (c *codeCache) GetSize(hash common.Hash) (int, bool) { + return c.codeSizeCache.Get(hash) +} + +// Put adds the provided contract code along with its size information into the cache. +func (c *codeCache) Put(hash common.Hash, code []byte) { + c.codeCache.Add(hash, code) + c.codeSizeCache.Add(hash, len(code)) +} + +// CodeReader implements state.ContractCodeReader, accessing contract code either in +// local key-value store or the shared code cache. +// +// Reader is safe for concurrent access. +type CodeReader struct { + db ethdb.KeyValueReader + cache *codeCache + + // Cache statistics + hit atomic.Int64 // Number of code lookups found in the cache + miss atomic.Int64 // Number of code lookups not found in the cache + hitBytes atomic.Int64 // Total number of bytes read from cache + missBytes atomic.Int64 // Total number of bytes read from database +} + +// newCodeReader constructs the code reader with provided key value store and the cache. +func newCodeReader(db ethdb.KeyValueReader, cache *codeCache) *CodeReader { + return &CodeReader{ + db: db, + cache: cache, + } +} + +// Has returns the flag indicating whether the contract code with +// specified address and hash exists or not. +func (r *CodeReader) Has(addr common.Address, codeHash common.Hash) bool { + return len(r.Code(addr, codeHash)) > 0 +} + +// Code implements state.ContractCodeReader, retrieving a particular contract's code. +// Null is returned if the contract code is not present. +func (r *CodeReader) Code(addr common.Address, codeHash common.Hash) []byte { + code, _ := r.cache.Get(codeHash) + if len(code) > 0 { + r.hit.Add(1) + r.hitBytes.Add(int64(len(code))) + return code + } + r.miss.Add(1) + + code = rawdb.ReadCode(r.db, codeHash) + if len(code) > 0 { + r.cache.Put(codeHash, code) + r.missBytes.Add(int64(len(code))) + } + return code +} + +// CodeSize implements state.ContractCodeReader, retrieving a particular contract +// code's size. Zero is returned if the contract code is not present. +func (r *CodeReader) CodeSize(addr common.Address, codeHash common.Hash) int { + if cached, ok := r.cache.GetSize(codeHash); ok { + r.hit.Add(1) + return cached + } + return len(r.Code(addr, codeHash)) +} + +// CodeWithPrefix retrieves the contract code for the specified account address +// and code hash. It is almost identical to Code, but uses rawdb.ReadCodeWithPrefix +// for database lookups. The intention is to gradually deprecate the old +// contract code scheme. +func (r *CodeReader) CodeWithPrefix(addr common.Address, codeHash common.Hash) []byte { + code, _ := r.cache.Get(codeHash) + if len(code) > 0 { + r.hit.Add(1) + r.hitBytes.Add(int64(len(code))) + return code + } + r.miss.Add(1) + + code = rawdb.ReadCodeWithPrefix(r.db, codeHash) + if len(code) > 0 { + r.cache.Put(codeHash, code) + r.missBytes.Add(int64(len(code))) + } + return code +} + +// GetCodeStats implements ContractCodeReaderStater, returning the statistics +// of the code reader. +func (r *CodeReader) GetCodeStats() ContractCodeReaderStats { + return ContractCodeReaderStats{ + CacheHit: r.hit.Load(), + CacheMiss: r.miss.Load(), + CacheHitBytes: r.hitBytes.Load(), + CacheMissBytes: r.missBytes.Load(), + } +} + +type CodeBatch struct { + db *CodeDB + codes [][]byte + codeHashes []common.Hash +} + +// newCodeBatch constructs the batch for writing contract code. +func newCodeBatch(db *CodeDB) *CodeBatch { + return &CodeBatch{ + db: db, + } +} + +// newCodeBatchWithSize constructs the batch with a pre-allocated capacity. +func newCodeBatchWithSize(db *CodeDB, size int) *CodeBatch { + return &CodeBatch{ + db: db, + codes: make([][]byte, 0, size), + codeHashes: make([]common.Hash, 0, size), + } +} + +// Put inserts the given contract code into the writer, waiting for commit. +func (b *CodeBatch) Put(codeHash common.Hash, code []byte) { + b.codes = append(b.codes, code) + b.codeHashes = append(b.codeHashes, codeHash) +} + +// Commit flushes the accumulated dirty contract code into the database and +// also place them in the cache. +func (b *CodeBatch) Commit() error { + batch := b.db.db.NewBatch() + for i, code := range b.codes { + rawdb.WriteCode(batch, b.codeHashes[i], code) + b.db.cache.Put(b.codeHashes[i], code) + } + if err := batch.Write(); err != nil { + return err + } + b.codes = b.codes[:0] + b.codeHashes = b.codeHashes[:0] + return nil +} + +// CodeDB is responsible for managing the contract code and provides the access +// to it. It can be used as a global object, sharing it between multiple entities. +type CodeDB struct { + db ethdb.KeyValueStore + cache *codeCache +} + +// NewCodeDB constructs the contract code database with the provided key value store. +func NewCodeDB(db ethdb.KeyValueStore) *CodeDB { + return &CodeDB{ + db: db, + cache: newCodeCache(), + } +} + +// Reader returns the contract code reader. +func (d *CodeDB) Reader() *CodeReader { + return newCodeReader(d.db, d.cache) +} + +// NewBatch returns the batch for flushing contract codes. +func (d *CodeDB) NewBatch() *CodeBatch { + return newCodeBatch(d) +} + +// NewBatchWithSize returns the batch with pre-allocated capacity. +func (d *CodeDB) NewBatchWithSize(size int) *CodeBatch { + return newCodeBatchWithSize(d, size) +} diff --git a/core/state/database_history.go b/core/state/database_history.go index b2fda19e4f..11bb5c0226 100644 --- a/core/state/database_history.go +++ b/core/state/database_history.go @@ -18,41 +18,39 @@ package state import ( "errors" + "fmt" + "sync" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie/utils" + "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/database" "github.com/ethereum/go-ethereum/triedb/pathdb" ) -// historicReader wraps a historical state reader defined in path database, -// providing historic state serving over the path scheme. -// -// TODO(rjl493456442): historicReader is not thread-safe and does not fully -// comply with the StateReader interface requirements, needs to be fixed. -// Currently, it is only used in a non-concurrent context, so it is safe for now. -type historicReader struct { +// historicStateReader implements StateReader, wrapping a historical state reader +// defined in path database and provide historic state serving over the path scheme. +type historicStateReader struct { reader *pathdb.HistoricalStateReader + lock sync.Mutex // Lock for protecting concurrent read } -// newHistoricReader constructs a reader for historic state serving. -func newHistoricReader(r *pathdb.HistoricalStateReader) *historicReader { - return &historicReader{reader: r} +// newHistoricStateReader constructs a reader for historical state serving. +func newHistoricStateReader(r *pathdb.HistoricalStateReader) *historicStateReader { + return &historicStateReader{reader: r} } // Account implements StateReader, retrieving the account specified by the address. -// -// An error will be returned if the associated snapshot is already stale or -// the requested account is not yet covered by the snapshot. -// -// The returned account might be nil if it's not existent. -func (r *historicReader) Account(addr common.Address) (*types.StateAccount, error) { +func (r *historicStateReader) Account(addr common.Address) (*types.StateAccount, error) { + r.lock.Lock() + defer r.lock.Unlock() + account, err := r.reader.Account(addr) if err != nil { return nil, err @@ -82,7 +80,10 @@ func (r *historicReader) Account(addr common.Address) (*types.StateAccount, erro // the requested storage slot is not yet covered by the snapshot. // // The returned storage slot might be empty if it's not existent. -func (r *historicReader) Storage(addr common.Address, key common.Hash) (common.Hash, error) { +func (r *historicStateReader) Storage(addr common.Address, key common.Hash) (common.Hash, error) { + r.lock.Lock() + defer r.lock.Unlock() + blob, err := r.reader.Storage(addr, key) if err != nil { return common.Hash{}, err @@ -99,53 +100,198 @@ func (r *historicReader) Storage(addr common.Address, key common.Hash) (common.H return slot, nil } +// historicTrieOpener is a wrapper of pathdb.HistoricalNodeReader, implementing +// the database.NodeDatabase by adding NodeReader function. +type historicTrieOpener struct { + root common.Hash + reader *pathdb.HistoricalNodeReader +} + +// newHistoricTrieOpener constructs the historical trie opener. +func newHistoricTrieOpener(root common.Hash, reader *pathdb.HistoricalNodeReader) *historicTrieOpener { + return &historicTrieOpener{ + root: root, + reader: reader, + } +} + +// NodeReader implements database.NodeDatabase, returning a node reader of a +// specified state. +func (o *historicTrieOpener) NodeReader(root common.Hash) (database.NodeReader, error) { + if root != o.root { + return nil, fmt.Errorf("state %x is not available", root) + } + return o.reader, nil +} + +// historicalTrieReader wraps a historical node reader defined in path database, +// providing historical node serving over the path scheme. +type historicalTrieReader struct { + root common.Hash + opener *historicTrieOpener + tr Trie + + subRoots map[common.Address]common.Hash // Set of storage roots, cached when the account is resolved + subTries map[common.Address]Trie // Group of storage tries, cached when it's resolved + lock sync.Mutex // Lock for protecting concurrent read +} + +// newHistoricalTrieReader constructs a reader for historical trie node serving. +func newHistoricalTrieReader(root common.Hash, r *pathdb.HistoricalNodeReader) (*historicalTrieReader, error) { + opener := newHistoricTrieOpener(root, r) + tr, err := trie.NewStateTrie(trie.StateTrieID(root), opener) + if err != nil { + return nil, err + } + return &historicalTrieReader{ + root: root, + opener: opener, + tr: tr, + subRoots: make(map[common.Address]common.Hash), + subTries: make(map[common.Address]Trie), + }, nil +} + +// account is the inner version of Account and assumes the r.lock is already held. +func (r *historicalTrieReader) account(addr common.Address) (*types.StateAccount, error) { + account, err := r.tr.GetAccount(addr) + if err != nil { + return nil, err + } + if account == nil { + r.subRoots[addr] = types.EmptyRootHash + } else { + r.subRoots[addr] = account.Root + } + return account, nil +} + +// Account implements StateReader, retrieving the account specified by the address. +// +// An error will be returned if the associated snapshot is already stale or +// the requested account is not yet covered by the snapshot. +// +// The returned account might be nil if it's not existent. +func (r *historicalTrieReader) Account(addr common.Address) (*types.StateAccount, error) { + r.lock.Lock() + defer r.lock.Unlock() + + return r.account(addr) +} + +// Storage implements StateReader, retrieving the storage slot specified by the +// address and slot key. +// +// An error will be returned if the associated snapshot is already stale or +// the requested storage slot is not yet covered by the snapshot. +// +// The returned storage slot might be empty if it's not existent. +func (r *historicalTrieReader) Storage(addr common.Address, key common.Hash) (common.Hash, error) { + r.lock.Lock() + defer r.lock.Unlock() + + tr, found := r.subTries[addr] + if !found { + root, ok := r.subRoots[addr] + + // The storage slot is accessed without account caching. It's unexpected + // behavior but try to resolve the account first anyway. + if !ok { + _, err := r.account(addr) + if err != nil { + return common.Hash{}, err + } + root = r.subRoots[addr] + } + var err error + tr, err = trie.NewStateTrie(trie.StorageTrieID(r.root, crypto.Keccak256Hash(addr.Bytes()), root), r.opener) + if err != nil { + return common.Hash{}, err + } + r.subTries[addr] = tr + } + ret, err := tr.GetStorage(addr, key.Bytes()) + if err != nil { + return common.Hash{}, err + } + var value common.Hash + value.SetBytes(ret) + return value, nil +} + // HistoricDB is the implementation of Database interface, with the ability to // access historical state. type HistoricDB struct { - disk ethdb.KeyValueStore - wasmdb ethdb.KeyValueStore - triedb *triedb.Database - codeCache *lru.SizeConstrainedCache[common.Hash, []byte] - codeSizeCache *lru.Cache[common.Hash, int] - pointCache *utils.PointCache + disk ethdb.KeyValueStore + wasmdb ethdb.KeyValueStore + triedb *triedb.Database + codedb *CodeDB } // NewHistoricDatabase creates a historic state database. -func NewHistoricDatabase(disk ethdb.KeyValueStore, triedb *triedb.Database) *HistoricDB { +func NewHistoricDatabase(triedb *triedb.Database, codedb *CodeDB) *HistoricDB { + if codedb == nil { + codedb = NewCodeDB(triedb.Disk()) + } wasmdb := triedb.Disk().WasmDataBase() return &HistoricDB{ - disk: disk, - wasmdb: wasmdb, - triedb: triedb, - codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), - codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), - pointCache: utils.NewPointCache(pointCacheSize), + disk: triedb.Disk(), + wasmdb: wasmdb, + triedb: triedb, + codedb: codedb, } } // Reader implements Database interface, returning a reader of the specific state. func (db *HistoricDB) Reader(stateRoot common.Hash) (Reader, error) { - hr, err := db.triedb.HistoricReader(stateRoot) + var readers []StateReader + sr, err := db.triedb.HistoricStateReader(stateRoot) + if err == nil { + readers = append(readers, newHistoricStateReader(sr)) + } + nr, err := db.triedb.HistoricNodeReader(stateRoot) + if err == nil { + tr, err := newHistoricalTrieReader(stateRoot, nr) + if err == nil { + readers = append(readers, tr) + } + } + if len(readers) == 0 { + return nil, fmt.Errorf("historical state %x is not available", stateRoot) + } + combined, err := newMultiStateReader(readers...) if err != nil { return nil, err } - return newReader(newCachingCodeReader(db.disk, db.codeCache, db.codeSizeCache), newHistoricReader(hr)), nil + return newReader(db.codedb.Reader(), combined), nil } // OpenTrie opens the main account trie. It's not supported by historic database. func (db *HistoricDB) OpenTrie(root common.Hash) (Trie, error) { - return nil, errors.New("not implemented") + nr, err := db.triedb.HistoricNodeReader(root) + if err != nil { + return nil, err + } + tr, err := trie.NewStateTrie(trie.StateTrieID(root), newHistoricTrieOpener(root, nr)) + if err != nil { + return nil, err + } + return tr, nil } // OpenStorageTrie opens the storage trie of an account. It's not supported by // historic database. -func (db *HistoricDB) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash, trie Trie) (Trie, error) { - return nil, errors.New("not implemented") -} - -// PointCache returns the cache holding points used in verkle tree key computation -func (db *HistoricDB) PointCache() *utils.PointCache { - return db.pointCache +func (db *HistoricDB) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash, _ Trie) (Trie, error) { + nr, err := db.triedb.HistoricNodeReader(stateRoot) + if err != nil { + return nil, err + } + id := trie.StorageTrieID(stateRoot, crypto.Keccak256Hash(address.Bytes()), root) + tr, err := trie.NewStateTrie(id, newHistoricTrieOpener(stateRoot, nr)) + if err != nil { + return nil, err + } + return tr, nil } // TrieDB returns the underlying trie database for managing trie nodes. @@ -170,3 +316,10 @@ func (db *HistoricDB) WasmStore() ethdb.KeyValueStore { func (db *HistoricDB) DiskDB() ethdb.KeyValueStore { return db.disk } + +// Commit flushes all pending writes and finalizes the state transition, +// committing the changes to the underlying storage. It returns an error +// if the commit fails. +func (db *HistoricDB) Commit(update *stateUpdate) error { + return errors.New("not implemented") +} diff --git a/core/state/dump.go b/core/state/dump.go index a4abc33733..829d106ed3 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -18,6 +18,7 @@ package state import ( "encoding/json" + "errors" "fmt" "time" @@ -27,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/bintrie" ) // DumpConfig is a set of options to control what portions of the state will be @@ -221,6 +223,28 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] return nextKey } +// DumpBinTrieLeaves collects all binary trie leaf nodes into the provided map. +func (s *StateDB) DumpBinTrieLeaves(collector map[common.Hash]hexutil.Bytes) error { + tr, err := s.db.OpenTrie(s.originalRoot) + if err != nil { + return err + } + btr, ok := tr.(*bintrie.BinaryTrie) + if !ok { + return errors.New("trie is not a binary trie") + } + it, err := btr.NodeIterator(nil) + if err != nil { + return err + } + for it.Next(true) { + if it.Leaf() { + collector[common.BytesToHash(it.LeafKey())] = it.LeafBlob() + } + } + return nil +} + // RawDump returns the state. If the processing is aborted e.g. due to options // reaching Max, the `Next` key is set on the returned Dump. func (s *StateDB) RawDump(opts *DumpConfig) Dump { diff --git a/core/state/iterator.go b/core/state/iterator.go index 0abae091d9..0050a840d8 100644 --- a/core/state/iterator.go +++ b/core/state/iterator.go @@ -144,10 +144,7 @@ func (it *nodeIterator) step() error { } if !bytes.Equal(account.CodeHash, types.EmptyCodeHash.Bytes()) { it.codeHash = common.BytesToHash(account.CodeHash) - it.code, err = it.state.reader.Code(address, common.BytesToHash(account.CodeHash)) - if err != nil { - return fmt.Errorf("code %x: %v", account.CodeHash, err) - } + it.code = it.state.reader.Code(address, common.BytesToHash(account.CodeHash)) if len(it.code) == 0 { return fmt.Errorf("code is not found: %x", account.CodeHash) } diff --git a/core/state/journal.go b/core/state/journal.go index 28af9317f3..89877662f9 100644 --- a/core/state/journal.go +++ b/core/state/journal.go @@ -43,7 +43,8 @@ type journalEntry interface { revert(*StateDB) // dirtied returns the Ethereum address modified by this journal entry. - dirtied() *common.Address + // indicates false if no address was changed. + dirtied() (common.Address, bool) // copy returns a deep-copied journal entry. copy() journalEntry @@ -110,11 +111,11 @@ func (j *journal) revertToSnapshot(revid int, s *StateDB) { // append inserts a new modification entry to the end of the change journal. func (j *journal) append(entry journalEntry) { j.entries = append(j.entries, entry) - if addr := entry.dirtied(); addr != nil { - j.dirties[*addr]++ + if addr, dirty := entry.dirtied(); dirty { + j.dirties[addr]++ // Arbitrum: also track the number of zombie changes if isZombie(entry) { - j.zombieEntries[*addr]++ + j.zombieEntries[addr]++ } } } @@ -127,14 +128,14 @@ func (j *journal) revert(statedb *StateDB, snapshot int) { j.entries[i].revert(statedb) // Drop any dirty tracking induced by the change - if addr := j.entries[i].dirtied(); addr != nil { - if j.dirties[*addr]--; j.dirties[*addr] == 0 { - delete(j.dirties, *addr) + if addr, dirty := j.entries[i].dirtied(); dirty { + if j.dirties[addr]--; j.dirties[addr] == 0 { + delete(j.dirties, addr) // Revert zombieEntries tracking if isZombie(j.entries[i]) { - if j.zombieEntries[*addr]--; j.zombieEntries[*addr] == 0 { - delete(j.zombieEntries, *addr) + if j.zombieEntries[addr]--; j.zombieEntries[addr] == 0 { + delete(j.zombieEntries, addr) } } } @@ -323,8 +324,8 @@ func (ch createObjectChange) revert(s *StateDB) { delete(s.stateObjects, ch.account) } -func (ch createObjectChange) dirtied() *common.Address { - return &ch.account +func (ch createObjectChange) dirtied() (common.Address, bool) { + return ch.account, true } func (ch createObjectChange) copy() journalEntry { @@ -337,8 +338,8 @@ func (ch createContractChange) revert(s *StateDB) { s.getStateObject(ch.account).newContract = false } -func (ch createContractChange) dirtied() *common.Address { - return nil +func (ch createContractChange) dirtied() (common.Address, bool) { + return common.Address{}, false } func (ch createContractChange) copy() journalEntry { @@ -354,8 +355,8 @@ func (ch selfDestructChange) revert(s *StateDB) { } } -func (ch selfDestructChange) dirtied() *common.Address { - return &ch.account +func (ch selfDestructChange) dirtied() (common.Address, bool) { + return ch.account, true } func (ch selfDestructChange) copy() journalEntry { @@ -369,8 +370,8 @@ var ripemd = common.HexToAddress("0000000000000000000000000000000000000003") func (ch touchChange) revert(s *StateDB) { } -func (ch touchChange) dirtied() *common.Address { - return &ch.account +func (ch touchChange) dirtied() (common.Address, bool) { + return ch.account, true } func (ch touchChange) copy() journalEntry { @@ -383,8 +384,8 @@ func (ch balanceChange) revert(s *StateDB) { s.getStateObject(ch.account).setBalance(ch.prev) } -func (ch balanceChange) dirtied() *common.Address { - return &ch.account +func (ch balanceChange) dirtied() (common.Address, bool) { + return ch.account, true } func (ch balanceChange) copy() journalEntry { @@ -398,8 +399,8 @@ func (ch nonceChange) revert(s *StateDB) { s.getStateObject(ch.account).setNonce(ch.prev) } -func (ch nonceChange) dirtied() *common.Address { - return &ch.account +func (ch nonceChange) dirtied() (common.Address, bool) { + return ch.account, true } func (ch nonceChange) copy() journalEntry { @@ -413,8 +414,8 @@ func (ch codeChange) revert(s *StateDB) { s.getStateObject(ch.account).setCode(crypto.Keccak256Hash(ch.prevCode), ch.prevCode) } -func (ch codeChange) dirtied() *common.Address { - return &ch.account +func (ch codeChange) dirtied() (common.Address, bool) { + return ch.account, true } func (ch codeChange) copy() journalEntry { @@ -428,8 +429,8 @@ func (ch storageChange) revert(s *StateDB) { s.getStateObject(ch.account).setState(ch.key, ch.prevvalue, ch.origvalue) } -func (ch storageChange) dirtied() *common.Address { - return &ch.account +func (ch storageChange) dirtied() (common.Address, bool) { + return ch.account, true } func (ch storageChange) copy() journalEntry { @@ -445,8 +446,8 @@ func (ch transientStorageChange) revert(s *StateDB) { s.setTransientState(ch.account, ch.key, ch.prevalue) } -func (ch transientStorageChange) dirtied() *common.Address { - return nil +func (ch transientStorageChange) dirtied() (common.Address, bool) { + return common.Address{}, false } func (ch transientStorageChange) copy() journalEntry { @@ -461,8 +462,8 @@ func (ch refundChange) revert(s *StateDB) { s.refund = ch.prev } -func (ch refundChange) dirtied() *common.Address { - return nil +func (ch refundChange) dirtied() (common.Address, bool) { + return common.Address{}, false } func (ch refundChange) copy() journalEntry { @@ -481,8 +482,8 @@ func (ch addLogChange) revert(s *StateDB) { s.logSize-- } -func (ch addLogChange) dirtied() *common.Address { - return nil +func (ch addLogChange) dirtied() (common.Address, bool) { + return common.Address{}, false } func (ch addLogChange) copy() journalEntry { @@ -504,8 +505,8 @@ func (ch accessListAddAccountChange) revert(s *StateDB) { s.accessList.DeleteAddress(ch.address) } -func (ch accessListAddAccountChange) dirtied() *common.Address { - return nil +func (ch accessListAddAccountChange) dirtied() (common.Address, bool) { + return common.Address{}, false } func (ch accessListAddAccountChange) copy() journalEntry { @@ -518,8 +519,8 @@ func (ch accessListAddSlotChange) revert(s *StateDB) { s.accessList.DeleteSlot(ch.address, ch.slot) } -func (ch accessListAddSlotChange) dirtied() *common.Address { - return nil +func (ch accessListAddSlotChange) dirtied() (common.Address, bool) { + return common.Address{}, false } func (ch accessListAddSlotChange) copy() journalEntry { diff --git a/core/state/journal_arbitrum.go b/core/state/journal_arbitrum.go index 1561b4304e..039d4701fd 100644 --- a/core/state/journal_arbitrum.go +++ b/core/state/journal_arbitrum.go @@ -13,8 +13,8 @@ func (ch wasmActivation) revert(s *StateDB) { delete(s.arbExtraData.activatedWasms, ch.moduleHash) } -func (ch wasmActivation) dirtied() *common.Address { - return nil +func (ch wasmActivation) dirtied() (common.Address, bool) { + return common.Address{}, false } func (ch wasmActivation) copy() journalEntry { @@ -38,8 +38,8 @@ func (ch CacheWasm) revert(s *StateDB) { EvictWasmRust(ch.ModuleHash, ch.Version, ch.Tag, ch.Debug) } -func (ch CacheWasm) dirtied() *common.Address { - return nil +func (ch CacheWasm) dirtied() (common.Address, bool) { + return common.Address{}, false } func (ch CacheWasm) copy() journalEntry { @@ -67,8 +67,8 @@ func (ch EvictWasm) revert(s *StateDB) { } } -func (ch EvictWasm) dirtied() *common.Address { - return nil +func (ch EvictWasm) dirtied() (common.Address, bool) { + return common.Address{}, false } func (ch EvictWasm) copy() journalEntry { @@ -96,8 +96,11 @@ func (ch createZombieChange) revert(s *StateDB) { delete(s.stateObjects, *ch.account) } -func (ch createZombieChange) dirtied() *common.Address { - return ch.account +func (ch createZombieChange) dirtied() (common.Address, bool) { + if ch.account == nil { + return common.Address{}, false + } + return *ch.account, true } func (ch createZombieChange) copy() journalEntry { diff --git a/core/state/reader.go b/core/state/reader.go index 3e8b31b6be..fe0ec71f2d 100644 --- a/core/state/reader.go +++ b/core/state/reader.go @@ -22,41 +22,38 @@ import ( "sync/atomic" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/core/overlay" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/utils" + "github.com/ethereum/go-ethereum/trie/bintrie" + "github.com/ethereum/go-ethereum/trie/transitiontrie" "github.com/ethereum/go-ethereum/triedb" "github.com/ethereum/go-ethereum/triedb/database" ) // ContractCodeReader defines the interface for accessing contract code. +// +// ContractCodeReader is supposed to be thread-safe. type ContractCodeReader interface { - // Code retrieves a particular contract's code. - // - // - Returns nil code along with nil error if the requested contract code - // doesn't exist - // - Returns an error only if an unexpected issue occurs - Code(addr common.Address, codeHash common.Hash) ([]byte, error) + // Has returns the flag indicating whether the contract code with + // specified address and hash exists or not. + Has(addr common.Address, codeHash common.Hash) bool - // CodeSize retrieves a particular contracts code's size. - // - // - Returns zero code size along with nil error if the requested contract code - // doesn't exist - // - Returns an error only if an unexpected issue occurs - CodeSize(addr common.Address, codeHash common.Hash) (int, error) + // Code retrieves a particular contract's code. Returns nil code if the + // requested contract code doesn't exist. + Code(addr common.Address, codeHash common.Hash) []byte + + // CodeSize retrieves a particular contracts code's size. Returns zero code + // size if the requested contract code doesn't exist. + CodeSize(addr common.Address, codeHash common.Hash) int } // StateReader defines the interface for accessing accounts and storage slots // associated with a specific state. // -// StateReader is assumed to be thread-safe and implementation must take care -// of the concurrency issue by themselves. +// StateReader is supposed to be thread-safe. type StateReader interface { // Account retrieves the account associated with a particular address. // @@ -84,70 +81,6 @@ type Reader interface { StateReader } -// ReaderStats wraps the statistics of reader. -type ReaderStats struct { - AccountHit int64 - AccountMiss int64 - StorageHit int64 - StorageMiss int64 -} - -// ReaderWithStats wraps the additional method to retrieve the reader statistics from. -type ReaderWithStats interface { - Reader - GetStats() ReaderStats -} - -// cachingCodeReader implements ContractCodeReader, accessing contract code either in -// local key-value store or the shared code cache. -// -// cachingCodeReader is safe for concurrent access. -type cachingCodeReader struct { - db ethdb.KeyValueReader - - // These caches could be shared by multiple code reader instances, - // they are natively thread-safe. - codeCache *lru.SizeConstrainedCache[common.Hash, []byte] - codeSizeCache *lru.Cache[common.Hash, int] -} - -// newCachingCodeReader constructs the code reader. -func newCachingCodeReader(db ethdb.KeyValueReader, codeCache *lru.SizeConstrainedCache[common.Hash, []byte], codeSizeCache *lru.Cache[common.Hash, int]) *cachingCodeReader { - return &cachingCodeReader{ - db: db, - codeCache: codeCache, - codeSizeCache: codeSizeCache, - } -} - -// Code implements ContractCodeReader, retrieving a particular contract's code. -// If the contract code doesn't exist, no error will be returned. -func (r *cachingCodeReader) Code(addr common.Address, codeHash common.Hash) ([]byte, error) { - code, _ := r.codeCache.Get(codeHash) - if len(code) > 0 { - return code, nil - } - code = rawdb.ReadCode(r.db, codeHash) - if len(code) > 0 { - r.codeCache.Add(codeHash, code) - r.codeSizeCache.Add(codeHash, len(code)) - } - return code, nil -} - -// CodeSize implements ContractCodeReader, retrieving a particular contracts code's size. -// If the contract code doesn't exist, no error will be returned. -func (r *cachingCodeReader) CodeSize(addr common.Address, codeHash common.Hash) (int, error) { - if cached, ok := r.codeSizeCache.Get(codeHash); ok { - return cached, nil - } - code, err := r.Code(addr, codeHash) - if err != nil { - return 0, err - } - return len(code), nil -} - // flatReader wraps a database state reader and is safe for concurrent access. type flatReader struct { reader database.StateReader @@ -165,7 +98,7 @@ func newFlatReader(reader database.StateReader) *flatReader { // // The returned account might be nil if it's not existent. func (r *flatReader) Account(addr common.Address) (*types.StateAccount, error) { - account, err := r.reader.Account(crypto.Keccak256Hash(addr.Bytes())) + account, err := r.reader.Account(crypto.Keccak256Hash(addr[:])) if err != nil { return nil, err } @@ -195,8 +128,8 @@ func (r *flatReader) Account(addr common.Address) (*types.StateAccount, error) { // // The returned storage slot might be empty if it's not existent. func (r *flatReader) Storage(addr common.Address, key common.Hash) (common.Hash, error) { - addrHash := crypto.Keccak256Hash(addr.Bytes()) - slotHash := crypto.Keccak256Hash(key.Bytes()) + addrHash := crypto.Keccak256Hash(addr[:]) + slotHash := crypto.Keccak256Hash(key[:]) ret, err := r.reader.Storage(addrHash, slotHash) if err != nil { return common.Hash{}, err @@ -234,7 +167,7 @@ type trieReader struct { // newTrieReader constructs a trie reader of the specific state. An error will be // returned if the associated trie specified by root is not existent. -func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCache) (*trieReader, error) { +func newTrieReader(root common.Hash, db *triedb.Database) (*trieReader, error) { var ( tr Trie err error @@ -242,7 +175,11 @@ func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCach if !db.IsVerkle() { tr, err = trie.NewStateTrie(trie.StateTrieID(root), db) } else { - tr, err = trie.NewVerkleTrie(root, db, cache) + // When IsVerkle() is true, create a BinaryTrie wrapped in TransitionTrie + binTrie, binErr := bintrie.NewBinaryTrie(root, db) + if binErr != nil { + return nil, binErr + } // Based on the transition status, determine if the overlay // tree needs to be created, or if a single, target tree is @@ -253,7 +190,22 @@ func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCach if err != nil { return nil, err } - tr = trie.NewTransitionTrie(mpt, tr.(*trie.VerkleTrie), false) + tr = transitiontrie.NewTransitionTrie(mpt, binTrie, false) + } else { + // HACK: Use TransitionTrie with nil base as a wrapper to make BinaryTrie + // satisfy the Trie interface. This works around the import cycle between + // trie and trie/bintrie packages. + // + // TODO: In future PRs, refactor the package structure to avoid this hack: + // - Option 1: Move common interfaces (Trie, NodeIterator) to a separate + // package that both trie and trie/bintrie can import + // - Option 2: Create a factory function in the trie package that returns + // BinaryTrie as a Trie interface without direct import + // - Option 3: Move BinaryTrie to the main trie package + // + // The current approach works but adds unnecessary overhead and complexity + // by using TransitionTrie when there's no actual transition happening. + tr = transitiontrie.NewTransitionTrie(nil, binTrie, false) } } if err != nil { @@ -397,24 +349,10 @@ func (r *multiStateReader) Storage(addr common.Address, slot common.Hash) (commo return common.Hash{}, errors.Join(errs...) } -// reader is the wrapper of ContractCodeReader and StateReader interface. -type reader struct { - ContractCodeReader +// stateReaderWithCache is a wrapper around StateReader that maintains additional +// state caches to support concurrent state access. +type stateReaderWithCache struct { StateReader -} - -// newReader constructs a reader with the supplied code reader and state reader. -func newReader(codeReader ContractCodeReader, stateReader StateReader) *reader { - return &reader{ - ContractCodeReader: codeReader, - StateReader: stateReader, - } -} - -// readerWithCache is a wrapper around Reader that maintains additional state caches -// to support concurrent state access. -type readerWithCache struct { - Reader // safe for concurrent read // Previously resolved state entries. accounts map[common.Address]*types.StateAccount @@ -430,11 +368,11 @@ type readerWithCache struct { } } -// newReaderWithCache constructs the reader with local cache. -func newReaderWithCache(reader Reader) *readerWithCache { - r := &readerWithCache{ - Reader: reader, - accounts: make(map[common.Address]*types.StateAccount), +// newStateReaderWithCache constructs the state reader with local cache. +func newStateReaderWithCache(sr StateReader) *stateReaderWithCache { + r := &stateReaderWithCache{ + StateReader: sr, + accounts: make(map[common.Address]*types.StateAccount), } for i := range r.storageBuckets { r.storageBuckets[i].storages = make(map[common.Address]map[common.Hash]common.Hash) @@ -447,7 +385,7 @@ func newReaderWithCache(reader Reader) *readerWithCache { // might be nil if it's not existent. // // An error will be returned if the state is corrupted in the underlying reader. -func (r *readerWithCache) account(addr common.Address) (*types.StateAccount, bool, error) { +func (r *stateReaderWithCache) account(addr common.Address) (*types.StateAccount, bool, error) { // Try to resolve the requested account in the local cache r.accountLock.RLock() acct, ok := r.accounts[addr] @@ -456,7 +394,7 @@ func (r *readerWithCache) account(addr common.Address) (*types.StateAccount, boo return acct, true, nil } // Try to resolve the requested account from the underlying reader - acct, err := r.Reader.Account(addr) + acct, err := r.StateReader.Account(addr) if err != nil { return nil, false, err } @@ -470,7 +408,7 @@ func (r *readerWithCache) account(addr common.Address) (*types.StateAccount, boo // The returned account might be nil if it's not existent. // // An error will be returned if the state is corrupted in the underlying reader. -func (r *readerWithCache) Account(addr common.Address) (*types.StateAccount, error) { +func (r *stateReaderWithCache) Account(addr common.Address) (*types.StateAccount, error) { account, _, err := r.account(addr) return account, err } @@ -478,7 +416,7 @@ func (r *readerWithCache) Account(addr common.Address) (*types.StateAccount, err // storage retrieves the storage slot specified by the address and slot key, along // with a flag indicating whether it's found in the cache or not. The returned // storage slot might be empty if it's not existent. -func (r *readerWithCache) storage(addr common.Address, slot common.Hash) (common.Hash, bool, error) { +func (r *stateReaderWithCache) storage(addr common.Address, slot common.Hash) (common.Hash, bool, error) { var ( value common.Hash ok bool @@ -495,7 +433,7 @@ func (r *readerWithCache) storage(addr common.Address, slot common.Hash) (common return value, true, nil } // Try to resolve the requested storage slot from the underlying reader - value, err := r.Reader.Storage(addr, slot) + value, err := r.StateReader.Storage(addr, slot) if err != nil { return common.Hash{}, false, err } @@ -516,23 +454,26 @@ func (r *readerWithCache) storage(addr common.Address, slot common.Hash) (common // existent. // // An error will be returned if the state is corrupted in the underlying reader. -func (r *readerWithCache) Storage(addr common.Address, slot common.Hash) (common.Hash, error) { +func (r *stateReaderWithCache) Storage(addr common.Address, slot common.Hash) (common.Hash, error) { value, _, err := r.storage(addr, slot) return value, err } -type readerWithCacheStats struct { - *readerWithCache - accountHit atomic.Int64 - accountMiss atomic.Int64 - storageHit atomic.Int64 - storageMiss atomic.Int64 +// stateReaderWithStats is a wrapper over the stateReaderWithCache, tracking +// the cache hit statistics of the reader. +type stateReaderWithStats struct { + *stateReaderWithCache + + accountCacheHit atomic.Int64 + accountCacheMiss atomic.Int64 + storageCacheHit atomic.Int64 + storageCacheMiss atomic.Int64 } -// newReaderWithCacheStats constructs the reader with additional statistics tracked. -func newReaderWithCacheStats(reader *readerWithCache) *readerWithCacheStats { - return &readerWithCacheStats{ - readerWithCache: reader, +// newReaderWithStats constructs the state reader with additional statistics tracked. +func newStateReaderWithStats(sr *stateReaderWithCache) *stateReaderWithStats { + return &stateReaderWithStats{ + stateReaderWithCache: sr, } } @@ -540,15 +481,15 @@ func newReaderWithCacheStats(reader *readerWithCache) *readerWithCacheStats { // The returned account might be nil if it's not existent. // // An error will be returned if the state is corrupted in the underlying reader. -func (r *readerWithCacheStats) Account(addr common.Address) (*types.StateAccount, error) { - account, incache, err := r.readerWithCache.account(addr) +func (r *stateReaderWithStats) Account(addr common.Address) (*types.StateAccount, error) { + account, incache, err := r.stateReaderWithCache.account(addr) if err != nil { return nil, err } if incache { - r.accountHit.Add(1) + r.accountCacheHit.Add(1) } else { - r.accountMiss.Add(1) + r.accountCacheMiss.Add(1) } return account, nil } @@ -558,25 +499,64 @@ func (r *readerWithCacheStats) Account(addr common.Address) (*types.StateAccount // existent. // // An error will be returned if the state is corrupted in the underlying reader. -func (r *readerWithCacheStats) Storage(addr common.Address, slot common.Hash) (common.Hash, error) { - value, incache, err := r.readerWithCache.storage(addr, slot) +func (r *stateReaderWithStats) Storage(addr common.Address, slot common.Hash) (common.Hash, error) { + value, incache, err := r.stateReaderWithCache.storage(addr, slot) if err != nil { return common.Hash{}, err } if incache { - r.storageHit.Add(1) + r.storageCacheHit.Add(1) } else { - r.storageMiss.Add(1) + r.storageCacheMiss.Add(1) } return value, nil } -// GetStats implements ReaderWithStats, returning the statistics of state reader. -func (r *readerWithCacheStats) GetStats() ReaderStats { +// GetStateStats implements StateReaderStater, returning the statistics of the +// state reader. +func (r *stateReaderWithStats) GetStateStats() StateReaderStats { + return StateReaderStats{ + AccountCacheHit: r.accountCacheHit.Load(), + AccountCacheMiss: r.accountCacheMiss.Load(), + StorageCacheHit: r.storageCacheHit.Load(), + StorageCacheMiss: r.storageCacheMiss.Load(), + } +} + +// reader aggregates a code reader and a state reader into a single object. +type reader struct { + ContractCodeReader + StateReader +} + +// newReader constructs a reader with the supplied code reader and state reader. +func newReader(codeReader ContractCodeReader, stateReader StateReader) *reader { + return &reader{ + ContractCodeReader: codeReader, + StateReader: stateReader, + } +} + +// GetCodeStats returns the statistics of code access. +func (r *reader) GetCodeStats() ContractCodeReaderStats { + if stater, ok := r.ContractCodeReader.(ContractCodeReaderStater); ok { + return stater.GetCodeStats() + } + return ContractCodeReaderStats{} +} + +// GetStateStats returns the statistics of state access. +func (r *reader) GetStateStats() StateReaderStats { + if stater, ok := r.StateReader.(StateReaderStater); ok { + return stater.GetStateStats() + } + return StateReaderStats{} +} + +// GetStats returns the aggregated statistics for both state and code access. +func (r *reader) GetStats() ReaderStats { return ReaderStats{ - AccountHit: r.accountHit.Load(), - AccountMiss: r.accountMiss.Load(), - StorageHit: r.storageHit.Load(), - StorageMiss: r.storageMiss.Load(), + CodeStats: r.GetCodeStats(), + StateStats: r.GetStateStats(), } } diff --git a/core/state/reader_stater.go b/core/state/reader_stater.go new file mode 100644 index 0000000000..5294275953 --- /dev/null +++ b/core/state/reader_stater.go @@ -0,0 +1,82 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +// ContractCodeReaderStats aggregates statistics for the contract code reader. +type ContractCodeReaderStats struct { + CacheHit int64 // Number of cache hits + CacheMiss int64 // Number of cache misses + CacheHitBytes int64 // Total bytes served from cache + CacheMissBytes int64 // Total bytes read on cache misses +} + +// HitRate returns the cache hit rate in percentage. +func (s ContractCodeReaderStats) HitRate() float64 { + total := s.CacheHit + s.CacheMiss + if total == 0 { + return 0 + } + return float64(s.CacheHit) / float64(total) * 100 +} + +// ContractCodeReaderStater wraps the method to retrieve the statistics of +// contract code reader. +type ContractCodeReaderStater interface { + GetCodeStats() ContractCodeReaderStats +} + +// StateReaderStats aggregates statistics for the state reader. +type StateReaderStats struct { + AccountCacheHit int64 // Number of account cache hits + AccountCacheMiss int64 // Number of account cache misses + StorageCacheHit int64 // Number of storage cache hits + StorageCacheMiss int64 // Number of storage cache misses +} + +// AccountCacheHitRate returns the cache hit rate of account requests in percentage. +func (s StateReaderStats) AccountCacheHitRate() float64 { + total := s.AccountCacheHit + s.AccountCacheMiss + if total == 0 { + return 0 + } + return float64(s.AccountCacheHit) / float64(total) * 100 +} + +// StorageCacheHitRate returns the cache hit rate of storage requests in percentage. +func (s StateReaderStats) StorageCacheHitRate() float64 { + total := s.StorageCacheHit + s.StorageCacheMiss + if total == 0 { + return 0 + } + return float64(s.StorageCacheHit) / float64(total) * 100 +} + +// StateReaderStater wraps the method to retrieve the statistics of state reader. +type StateReaderStater interface { + GetStateStats() StateReaderStats +} + +// ReaderStats wraps the statistics of reader. +type ReaderStats struct { + CodeStats ContractCodeReaderStats + StateStats StateReaderStats +} + +// ReaderStater defines the capability to retrieve aggregated statistics. +type ReaderStater interface { + GetStats() ReaderStats +} diff --git a/core/state/snapshot/difflayer.go b/core/state/snapshot/difflayer.go index 28957051d4..1286ded7e1 100644 --- a/core/state/snapshot/difflayer.go +++ b/core/state/snapshot/difflayer.go @@ -465,6 +465,6 @@ func (dl *diffLayer) StorageList(accountHash common.Hash) []common.Hash { storageList := slices.SortedFunc(maps.Keys(dl.storageData[accountHash]), common.Hash.Cmp) dl.storageList[accountHash] = storageList - dl.memory += uint64(len(dl.storageList)*common.HashLength + common.HashLength) + dl.memory += uint64(len(storageList)*common.HashLength + common.HashLength) return storageList } diff --git a/core/state/snapshot/difflayer_test.go b/core/state/snapshot/difflayer_test.go index 2c868b3010..90a265645d 100644 --- a/core/state/snapshot/difflayer_test.go +++ b/core/state/snapshot/difflayer_test.go @@ -198,6 +198,39 @@ func TestInsertAndMerge(t *testing.T) { } } +// TestStorageListMemoryAccounting ensures that StorageList increases dl.memory +// proportionally to the number of storage slots in the requested account and +// does not change memory usage on repeated calls for the same account. +func TestStorageListMemoryAccounting(t *testing.T) { + parent := newDiffLayer(emptyLayer(), common.Hash{}, nil, nil) + account := common.HexToHash("0x01") + + slots := make(map[common.Hash][]byte) + for i := 0; i < 3; i++ { + slots[randomHash()] = []byte{0x01} + } + storage := map[common.Hash]map[common.Hash][]byte{ + account: slots, + } + dl := newDiffLayer(parent, common.Hash{}, nil, storage) + + before := dl.memory + list := dl.StorageList(account) + if have, want := len(list), len(slots); have != want { + t.Fatalf("StorageList length mismatch: have %d, want %d", have, want) + } + expectedDelta := uint64(len(list)*common.HashLength + common.HashLength) + if have, want := dl.memory-before, expectedDelta; have != want { + t.Fatalf("StorageList memory delta mismatch: have %d, want %d", have, want) + } + + before = dl.memory + _ = dl.StorageList(account) + if dl.memory != before { + t.Fatalf("StorageList changed memory on cached call: have %d, want %d", dl.memory, before) + } +} + func emptyLayer() *diskLayer { return &diskLayer{ diskdb: memorydb.New(), diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 644e624ea5..f0b13d4ba7 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -22,10 +22,10 @@ import ( "testing" "time" - "github.com/ethereum/go-ethereum/arbcrypto" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" @@ -38,12 +38,7 @@ import ( ) func hashData(input []byte) common.Hash { - var hasher = arbcrypto.NewLegacyKeccak256() - var hash common.Hash - hasher.Reset() - hasher.Write(input) - hasher.Sum(hash[:0]) - return hash + return crypto.Keccak256Hash(input) } // Tests that snapshot generation from an empty database. diff --git a/core/state/state_object.go b/core/state/state_object.go index 28ffcac735..2c7448e91a 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -30,6 +30,8 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/bintrie" + "github.com/ethereum/go-ethereum/trie/transitiontrie" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/holiman/uint256" ) @@ -47,11 +49,11 @@ func (s Storage) Copy() Storage { // - Account values as well as storages can be accessed and modified through the object. // - Finally, call commit to return the changes of storage trie and update account data. type stateObject struct { - db *StateDB - address common.Address // address of ethereum account - addrHash common.Hash // hash of ethereum address of the account - origin *types.StateAccount // Account original data without any change applied, nil means it was not existent - data types.StateAccount // Account data with all mutations applied in the scope of block + db *StateDB + address common.Address // address of ethereum account + addressHash *common.Hash // hash of ethereum address of the account + origin *types.StateAccount // Account original data without any change applied, nil means it was not existent + data types.StateAccount // Account data with all mutations applied in the scope of block // Write caches. trie Trie // storage trie, which becomes non-nil on first access @@ -101,7 +103,6 @@ func newObject(db *StateDB, address common.Address, acct *types.StateAccount) *s return &stateObject{ db: db, address: address, - addrHash: crypto.Keccak256Hash(address[:]), origin: origin, data: *acct, originStorage: make(Storage), @@ -111,6 +112,14 @@ func newObject(db *StateDB, address common.Address, acct *types.StateAccount) *s } } +func (s *stateObject) addrHash() common.Hash { + if s.addressHash == nil { + h := crypto.Keccak256Hash(s.address[:]) + s.addressHash = &h + } + return *s.addressHash +} + func (s *stateObject) markSelfdestructed() { s.selfDestructed = true } @@ -150,7 +159,7 @@ func (s *stateObject) getPrefetchedTrie() Trie { return nil } // Attempt to retrieve the trie from the prefetcher - return s.db.prefetcher.trie(s.addrHash, s.data.Root) + return s.db.prefetcher.trie(s.addrHash(), s.data.Root) } // GetState retrieves a value associated with the given storage key. @@ -187,6 +196,19 @@ func (s *stateObject) GetCommittedState(key common.Hash) common.Hash { // have been handles via pendingStorage above. // 2) we don't have new values, and can deliver empty response back if _, destructed := s.db.stateObjectsDestruct[s.address]; destructed { + // Invoke the reader regardless and discard the returned value. + // The returned value may not be empty, as it could belong to a + // self-destructed contract. + // + // The read operation is still essential for correctly building + // the block-level access list. + // + // TODO(rjl493456442) the reader interface can be extended with + // Touch, recording the read access without the actual disk load. + _, err := s.db.reader.Storage(s.address, key) + if err != nil { + s.db.setError(err) + } s.originStorage[key] = common.Hash{} // track the empty slot as origin value return common.Hash{} } @@ -202,7 +224,7 @@ func (s *stateObject) GetCommittedState(key common.Hash) common.Hash { // Schedule the resolved storage slots for prefetching if it's enabled. if s.db.prefetcher != nil && s.data.Root != types.EmptyRootHash { - if err = s.db.prefetcher.prefetch(s.addrHash, s.origin.Root, s.address, nil, []common.Hash{key}, true); err != nil { + if err = s.db.prefetcher.prefetch(s.addrHash(), s.origin.Root, s.address, nil, []common.Hash{key}, true); err != nil { log.Error("Failed to prefetch storage slot", "addr", s.address, "key", key, "err", err) } } @@ -263,7 +285,7 @@ func (s *stateObject) finalise() { s.pendingStorage[key] = value } if s.db.prefetcher != nil && len(slotsToPrefetch) > 0 && s.data.Root != types.EmptyRootHash { - if err := s.db.prefetcher.prefetch(s.addrHash, s.data.Root, s.address, nil, slotsToPrefetch, false); err != nil { + if err := s.db.prefetcher.prefetch(s.addrHash(), s.data.Root, s.address, nil, slotsToPrefetch, false); err != nil { log.Error("Failed to prefetch slots", "addr", s.address, "slots", len(slotsToPrefetch), "err", err) } } @@ -361,7 +383,7 @@ func (s *stateObject) updateTrie() (Trie, error) { s.db.StorageDeleted.Add(1) } if s.db.prefetcher != nil { - s.db.prefetcher.used(s.addrHash, s.data.Root, nil, used) + s.db.prefetcher.used(s.addrHash(), s.data.Root, nil, used) } s.uncommittedStorage = make(Storage) // empties the commit markers return tr, nil @@ -442,6 +464,12 @@ func (s *stateObject) commit() (*accountUpdate, *trienode.NodeSet, error) { blob: s.code, } s.dirtyCode = false // reset the dirty flag + + if s.origin == nil { + op.code.originHash = types.EmptyCodeHash + } else { + op.code.originHash = common.BytesToHash(s.origin.CodeHash) + } } // Commit storage changes and the associated storage trie s.commitStorage(op) @@ -450,6 +478,14 @@ func (s *stateObject) commit() (*accountUpdate, *trienode.NodeSet, error) { s.origin = s.data.Copy() return op, nil, nil } + // In Verkle/binary trie mode, all state objects share one unified trie. + // The main account trie commit in stateDB.commit() already calls + // CollectNodes on this trie, so calling Commit here again would + // redundantly traverse and serialize the entire tree per dirty account. + if s.db.GetTrie().IsVerkle() { + s.origin = s.data.Copy() + return op, nil, nil + } root, nodes := s.trie.Commit(false) s.data.Root = root s.origin = s.data.Copy() @@ -487,7 +523,7 @@ func (s *stateObject) deepCopy(db *StateDB) *stateObject { obj := &stateObject{ db: db, address: s.address, - addrHash: s.addrHash, + addressHash: nil, origin: s.origin, data: s.data, code: s.code, @@ -501,11 +537,11 @@ func (s *stateObject) deepCopy(db *StateDB) *stateObject { } switch s.trie.(type) { - case *trie.VerkleTrie: - // Verkle uses only one tree, and the copy has already been + case *bintrie.BinaryTrie: + // UBT uses only one tree, and the copy has already been // made in mustCopyTrie. obj.trie = db.trie - case *trie.TransitionTrie: + case *transitiontrie.TransitionTrie: // Same thing for the transition tree, since the MPT is // read-only. obj.trie = db.trie @@ -534,10 +570,13 @@ func (s *stateObject) Code() []byte { if bytes.Equal(s.CodeHash(), types.EmptyCodeHash.Bytes()) { return nil } - code, err := s.db.reader.Code(s.address, common.BytesToHash(s.CodeHash())) - if err != nil { - s.db.setError(fmt.Errorf("can't load code hash %x: %v", s.CodeHash(), err)) - } + defer func(start time.Time) { + s.db.CodeLoaded += 1 + s.db.CodeReads += time.Since(start) + s.db.CodeLoadBytes += len(s.code) + }(time.Now()) + + code := s.db.reader.Code(s.address, common.BytesToHash(s.CodeHash())) if len(code) == 0 { s.db.setError(fmt.Errorf("code is not found %x", s.CodeHash())) } @@ -555,10 +594,12 @@ func (s *stateObject) CodeSize() int { if bytes.Equal(s.CodeHash(), types.EmptyCodeHash.Bytes()) { return 0 } - size, err := s.db.reader.CodeSize(s.address, common.BytesToHash(s.CodeHash())) - if err != nil { - s.db.setError(fmt.Errorf("can't load code size %x: %v", s.CodeHash(), err)) - } + defer func(start time.Time) { + s.db.CodeLoaded += 1 + s.db.CodeReads += time.Since(start) + }(time.Now()) + + size := s.db.reader.CodeSize(s.address, common.BytesToHash(s.CodeHash())) if size == 0 { s.db.setError(fmt.Errorf("code is not found %x", s.CodeHash())) } diff --git a/core/state/state_sizer.go b/core/state/state_sizer.go index 2066c94845..02b73e5575 100644 --- a/core/state/state_sizer.go +++ b/core/state/state_sizer.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/triedb" "golang.org/x/sync/errgroup" ) @@ -48,6 +49,21 @@ var ( codeKeySize = int64(len(rawdb.CodePrefix) + common.HashLength) ) +// State size metrics +var ( + stateSizeChainHeightGauge = metrics.NewRegisteredGauge("state/height", nil) + stateSizeAccountsCountGauge = metrics.NewRegisteredGauge("state/accounts/count", nil) + stateSizeAccountsBytesGauge = metrics.NewRegisteredGauge("state/accounts/bytes", nil) + stateSizeStoragesCountGauge = metrics.NewRegisteredGauge("state/storages/count", nil) + stateSizeStoragesBytesGauge = metrics.NewRegisteredGauge("state/storages/bytes", nil) + stateSizeAccountTrieNodesCountGauge = metrics.NewRegisteredGauge("state/trienodes/account/count", nil) + stateSizeAccountTrieNodesBytesGauge = metrics.NewRegisteredGauge("state/trienodes/account/bytes", nil) + stateSizeStorageTrieNodesCountGauge = metrics.NewRegisteredGauge("state/trienodes/storage/count", nil) + stateSizeStorageTrieNodesBytesGauge = metrics.NewRegisteredGauge("state/trienodes/storage/bytes", nil) + stateSizeContractsCountGauge = metrics.NewRegisteredGauge("state/contracts/count", nil) + stateSizeContractsBytesGauge = metrics.NewRegisteredGauge("state/contracts/bytes", nil) +) + // SizeStats represents either the current state size statistics or the size // differences resulting from a state transition. type SizeStats struct { @@ -76,6 +92,20 @@ func (s SizeStats) String() string { ) } +func (s SizeStats) publish() { + stateSizeChainHeightGauge.Update(int64(s.BlockNumber)) + stateSizeAccountsCountGauge.Update(s.Accounts) + stateSizeAccountsBytesGauge.Update(s.AccountBytes) + stateSizeStoragesCountGauge.Update(s.Storages) + stateSizeStoragesBytesGauge.Update(s.StorageBytes) + stateSizeAccountTrieNodesCountGauge.Update(s.AccountTrienodes) + stateSizeAccountTrieNodesBytesGauge.Update(s.AccountTrienodeBytes) + stateSizeStorageTrieNodesCountGauge.Update(s.StorageTrienodes) + stateSizeStorageTrieNodesBytesGauge.Update(s.StorageTrienodeBytes) + stateSizeContractsCountGauge.Update(s.ContractCodes) + stateSizeContractsBytesGauge.Update(s.ContractCodeBytes) +} + // add applies the given state diffs and produces a new version of the statistics. func (s SizeStats) add(diff SizeStats) SizeStats { s.StateRoot = diff.StateRoot @@ -213,12 +243,14 @@ func calSizeStats(update *stateUpdate) (SizeStats, error) { } } - // Measure code changes. Note that the reported contract code size may be slightly - // inaccurate due to database deduplication (code is stored by its hash). However, - // this deviation is negligible and acceptable for measurement purposes. + codeExists := make(map[common.Hash]struct{}) for _, code := range update.codes { + if _, ok := codeExists[code.hash]; ok || code.duplicate { + continue + } stats.ContractCodes += 1 stats.ContractCodeBytes += codeKeySize + int64(len(code.blob)) + codeExists[code.hash] = struct{}{} } return stats, nil } @@ -309,8 +341,12 @@ func (t *SizeTracker) run() { stats[u.root] = stat last = u.root + // Publish statistics to metric system + stat.publish() + + // Evict the stale statistics heap.Push(&h, stats[u.root]) - for u.blockNumber-h[0].BlockNumber > statEvictThreshold { + for len(h) > 0 && u.blockNumber-h[0].BlockNumber > statEvictThreshold { delete(stats, h[0].StateRoot) heap.Pop(&h) } diff --git a/core/state/state_sizer_test.go b/core/state/state_sizer_test.go index 65f652e424..b3203afd74 100644 --- a/core/state/state_sizer_test.go +++ b/core/state/state_sizer_test.go @@ -58,7 +58,7 @@ func TestSizeTracker(t *testing.T) { state.AddBalance(addr3, uint256.NewInt(3000), tracing.BalanceChangeUnspecified) state.SetNonce(addr3, 3, tracing.NonceChangeUnspecified) - currentRoot, _, err := state.CommitWithUpdate(1, true, false) + currentRoot, err := state.Commit(1, true, false) if err != nil { t.Fatalf("Failed to commit initial state: %v", err) } @@ -83,7 +83,7 @@ func TestSizeTracker(t *testing.T) { if i%3 == 0 { newState.SetCode(testAddr, []byte{byte(i), 0x60, 0x80, byte(i + 1), 0x52}, tracing.CodeChangeUnspecified) } - root, _, err := newState.CommitWithUpdate(blockNum, true, false) + root, err := newState.Commit(blockNum, true, false) if err != nil { t.Fatalf("Failed to commit state at block %d: %v", blockNum, err) } @@ -154,21 +154,22 @@ func TestSizeTracker(t *testing.T) { if i%3 == 0 { newState.SetCode(testAddr, []byte{byte(i), 0x60, 0x80, byte(i + 1), 0x52}, tracing.CodeChangeUnspecified) } - root, update, err := newState.CommitWithUpdate(blockNum, true, false) + ret, err := newState.commitAndFlush(blockNum, true, false, true) if err != nil { t.Fatalf("Failed to commit state at block %d: %v", blockNum, err) } - if err := tdb.Commit(root, false); err != nil { + tracker.Notify(ret) + + if err := tdb.Commit(ret.root, false); err != nil { t.Fatalf("Failed to commit trie at block %d: %v", blockNum, err) } - diff, err := calSizeStats(update) + diff, err := calSizeStats(ret) if err != nil { t.Fatalf("Failed to calculate size stats for block %d: %v", blockNum, err) } trackedUpdates = append(trackedUpdates, diff) - tracker.Notify(update) - currentRoot = root + currentRoot = ret.root } finalRoot := rawdb.ReadSnapshotRoot(db) diff --git a/core/state/statedb.go b/core/state/statedb.go index cc005b451b..487fff966f 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -41,7 +41,6 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" - "github.com/ethereum/go-ethereum/trie/utils" "github.com/holiman/uint256" "golang.org/x/sync/errgroup" ) @@ -142,19 +141,19 @@ type StateDB struct { journal *journal // State witness if cross validation is needed - witness *stateless.Witness - witnessStats *stateless.WitnessStats + witness *stateless.Witness // Measurements gathered during execution for debugging purposes - AccountReads time.Duration - AccountHashes time.Duration - AccountUpdates time.Duration - AccountCommits time.Duration + AccountReads time.Duration + AccountHashes time.Duration + AccountUpdates time.Duration + AccountCommits time.Duration + StorageReads time.Duration StorageUpdates time.Duration StorageCommits time.Duration - SnapshotCommits time.Duration - TrieDBCommits time.Duration + DatabaseCommits time.Duration + CodeReads time.Duration AccountLoaded int // Number of accounts retrieved from the database during the state transition AccountUpdated int // Number of accounts updated during the state transition @@ -163,6 +162,15 @@ type StateDB struct { StorageUpdated atomic.Int64 // Number of storage slots updated during the state transition StorageDeleted atomic.Int64 // Number of storage slots deleted during the state transition + // CodeLoadBytes is the total number of bytes read from contract code. + // This value may be smaller than the actual number of bytes read, since + // some APIs (e.g. CodeSize) may load the entire code from either the + // cache or the database when the size is not available in the cache. + CodeLoaded int // Number of contract code loaded during the state transition + CodeLoadBytes int // Total bytes of resolved code + CodeUpdated int // Number of contracts with code changes that persisted + CodeUpdateBytes int // Total bytes of persisted code written + deterministic bool recording bool } @@ -201,7 +209,7 @@ func NewWithReader(root common.Hash, db Database, reader Reader) (*StateDB, erro transientStorage: newTransientStorage(), } if db.TrieDB().IsVerkle() { - sdb.accessEvents = NewAccessEvents(db.PointCache()) + sdb.accessEvents = NewAccessEvents() } return sdb, nil } @@ -238,13 +246,12 @@ func (s *StateDB) IsAddressFiltered() (bool, []filter.FilteredAddressRecord) { // StartPrefetcher initializes a new trie prefetcher to pull in nodes from the // state trie concurrently while the state is mutated so that when we reach the // commit phase, most of the needed data is already hot. -func (s *StateDB) StartPrefetcher(namespace string, witness *stateless.Witness, witnessStats *stateless.WitnessStats) { +func (s *StateDB) StartPrefetcher(namespace string, witness *stateless.Witness) { // Terminate any previously running prefetcher s.StopPrefetcher() // Enable witness collection if requested s.witness = witness - s.witnessStats = witnessStats // With the switch to the Proof-of-Stake consensus algorithm, block production // rewards are now handled at the consensus layer. Consequently, a block may @@ -310,6 +317,9 @@ func (s *StateDB) Logs() []*types.Log { for _, lgs := range s.logs { logs = append(logs, lgs...) } + sort.Slice(logs, func(i, j int) bool { + return logs[i].Index < logs[j].Index + }) return logs } @@ -571,22 +581,13 @@ func (s *StateDB) SetStorage(addr common.Address, storage map[common.Hash]common } // SelfDestruct marks the given account as selfdestructed. -// This clears the account balance. // // The account's state object is still available until the state is committed, // getStateObject will return a non-nil account after SelfDestruct. -func (s *StateDB) SelfDestruct(addr common.Address) uint256.Int { +func (s *StateDB) SelfDestruct(addr common.Address) { stateObject := s.getStateObject(addr) - var prevBalance uint256.Int if stateObject == nil { - return prevBalance - } - prevBalance = *(stateObject.Balance()) - // Regardless of whether it is already destructed or not, we do have to - // journal the balance-change, if we set it to zero here. - if !stateObject.Balance().IsZero() { - s.arbExtraData.unexpectedBalanceDelta.Sub(s.arbExtraData.unexpectedBalanceDelta, stateObject.data.Balance.ToBig()) - stateObject.SetBalance(new(uint256.Int)) + return } // If it is already marked as self-destructed, we do not need to add it // for journalling a second time. @@ -594,18 +595,6 @@ func (s *StateDB) SelfDestruct(addr common.Address) uint256.Int { s.journal.destruct(addr) stateObject.markSelfdestructed() } - return prevBalance -} - -func (s *StateDB) SelfDestruct6780(addr common.Address) (uint256.Int, bool) { - stateObject := s.getStateObject(addr) - if stateObject == nil { - return uint256.Int{}, false - } - if stateObject.newContract { - return s.SelfDestruct(addr), true - } - return *(stateObject.Balance()), false } // SetTransientState sets transient storage for a given account. It @@ -742,6 +731,16 @@ func (s *StateDB) CreateContract(addr common.Address) { } } +// IsNewContract reports whether the contract at the given address was deployed +// during the current transaction. +func (s *StateDB) IsNewContract(addr common.Address) bool { + obj := s.getStateObject(addr) + if obj == nil { + return false + } + return obj.newContract +} + // Copy creates a deep, independent copy of the state. // Snapshots of the copied state cannot be applied to the copy. func (s *StateDB) Copy() *StateDB { @@ -843,6 +842,41 @@ func (s *StateDB) GetRefund() uint64 { return s.refund } +type removedAccountWithBalance struct { + address common.Address + balance *uint256.Int +} + +// EmitLogsForBurnAccounts emits the eth burn logs for accounts scheduled for +// removal which still have positive balance. The purpose of this function is +// to handle a corner case of EIP-7708 where a self-destructed account might +// still receive funds between sending/burning its previous balance and actual +// removal. In this case the burning of these remaining balances still need to +// be logged. +// Specification EIP-7708: https://eips.ethereum.org/EIPS/eip-7708 +// +// This function should only be invoked at the transaction boundary, specifically +// before the Finalise. +func (s *StateDB) EmitLogsForBurnAccounts() { + var list []removedAccountWithBalance + for addr := range s.journal.dirties { + if obj, exist := s.stateObjects[addr]; exist && obj.selfDestructed && !obj.Balance().IsZero() { + list = append(list, removedAccountWithBalance{ + address: obj.address, + balance: obj.Balance(), + }) + } + } + if list != nil { + sort.Slice(list, func(i, j int) bool { + return list[i].address.Cmp(list[j].address) < 0 + }) + } + for _, acct := range list { + s.AddLog(types.EthBurnLog(acct.address, acct.balance)) + } +} + // Finalise finalises the state by removing the destructed objects and clears // the journal as well as the refunds. Finalise, however, will not push any updates // into the tries just yet. Only IntermediateRoot or Commit will do that. @@ -925,14 +959,50 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { workers errgroup.Group ) if s.db.TrieDB().IsVerkle() { - // Whilst MPT storage tries are independent, Verkle has one single trie - // for all the accounts and all the storage slots merged together. The - // former can thus be simply parallelized, but updating the latter will - // need concurrency support within the trie itself. That's a TODO for a - // later time. - workers.SetLimit(1) - } - if s.deterministic { + // Bypass per-account updateTrie() for binary trie. In binary trie mode + // there is only one unified trie (OpenStorageTrie returns self), so the + // per-account trie setup in updateTrie() (getPrefetchedTrie, getTrie, + // prefetcher.used) is redundant overhead. Apply all storage updates + // directly in a single pass. + for addr, op := range s.mutations { + if op.applied || op.isDelete() { + continue + } + obj := s.stateObjects[addr] + if len(obj.uncommittedStorage) == 0 { + continue + } + for key, origin := range obj.uncommittedStorage { + value, exist := obj.pendingStorage[key] + if value == origin || !exist { + continue + } + if (value != common.Hash{}) { + if err := s.trie.UpdateStorage(addr, key[:], common.TrimLeftZeroes(value[:])); err != nil { + s.setError(err) + } + } else { + if err := s.trie.DeleteStorage(addr, key[:]); err != nil { + s.setError(err) + } + } + } + } + // Clear uncommittedStorage and assign trie on each touched object. + // obj.trie must be set because this path bypasses updateTrie(), which + // is where obj.trie normally gets lazily loaded via getTrie(). + for addr, op := range s.mutations { + if op.applied || op.isDelete() { + continue + } + obj := s.stateObjects[addr] + if len(obj.uncommittedStorage) > 0 { + obj.uncommittedStorage = make(Storage) + } + obj.trie = s.trie + } + } else if s.deterministic { + // Arbitrum: deterministic state replay needs sorted iteration order. addressesToUpdate := make([]common.Address, 0, len(s.mutations)) for addr := range s.mutations { addressesToUpdate = append(addressesToUpdate, addr) @@ -942,16 +1012,12 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { if obj := s.mutations[addr]; !obj.applied && !obj.isDelete() { obj := s.stateObjects[addr] // closure for the task runner below workers.Go(func() error { - if s.db.TrieDB().IsVerkle() { - obj.updateTrie() - } else { - obj.updateRoot() - - // If witness building is enabled and the state object has a trie, - // gather the witnesses for its specific storage trie - if s.witness != nil && obj.trie != nil { - s.witness.AddState(obj.trie.Witness()) - } + obj.updateRoot() + + // If witness building is enabled and the state object has a trie, + // gather the witnesses for its specific storage trie + if s.witness != nil && obj.trie != nil { + s.witness.AddState(obj.trie.Witness(), obj.addrHash()) } return nil }) @@ -964,16 +1030,12 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { } obj := s.stateObjects[addr] // closure for the task runner below workers.Go(func() error { - if s.db.TrieDB().IsVerkle() { - obj.updateTrie() - } else { - obj.updateRoot() + obj.updateRoot() - // If witness building is enabled and the state object has a trie, - // gather the witnesses for its specific storage trie - if s.witness != nil && obj.trie != nil { - s.witness.AddState(obj.trie.Witness()) - } + // If witness building is enabled and the state object has a trie, + // gather the witnesses for its specific storage trie + if s.witness != nil && obj.trie != nil { + s.witness.AddState(obj.trie.Witness(), obj.addrHash()) } return nil }) @@ -990,17 +1052,9 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { continue } if trie := obj.getPrefetchedTrie(); trie != nil { - witness := trie.Witness() - s.witness.AddState(witness) - if s.witnessStats != nil { - s.witnessStats.Add(witness, obj.addrHash) - } + s.witness.AddState(trie.Witness(), obj.addrHash()) } else if obj.trie != nil { - witness := obj.trie.Witness() - s.witness.AddState(witness) - if s.witnessStats != nil { - s.witnessStats.Add(witness, obj.addrHash) - } + s.witness.AddState(obj.trie.Witness(), obj.addrHash()) } } // Pull in only-read and non-destructed trie witnesses @@ -1014,17 +1068,9 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { continue } if trie := obj.getPrefetchedTrie(); trie != nil { - witness := trie.Witness() - s.witness.AddState(witness) - if s.witnessStats != nil { - s.witnessStats.Add(witness, obj.addrHash) - } + s.witness.AddState(trie.Witness(), obj.addrHash()) } else if obj.trie != nil { - witness := obj.trie.Witness() - s.witness.AddState(witness) - if s.witnessStats != nil { - s.witnessStats.Add(witness, obj.addrHash) - } + s.witness.AddState(obj.trie.Witness(), obj.addrHash()) } } } @@ -1039,7 +1085,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { // only a single trie is used for state hashing. Replacing a non-nil verkle tree // here could result in losing uncommitted changes from storage. start = time.Now() - if s.prefetcher != nil { + if s.prefetcher != nil && !s.db.TrieDB().IsVerkle() { if trie := s.prefetcher.trie(common.Hash{}, s.originalRoot); trie == nil { log.Error("Failed to retrieve account pre-fetcher trie") } else { @@ -1084,8 +1130,15 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { if op.isDelete() { deletedAddrs = append(deletedAddrs, addr) } else { - s.updateStateObject(s.stateObjects[addr]) + obj := s.stateObjects[addr] + s.updateStateObject(obj) s.AccountUpdated += 1 + + // Count code writes post-Finalise so reverted CREATEs are excluded. + if obj.dirtyCode { + s.CodeUpdated += 1 + s.CodeUpdateBytes += len(obj.code) + } } usedAddrs = append(usedAddrs, addr) // Copy needed for closure } @@ -1108,11 +1161,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { // If witness building is enabled, gather the account trie witness if s.witness != nil { - witness := s.trie.Witness() - s.witness.AddState(witness) - if s.witnessStats != nil { - s.witnessStats.Add(witness, common.Hash{}) - } + s.witness.AddState(s.trie.Witness(), common.Hash{}) } return hash } @@ -1435,7 +1484,7 @@ func (s *StateDB) commit(deleteEmptyObjects bool, noStorageWiping bool, blockNum return err } lock.Lock() - updates[obj.addrHash] = update + updates[obj.addrHash()] = update s.StorageCommits = time.Since(start) // overwrite with the longest storage commit runtime lock.Unlock() return nil @@ -1473,22 +1522,16 @@ func (s *StateDB) commit(deleteEmptyObjects bool, noStorageWiping bool, blockNum // commitAndFlush is a wrapper of commit which also commits the state mutations // to the configured data stores. -func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool, noStorageWiping bool) (*stateUpdate, error) { +func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool, noStorageWiping bool, deriveCodeFields bool) (*stateUpdate, error) { ret, err := s.commit(deleteEmptyObjects, noStorageWiping, block) if err != nil { return nil, err } - // Commit dirty contract code if any exists - if db := s.db.TrieDB().Disk(); db != nil && len(ret.codes) > 0 { - batch := db.NewBatch() - for _, code := range ret.codes { - rawdb.WriteCode(batch, code.hash, code.blob) - } - if err := batch.Write(); err != nil { + if deriveCodeFields { + if err := ret.deriveCodeFields(s.reader); err != nil { return nil, err } } - if db := s.db.WasmStore(); db != nil && len(ret.activatedWasms) > 0 { batch := db.NewBatch() // Arbitrum: write Stylus programs to disk @@ -1501,31 +1544,14 @@ func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool, noStorag } } - if !ret.empty() { - // If snapshotting is enabled, update the snapshot tree with this new version - if snap := s.db.Snapshot(); snap != nil && snap.Snapshot(ret.originRoot) != nil { - start := time.Now() - if err := snap.Update(ret.root, ret.originRoot, ret.accounts, ret.storages); err != nil { - log.Warn("Failed to update snapshot tree", "from", ret.originRoot, "to", ret.root, "err", err) - } - // Keep 128 diff layers in the memory, persistent layer is 129th. - // - head layer is paired with HEAD state - // - head-1 layer is paired with HEAD-1 state - // - head-127 layer(bottom-most diff layer) is paired with HEAD-127 state - if err := snap.Cap(ret.root, DefaultTriesInMemory); err != nil { - log.Warn("Failed to cap snapshot tree", "root", ret.root, "layers", DefaultTriesInMemory, "err", err) - } - s.SnapshotCommits += time.Since(start) - } - // If trie database is enabled, commit the state update as a new layer - if db := s.db.TrieDB(); db != nil { - start := time.Now() - if err := db.Update(ret.root, ret.originRoot, block, ret.nodes, ret.stateSet()); err != nil { - return nil, err - } - s.TrieDBCommits += time.Since(start) - } + start := time.Now() + if err := s.db.Commit(ret); err != nil { + return nil, err } + s.DatabaseCommits = time.Since(start) + + // The reader update must be performed as the final step, otherwise, + // the new state would not be visible before db.commit. s.reader, _ = s.db.Reader(s.originalRoot) s.arbExtraData.unexpectedBalanceDelta.Set(new(big.Int)) return ret, err @@ -1546,17 +1572,17 @@ func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool, noStorag // no empty accounts left that could be deleted by EIP-158, storage wiping // should not occur. func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool, noStorageWiping bool) (common.Hash, error) { - ret, err := s.commitAndFlush(block, deleteEmptyObjects, noStorageWiping) + ret, err := s.commitAndFlush(block, deleteEmptyObjects, noStorageWiping, false) if err != nil { return common.Hash{}, err } return ret.root, nil } -// CommitWithUpdate writes the state mutations and returns both the root hash and the state update. -// This is useful for tracking state changes at the blockchain level. +// CommitWithUpdate writes the state mutations and returns the state update for +// external processing (e.g., live tracing hooks or size tracker). func (s *StateDB) CommitWithUpdate(block uint64, deleteEmptyObjects bool, noStorageWiping bool) (common.Hash, *stateUpdate, error) { - ret, err := s.commitAndFlush(block, deleteEmptyObjects, noStorageWiping) + ret, err := s.commitAndFlush(block, deleteEmptyObjects, noStorageWiping, true) if err != nil { return common.Hash{}, nil, err } @@ -1667,11 +1693,6 @@ func (s *StateDB) markUpdate(addr common.Address) { s.mutations[addr].typ = update } -// PointCache returns the point cache used by verkle tree. -func (s *StateDB) PointCache() *utils.PointCache { - return s.db.PointCache() -} - // Witness retrieves the current state witness being collected. func (s *StateDB) Witness() *stateless.Witness { return s.witness diff --git a/core/state/statedb_fuzz_test.go b/core/state/statedb_fuzz_test.go index b2963b9be8..1ea699fb3d 100644 --- a/core/state/statedb_fuzz_test.go +++ b/core/state/statedb_fuzz_test.go @@ -209,7 +209,7 @@ func (test *stateTest) run() bool { if i != 0 { root = roots[len(roots)-1] } - state, err := New(root, NewDatabase(tdb, snaps)) + state, err := New(root, NewDatabase(tdb, nil).WithSnapshot(snaps)) if err != nil { panic(err) } @@ -228,7 +228,7 @@ func (test *stateTest) run() bool { } else { state.IntermediateRoot(true) // call intermediateRoot at the transaction boundary } - ret, err := state.commitAndFlush(0, true, false) // call commit at the block boundary + ret, err := state.commitAndFlush(0, true, false, false) // call commit at the block boundary if err != nil { panic(err) } diff --git a/core/state/statedb_hooked.go b/core/state/statedb_hooked.go index c8b2b8204a..ddff50ffa3 100644 --- a/core/state/statedb_hooked.go +++ b/core/state/statedb_hooked.go @@ -17,7 +17,9 @@ package state import ( + "bytes" "math/big" + "sort" "github.com/ethereum/go-ethereum/arbitrum/filter" "github.com/ethereum/go-ethereum/common" @@ -27,7 +29,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie/utils" "github.com/holiman/uint256" ) @@ -55,6 +56,10 @@ func (s *hookedStateDB) CreateContract(addr common.Address) { s.inner.CreateContract(addr) } +func (s *hookedStateDB) IsNewContract(addr common.Address) bool { + return s.inner.IsNewContract(addr) +} + func (s *hookedStateDB) GetBalance(addr common.Address) *uint256.Int { return s.inner.GetBalance(addr) } @@ -135,10 +140,6 @@ func (s *hookedStateDB) AddSlotToAccessList(addr common.Address, slot common.Has s.inner.AddSlotToAccessList(addr, slot) } -func (s *hookedStateDB) PointCache() *utils.PointCache { - return s.inner.PointCache() -} - func (s *hookedStateDB) Prepare(rules params.Rules, sender, coinbase common.Address, dest *common.Address, precompiles []common.Address, txAccesses types.AccessList) { s.inner.Prepare(rules, sender, coinbase, dest, precompiles, txAccesses) } @@ -218,56 +219,8 @@ func (s *hookedStateDB) SetState(address common.Address, key common.Hash, value return prev } -func (s *hookedStateDB) SelfDestruct(address common.Address) uint256.Int { - var prevCode []byte - var prevCodeHash common.Hash - - if s.hooks.OnCodeChange != nil { - prevCode = s.inner.GetCode(address) - prevCodeHash = s.inner.GetCodeHash(address) - } - - prev := s.inner.SelfDestruct(address) - - if s.hooks.OnBalanceChange != nil && !prev.IsZero() { - s.hooks.OnBalanceChange(address, prev.ToBig(), new(big.Int), tracing.BalanceDecreaseSelfdestruct) - } - - if len(prevCode) > 0 { - if s.hooks.OnCodeChangeV2 != nil { - s.hooks.OnCodeChangeV2(address, prevCodeHash, prevCode, types.EmptyCodeHash, nil, tracing.CodeChangeSelfDestruct) - } else if s.hooks.OnCodeChange != nil { - s.hooks.OnCodeChange(address, prevCodeHash, prevCode, types.EmptyCodeHash, nil) - } - } - - return prev -} - -func (s *hookedStateDB) SelfDestruct6780(address common.Address) (uint256.Int, bool) { - var prevCode []byte - var prevCodeHash common.Hash - - if s.hooks.OnCodeChange != nil { - prevCodeHash = s.inner.GetCodeHash(address) - prevCode = s.inner.GetCode(address) - } - - prev, changed := s.inner.SelfDestruct6780(address) - - if s.hooks.OnBalanceChange != nil && !prev.IsZero() { - s.hooks.OnBalanceChange(address, prev.ToBig(), new(big.Int), tracing.BalanceDecreaseSelfdestruct) - } - - if changed && len(prevCode) > 0 { - if s.hooks.OnCodeChangeV2 != nil { - s.hooks.OnCodeChangeV2(address, prevCodeHash, prevCode, types.EmptyCodeHash, nil, tracing.CodeChangeSelfDestruct) - } else if s.hooks.OnCodeChange != nil { - s.hooks.OnCodeChange(address, prevCodeHash, prevCode, types.EmptyCodeHash, nil) - } - } - - return prev, changed +func (s *hookedStateDB) SelfDestruct(address common.Address) { + s.inner.SelfDestruct(address) } func (s *hookedStateDB) AddLog(log *types.Log) { @@ -278,20 +231,65 @@ func (s *hookedStateDB) AddLog(log *types.Log) { } } +func (s *hookedStateDB) EmitLogsForBurnAccounts() { + s.inner.EmitLogsForBurnAccounts() +} + func (s *hookedStateDB) Finalise(deleteEmptyObjects bool) { - defer s.inner.Finalise(deleteEmptyObjects) - if s.hooks.OnBalanceChange == nil { + if s.hooks.OnBalanceChange == nil && s.hooks.OnNonceChangeV2 == nil && s.hooks.OnNonceChange == nil && s.hooks.OnCodeChangeV2 == nil && s.hooks.OnCodeChange == nil { + // Short circuit if no relevant hooks are set. + s.inner.Finalise(deleteEmptyObjects) return } + + // Collect all self-destructed addresses first, then sort them to ensure + // that state change hooks will be invoked in deterministic + // order when the accounts are deleted below + var selfDestructedAddrs []common.Address for addr := range s.inner.journal.dirties { obj := s.inner.stateObjects[addr] - if obj != nil && obj.selfDestructed { - // If ether was sent to account post-selfdestruct it is burnt. + if obj == nil || !obj.selfDestructed { + // Not self-destructed, keep searching. + continue + } + selfDestructedAddrs = append(selfDestructedAddrs, addr) + } + sort.Slice(selfDestructedAddrs, func(i, j int) bool { + return bytes.Compare(selfDestructedAddrs[i][:], selfDestructedAddrs[j][:]) < 0 + }) + + for _, addr := range selfDestructedAddrs { + obj := s.inner.stateObjects[addr] + // Bingo: state object was self-destructed, call relevant hooks. + + // If ether was sent to account post-selfdestruct, record as burnt. + if s.hooks.OnBalanceChange != nil { if bal := obj.Balance(); bal.Sign() != 0 { s.hooks.OnBalanceChange(addr, bal.ToBig(), new(big.Int), tracing.BalanceDecreaseSelfdestructBurn) } } + + // Nonce is set to reset on self-destruct. + if s.hooks.OnNonceChangeV2 != nil { + s.hooks.OnNonceChangeV2(addr, obj.Nonce(), 0, tracing.NonceChangeSelfdestruct) + } else if s.hooks.OnNonceChange != nil { + s.hooks.OnNonceChange(addr, obj.Nonce(), 0) + } + + // If an initcode invokes selfdestruct, do not emit a code change. + prevCodeHash := s.inner.GetCodeHash(addr) + if prevCodeHash == types.EmptyCodeHash { + continue + } + // Otherwise, trace the change. + if s.hooks.OnCodeChangeV2 != nil { + s.hooks.OnCodeChangeV2(addr, prevCodeHash, s.inner.GetCode(addr), types.EmptyCodeHash, nil, tracing.CodeChangeSelfDestruct) + } else if s.hooks.OnCodeChange != nil { + s.hooks.OnCodeChange(addr, prevCodeHash, s.inner.GetCode(addr), types.EmptyCodeHash, nil) + } } + + s.inner.Finalise(deleteEmptyObjects) } func (s *hookedStateDB) ActivateWasm(moduleHash common.Hash, asmMap map[rawdb.WasmTarget][]byte) error { diff --git a/core/state/statedb_hooked_test.go b/core/state/statedb_hooked_test.go index bacb7baee1..6fe17ec1b4 100644 --- a/core/state/statedb_hooked_test.go +++ b/core/state/statedb_hooked_test.go @@ -49,6 +49,8 @@ func TestBurn(t *testing.T) { createAndDestroy := func(addr common.Address) { hooked.AddBalance(addr, uint256.NewInt(100), tracing.BalanceChangeUnspecified) hooked.CreateContract(addr) + // Simulate what the opcode handler does: clear balance before selfdestruct + hooked.SubBalance(addr, hooked.GetBalance(addr), tracing.BalanceDecreaseSelfdestruct) hooked.SelfDestruct(addr) // sanity-check that balance is now 0 if have, want := hooked.GetBalance(addr), new(uint256.Int); !have.Eq(want) { @@ -122,9 +124,51 @@ func TestHooks(t *testing.T) { sdb.AddLog(&types.Log{ Address: common.Address{0xbb}, }) + + if len(result) != len(wants) { + t.Fatalf("number of tracing events wrong, have %d want %d", len(result), len(wants)) + } + + for i, want := range wants { + if have := result[i]; have != want { + t.Fatalf("error event %d\nhave: %v\nwant: %v", i, have, want) + } + } +} + +func TestHooks_OnCodeChangeV2(t *testing.T) { + inner, _ := New(types.EmptyRootHash, NewDatabaseForTesting()) + + var result []string + var wants = []string{ + "0xaa00000000000000000000000000000000000000.code: (0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470) ->0x1325 (0xa12ae05590de0c93a00bc7ac773c2fdb621e44f814985e72194f921c0050f728) ContractCreation", + "0xbb00000000000000000000000000000000000000.code: (0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470) ->0x1326 (0x3c54516221d604e623f358bc95996ca3242aaa109bddabcebda13db9b3f90dcb) ContractCreation", + "0xaa00000000000000000000000000000000000000.code: 0x1325 (0xa12ae05590de0c93a00bc7ac773c2fdb621e44f814985e72194f921c0050f728) -> (0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470) SelfDestruct", + "0xbb00000000000000000000000000000000000000.code: 0x1326 (0x3c54516221d604e623f358bc95996ca3242aaa109bddabcebda13db9b3f90dcb) -> (0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470) SelfDestruct", + } + emitF := func(format string, a ...any) { + result = append(result, fmt.Sprintf(format, a...)) + } + sdb := NewHookedState(inner, &tracing.Hooks{ + OnCodeChangeV2: func(addr common.Address, prevCodeHash common.Hash, prevCode []byte, codeHash common.Hash, code []byte, reason tracing.CodeChangeReason) { + emitF("%v.code: %#x (%v) ->%#x (%v) %s", addr, prevCode, prevCodeHash, code, codeHash, reason) + }, + }) + sdb.SetCode(common.Address{0xaa}, []byte{0x13, 37}, tracing.CodeChangeContractCreation) + sdb.SelfDestruct(common.Address{0xaa}) + + sdb.SetCode(common.Address{0xbb}, []byte{0x13, 38}, tracing.CodeChangeContractCreation) + sdb.CreateContract(common.Address{0xbb}) + sdb.SelfDestruct(common.Address{0xbb}) + sdb.Finalise(true) + + if len(result) != len(wants) { + t.Fatalf("number of tracing events wrong, have %d want %d", len(result), len(wants)) + } + for i, want := range wants { if have := result[i]; have != want { - t.Fatalf("error event %d, have\n%v\nwant%v\n", i, have, want) + t.Fatalf("error event %d\nhave: %v\nwant: %v", i, have, want) } } } diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index f8ab838987..e00a6c555e 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -1233,7 +1233,7 @@ func TestDeleteStorage(t *testing.T) { disk = rawdb.NewMemoryDatabase() tdb = triedb.NewDatabase(disk, nil) snaps, _ = snapshot.New(snapshot.Config{CacheSize: 10}, disk, tdb, types.EmptyRootHash) - db = NewDatabase(tdb, snaps) + db = NewDatabase(tdb, nil).WithSnapshot(snaps) state, _ = New(types.EmptyRootHash, db) addr = common.HexToAddress("0x1") ) @@ -1247,7 +1247,7 @@ func TestDeleteStorage(t *testing.T) { } root, _ := state.Commit(0, true, false) // Init phase done, create two states, one with snap and one without - fastState, _ := New(root, NewDatabase(tdb, snaps)) + fastState, _ := New(root, NewDatabase(tdb, nil).WithSnapshot(snaps)) slowState, _ := New(root, NewDatabase(tdb, nil)) obj := fastState.getOrNewStateObject(addr) diff --git a/core/state/stateupdate.go b/core/state/stateupdate.go index f404613cd0..5b14a15252 100644 --- a/core/state/stateupdate.go +++ b/core/state/stateupdate.go @@ -17,17 +17,27 @@ package state import ( + "fmt" "maps" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/tracing" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/triedb" ) -// contractCode represents a contract code with associated metadata. +// contractCode represents contract bytecode along with its associated metadata. type contractCode struct { - hash common.Hash // hash is the cryptographic hash of the contract code. - blob []byte // blob is the binary representation of the contract code. + hash common.Hash // hash is the cryptographic hash of the current contract code. + blob []byte // blob is the binary representation of the current contract code. + originHash common.Hash // originHash is the cryptographic hash of the code before mutation. + + // Derived fields, populated only when state tracking is enabled. + duplicate bool // duplicate indicates whether the updated code already exists. + originBlob []byte // originBlob is the original binary representation of the contract code. } // accountDelete represents an operation for deleting an Ethereum account. @@ -82,8 +92,8 @@ type stateUpdate struct { storagesOrigin map[common.Address]map[common.Hash][]byte rawStorageKey bool - codes map[common.Address]contractCode // codes contains the set of dirty codes - nodes *trienode.MergedNodeSet // Aggregated dirty nodes caused by state changes + codes map[common.Address]*contractCode // codes contains the set of dirty codes + nodes *trienode.MergedNodeSet // Aggregated dirty nodes caused by state changes activatedWasms map[common.Hash]ActivatedWasm // newly activated WASMs } @@ -105,7 +115,7 @@ func newStateUpdate(rawStorageKey bool, originRoot common.Hash, root common.Hash accountsOrigin = make(map[common.Address][]byte) storages = make(map[common.Hash]map[common.Hash][]byte) storagesOrigin = make(map[common.Address]map[common.Hash][]byte) - codes = make(map[common.Address]contractCode) + codes = make(map[common.Address]*contractCode) ) // Since some accounts might be destroyed and recreated within the same // block, deletions must be aggregated first. @@ -127,7 +137,7 @@ func newStateUpdate(rawStorageKey bool, originRoot common.Hash, root common.Hash // Aggregate dirty contract codes if they are available. addr := op.address if op.code != nil { - codes[addr] = *op.code + codes[addr] = op.code } accounts[addrHash] = op.data @@ -193,3 +203,170 @@ func (sc *stateUpdate) stateSet() *triedb.StateSet { RawStorageKey: sc.rawStorageKey, } } + +// deriveCodeFields derives the missing fields of contract code changes +// such as original code value. +// +// Note: This operation is expensive and not needed during normal state +// transitions. It is only required when SizeTracker or StateUpdate hook +// is enabled to produce accurate state statistics. +func (sc *stateUpdate) deriveCodeFields(reader ContractCodeReader) error { + cache := make(map[common.Hash]bool) + for addr, code := range sc.codes { + if code.originHash != types.EmptyCodeHash { + blob := reader.Code(addr, code.originHash) + if len(blob) == 0 { + return fmt.Errorf("original code of %x is empty", addr) + } + code.originBlob = blob + } + if exists, ok := cache[code.hash]; ok { + code.duplicate = exists + continue + } + res := reader.Has(addr, code.hash) + cache[code.hash] = res + code.duplicate = res + } + return nil +} + +// ToTracingUpdate converts the internal stateUpdate to an exported tracing.StateUpdate. +func (sc *stateUpdate) ToTracingUpdate() (*tracing.StateUpdate, error) { + update := &tracing.StateUpdate{ + OriginRoot: sc.originRoot, + Root: sc.root, + BlockNumber: sc.blockNumber, + AccountChanges: make(map[common.Address]*tracing.AccountChange, len(sc.accountsOrigin)), + StorageChanges: make(map[common.Address]map[common.Hash]*tracing.StorageChange), + CodeChanges: make(map[common.Address]*tracing.CodeChange, len(sc.codes)), + TrieChanges: make(map[common.Hash]map[string]*tracing.TrieNodeChange), + } + // Gather all account changes + for addr, oldData := range sc.accountsOrigin { + addrHash := crypto.Keccak256Hash(addr.Bytes()) + newData, exists := sc.accounts[addrHash] + if !exists { + return nil, fmt.Errorf("account %x not found", addr) + } + change := &tracing.AccountChange{} + + if len(oldData) > 0 { + acct, err := types.FullAccount(oldData) + if err != nil { + return nil, err + } + change.Prev = &types.StateAccount{ + Nonce: acct.Nonce, + Balance: acct.Balance, + Root: acct.Root, + CodeHash: acct.CodeHash, + } + } + if len(newData) > 0 { + acct, err := types.FullAccount(newData) + if err != nil { + return nil, err + } + change.New = &types.StateAccount{ + Nonce: acct.Nonce, + Balance: acct.Balance, + Root: acct.Root, + CodeHash: acct.CodeHash, + } + } + update.AccountChanges[addr] = change + } + + // Gather all storage slot changes + for addr, slots := range sc.storagesOrigin { + addrHash := crypto.Keccak256Hash(addr.Bytes()) + subset, exists := sc.storages[addrHash] + if !exists { + return nil, fmt.Errorf("storage %x not found", addr) + } + storageChanges := make(map[common.Hash]*tracing.StorageChange, len(slots)) + + for key, encPrev := range slots { + // Get new value - handle both raw and hashed key formats + var ( + exists bool + encNew []byte + decPrev []byte + decNew []byte + err error + ) + if sc.rawStorageKey { + encNew, exists = subset[crypto.Keccak256Hash(key.Bytes())] + } else { + encNew, exists = subset[key] + } + if !exists { + return nil, fmt.Errorf("storage slot %x-%x not found", addr, key) + } + + // Decode the prev and new values + if len(encPrev) > 0 { + _, decPrev, _, err = rlp.Split(encPrev) + if err != nil { + return nil, fmt.Errorf("failed to decode prevValue: %v", err) + } + } + if len(encNew) > 0 { + _, decNew, _, err = rlp.Split(encNew) + if err != nil { + return nil, fmt.Errorf("failed to decode newValue: %v", err) + } + } + storageChanges[key] = &tracing.StorageChange{ + Prev: common.BytesToHash(decPrev), + New: common.BytesToHash(decNew), + } + } + update.StorageChanges[addr] = storageChanges + } + + // Gather all contract code changes + for addr, code := range sc.codes { + change := &tracing.CodeChange{ + New: &tracing.ContractCode{ + Hash: code.hash, + Code: code.blob, + Exists: code.duplicate, + }, + } + if code.originHash != types.EmptyCodeHash { + change.Prev = &tracing.ContractCode{ + Hash: code.originHash, + Code: code.originBlob, + Exists: true, + } + } + update.CodeChanges[addr] = change + } + + // Gather all trie node changes + if sc.nodes != nil { + for owner, subset := range sc.nodes.Sets { + nodeChanges := make(map[string]*tracing.TrieNodeChange, len(subset.Origins)) + for path, oldNode := range subset.Origins { + newNode, exists := subset.Nodes[path] + if !exists { + return nil, fmt.Errorf("node %x-%v not found", owner, path) + } + nodeChanges[path] = &tracing.TrieNodeChange{ + Prev: &trienode.Node{ + Hash: crypto.Keccak256Hash(oldNode), + Blob: oldNode, + }, + New: &trienode.Node{ + Hash: newNode.Hash, + Blob: newNode.Blob, + }, + } + } + update.TrieChanges[owner] = nodeChanges + } + } + return update, nil +} diff --git a/core/state/sync_test.go b/core/state/sync_test.go index cae0e0a936..e5e22deae5 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -222,8 +222,8 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool, s codeResults = make([]trie.CodeSyncResult, len(codeElements)) ) for i, element := range codeElements { - data, err := cReader.Code(common.Address{}, element.code) - if err != nil || len(data) == 0 { + data := cReader.Code(common.Address{}, element.code) + if len(data) == 0 { t.Fatalf("failed to retrieve contract bytecode for hash %x", element.code) } codeResults[i] = trie.CodeSyncResult{Hash: element.code, Data: data} @@ -346,8 +346,8 @@ func testIterativeDelayedStateSync(t *testing.T, scheme string) { if len(codeElements) > 0 { codeResults := make([]trie.CodeSyncResult, len(codeElements)/2+1) for i, element := range codeElements[:len(codeResults)] { - data, err := cReader.Code(common.Address{}, element.code) - if err != nil || len(data) == 0 { + data := cReader.Code(common.Address{}, element.code) + if len(data) == 0 { t.Fatalf("failed to retrieve contract bytecode for %x", element.code) } codeResults[i] = trie.CodeSyncResult{Hash: element.code, Data: data} @@ -452,8 +452,8 @@ func testIterativeRandomStateSync(t *testing.T, count int, scheme string) { if len(codeQueue) > 0 { results := make([]trie.CodeSyncResult, 0, len(codeQueue)) for hash := range codeQueue { - data, err := cReader.Code(common.Address{}, hash) - if err != nil || len(data) == 0 { + data := cReader.Code(common.Address{}, hash) + if len(data) == 0 { t.Fatalf("failed to retrieve node data for %x", hash) } results = append(results, trie.CodeSyncResult{Hash: hash, Data: data}) @@ -551,8 +551,8 @@ func testIterativeRandomDelayedStateSync(t *testing.T, scheme string) { for hash := range codeQueue { delete(codeQueue, hash) - data, err := cReader.Code(common.Address{}, hash) - if err != nil || len(data) == 0 { + data := cReader.Code(common.Address{}, hash) + if len(data) == 0 { t.Fatalf("failed to retrieve node data for %x", hash) } results = append(results, trie.CodeSyncResult{Hash: hash, Data: data}) @@ -671,8 +671,8 @@ func testIncompleteStateSync(t *testing.T, scheme string) { if len(codeQueue) > 0 { results := make([]trie.CodeSyncResult, 0, len(codeQueue)) for hash := range codeQueue { - data, err := cReader.Code(common.Address{}, hash) - if err != nil || len(data) == 0 { + data := cReader.Code(common.Address{}, hash) + if len(data) == 0 { t.Fatalf("failed to retrieve node data for %x", hash) } results = append(results, trie.CodeSyncResult{Hash: hash, Data: data}) diff --git a/core/state_prefetcher.go b/core/state_prefetcher.go index b6e398ae83..71eafa7f10 100644 --- a/core/state_prefetcher.go +++ b/core/state_prefetcher.go @@ -108,7 +108,7 @@ func (p *statePrefetcher) Prefetch(block *types.Block, statedb *state.StateDB, c // We attempt to apply a transaction. The goal is not to execute // the transaction successfully, rather to warm up touched data slots. - if _, err := ApplyMessage(evm, msg, new(GasPool).AddGas(block.GasLimit())); err != nil { + if _, err := ApplyMessage(evm, msg, nil); err != nil { fails.Add(1) return nil // Ugh, something went horribly wrong, bail out } diff --git a/core/state_processor.go b/core/state_processor.go index eace271136..834cf8c71a 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -17,6 +17,7 @@ package core import ( + "context" "fmt" "math/big" @@ -26,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/internal/telemetry" "github.com/ethereum/go-ethereum/params" ) @@ -56,31 +58,30 @@ func (p *StateProcessor) chainConfig() *params.ChainConfig { // Process returns the receipts and logs accumulated during the process and // returns the amount of gas that was used in the process. If any of the // transactions failed to execute due to insufficient gas it will return an error. -func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg vm.Config) (*ProcessResult, error) { +func (p *StateProcessor) Process(ctx context.Context, block *types.Block, statedb *state.StateDB, cfg vm.Config) (*ProcessResult, error) { var ( config = p.chainConfig() receipts types.Receipts - usedGas = new(uint64) header = block.Header() blockHash = block.Hash() blockNumber = block.Number() allLogs []*types.Log - gp = new(GasPool).AddGas(block.GasLimit()) + gp = NewGasPool(block.GasLimit()) ) + var tracingStateDB = vm.StateDB(statedb) + if hooks := cfg.Tracer; hooks != nil { + tracingStateDB = state.NewHookedState(statedb, hooks) + } // Mutate the block and state according to any hard-fork specs if config.DAOForkSupport && config.DAOForkBlock != nil && config.DAOForkBlock.Cmp(block.Number()) == 0 { - misc.ApplyDAOHardFork(statedb) + misc.ApplyDAOHardFork(tracingStateDB) } var ( context vm.BlockContext ) // Apply pre-execution system calls. - var tracingStateDB = vm.StateDB(statedb) - if hooks := cfg.Tracer; hooks != nil { - tracingStateDB = state.NewHookedState(statedb, hooks) - } context = NewEVMBlockContext(header, p.chain, nil) evm := vm.NewEVM(context, tracingStateDB, config, cfg) @@ -103,49 +104,66 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg return nil, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } statedb.SetTxContext(tx.Hash(), i) - - receipt, _, err := ApplyTransactionWithEVM(msg, gp, statedb, blockNumber, blockHash, context.Time, tx, usedGas, evm, nil) + _, _, spanEnd := telemetry.StartSpan(ctx, "core.ApplyTransactionWithEVM", + telemetry.StringAttribute("tx.hash", tx.Hash().Hex()), + telemetry.Int64Attribute("tx.index", int64(i)), + ) + receipt, _, err := ApplyTransactionWithEVM(msg, gp, statedb, blockNumber, blockHash, context.Time, tx, evm, nil) + spanEnd(&err) if err != nil { + spanEnd(&err) return nil, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } receipts = append(receipts, receipt) allLogs = append(allLogs, receipt.Logs...) + spanEnd(nil) + } + requests, err := postExecution(ctx, config, block, allLogs, evm) + if err != nil { + return nil, err } + // Finalize the block, applying any consensus engine specific extras (e.g. block rewards) + p.chain.Engine().Finalize(p.chain, header, tracingStateDB, block.Body()) + + return &ProcessResult{ + Receipts: receipts, + Requests: requests, + Logs: allLogs, + GasUsed: gp.Used(), + }, nil +} + +// postExecution processes the post-execution system calls if Prague is enabled. +func postExecution(ctx context.Context, config *params.ChainConfig, block *types.Block, allLogs []*types.Log, evm *vm.EVM) (requests [][]byte, err error) { + _, _, spanEnd := telemetry.StartSpan(ctx, "core.postExecution") + defer spanEnd(&err) + // Read requests if Prague is enabled. - var requests [][]byte // Arbitrum has no Deposit, Witdrawal, or Consolidation requests. - if !config.IsArbitrum() && config.IsPrague(block.Number(), block.Time(), context.ArbOSVersion) { + if !config.IsArbitrum() && config.IsPrague(block.Number(), block.Time(), evm.Context.ArbOSVersion) { requests = [][]byte{} // EIP-6110 if err := ParseDepositLogs(&requests, allLogs, config); err != nil { - return nil, fmt.Errorf("failed to parse deposit logs: %w", err) + return requests, fmt.Errorf("failed to parse deposit logs: %w", err) } // EIP-7002 if err := ProcessWithdrawalQueue(&requests, evm); err != nil { - return nil, fmt.Errorf("failed to process withdrawal queue: %w", err) + return requests, fmt.Errorf("failed to process withdrawal queue: %w", err) } // EIP-7251 if err := ProcessConsolidationQueue(&requests, evm); err != nil { - return nil, fmt.Errorf("failed to process consolidation queue: %w", err) + return requests, fmt.Errorf("failed to process consolidation queue: %w", err) } } - // Finalize the block, applying any consensus engine specific extras (e.g. block rewards) - p.chain.Engine().Finalize(p.chain, header, tracingStateDB, block.Body()) - - return &ProcessResult{ - Receipts: receipts, - Requests: requests, - Logs: allLogs, - GasUsed: *usedGas, - }, nil + return requests, nil } // ApplyTransactionWithEVM attempts to apply a transaction to the given state database // and uses the input parameters for its environment similar to ApplyTransaction. However, // this method takes an already created EVM instance as input. -func ApplyTransactionWithEVM(msg *Message, gp *GasPool, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, blockTime uint64, tx *types.Transaction, usedGas *uint64, evm *vm.EVM, resultFilter func(*ExecutionResult) error) (receipt *types.Receipt, result *ExecutionResult, err error) { +func ApplyTransactionWithEVM(msg *Message, gp *GasPool, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, blockTime uint64, tx *types.Transaction, evm *vm.EVM, resultFilter func(*ExecutionResult) error) (receipt *types.Receipt, result *ExecutionResult, err error) { if hooks := evm.Config.Tracer; hooks != nil { if hooks.OnTxStart != nil { hooks.OnTxStart(evm.GetVMContext(), tx, msg.From) @@ -173,28 +191,32 @@ func ApplyTransactionWithEVM(msg *Message, gp *GasPool, statedb *state.StateDB, } else { root = statedb.IntermediateRoot(evm.ChainConfig().IsEIP158(blockNumber)).Bytes() } - *usedGas += result.UsedGas - // Merge the tx-local access event into the "block-local" one, in order to collect // all values, so that the witness can be built. if statedb.Database().TrieDB().IsVerkle() { statedb.AccessEvents().Merge(evm.AccessEvents) } - return MakeReceipt(evm, result, statedb, blockNumber, blockHash, blockTime, tx, *usedGas, root), result, nil + return MakeReceipt(evm, result, statedb, blockNumber, blockHash, blockTime, tx, gp.CumulativeUsed(), root), result, nil } // MakeReceipt generates the receipt object for a transaction given its execution result. -func MakeReceipt(evm *vm.EVM, result *ExecutionResult, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, blockTime uint64, tx *types.Transaction, usedGas uint64, root []byte) *types.Receipt { - // Create a new receipt for the transaction, storing the intermediate root and gas used - // by the tx. - receipt := &types.Receipt{Type: tx.Type(), PostState: root, CumulativeGasUsed: usedGas} +func MakeReceipt(evm *vm.EVM, result *ExecutionResult, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, blockTime uint64, tx *types.Transaction, cumulativeGas uint64, root []byte) *types.Receipt { + // Create a new receipt for the transaction, storing the intermediate root + // and gas used by the tx. + // + // The cumulative gas used equals the sum of gasUsed across all preceding + // txs with refunded gas deducted. + receipt := &types.Receipt{Type: tx.Type(), PostState: root, CumulativeGasUsed: cumulativeGas} if result.Failed() { receipt.Status = types.ReceiptStatusFailed } else { receipt.Status = types.ReceiptStatusSuccessful } receipt.TxHash = tx.Hash() + + // GasUsed = max(tx_gas_used - gas_refund, calldata_floor_gas_cost), unchanged + // in the Amsterdam fork. receipt.GasUsed = result.UsedGas if evm.Config.ExposeMultiGas { @@ -223,19 +245,19 @@ func MakeReceipt(evm *vm.EVM, result *ExecutionResult, statedb *state.StateDB, b // ApplyTransaction attempts to apply a transaction to the given state database // and uses the input parameters for its environment. It returns the receipt -// for the transaction, gas used and an error if the transaction failed, +// for the transaction and an error if the transaction failed, // indicating the block was invalid. -func ApplyTransaction(evm *vm.EVM, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64) (*types.Receipt, *ExecutionResult, error) { - return ApplyTransactionWithResultFilter(evm, gp, statedb, header, tx, usedGas, NewMessageReplayContext(), nil) +func ApplyTransaction(evm *vm.EVM, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction) (*types.Receipt, *ExecutionResult, error) { + return ApplyTransactionWithResultFilter(evm, gp, statedb, header, tx, NewMessageReplayContext(), nil) } -func ApplyTransactionWithResultFilter(evm *vm.EVM, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64, runCtx *MessageRunContext, resultFilter func(*ExecutionResult) error) (*types.Receipt, *ExecutionResult, error) { +func ApplyTransactionWithResultFilter(evm *vm.EVM, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, runCtx *MessageRunContext, resultFilter func(*ExecutionResult) error) (*types.Receipt, *ExecutionResult, error) { msg, err := TransactionToMessage(tx, types.MakeSigner(evm.ChainConfig(), header.Number, header.Time, evm.Context.ArbOSVersion), header.BaseFee, runCtx) if err != nil { return nil, nil, err } // Create a new context to be used in the EVM environment - return ApplyTransactionWithEVM(msg, gp, statedb, header.Number, header.Hash(), header.Time, tx, usedGas, evm, resultFilter) + return ApplyTransactionWithEVM(msg, gp, statedb, header.Number, header.Hash(), header.Time, tx, evm, resultFilter) } // ProcessBeaconBlockRoot applies the EIP-4788 system call to the beacon block root diff --git a/core/state_processor_test.go b/core/state_processor_test.go index ea52b7b44b..ef7b3ba907 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -423,6 +423,9 @@ func GenerateBadBlock(parent *types.Block, engine consensus.Engine, txs types.Tr beaconRoot := common.HexToHash("0xbeac00") header.ParentBeaconRoot = &beaconRoot } + if config.IsAmsterdam(header.Number, header.Time) { + header.SlotNumber = new(uint64) + } // Assemble and return the final block for sealing body := &types.Body{Transactions: txs} if config.IsShanghai(header.Number, header.Time, types.DeserializeHeaderExtraInformation(header).ArbOSFormatVersion) { diff --git a/core/state_transition.go b/core/state_transition.go index 97331bf3cf..42e8c76244 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -36,7 +36,7 @@ import ( // ExecutionResult includes all output after executing given evm // message no matter the execution itself is successful or not. type ExecutionResult struct { - UsedGas uint64 // Total used gas but include the refunded gas + UsedGas uint64 // Total used gas, refunded gas is deducted MaxUsedGas uint64 // Maximum gas consumed during execution, excluding gas refunds. Err error // Any error encountered during the execution(listed in core/vm/errors.go) ReturnData []byte // Returned data from evm(function result or data supplied with revert opcode) @@ -350,9 +350,9 @@ func TransactionToMessage(tx *types.Transaction, s types.Signer, baseFee *big.In Nonce: tx.Nonce(), GasLimit: tx.Gas(), - GasPrice: new(big.Int).Set(tx.GasPrice()), - GasFeeCap: new(big.Int).Set(tx.GasFeeCap()), - GasTipCap: new(big.Int).Set(tx.GasTipCap()), + GasPrice: tx.GasPrice(), + GasFeeCap: tx.GasFeeCap(), + GasTipCap: tx.GasTipCap(), To: tx.To(), Value: tx.Value(), Data: tx.Data(), @@ -383,6 +383,11 @@ func TransactionToMessage(tx *types.Transaction, s types.Signer, baseFee *big.In // indicates a core error meaning that the message would always fail for that particular // state and would never be accepted within a block. func ApplyMessage(evm *vm.EVM, msg *Message, gp *GasPool) (*ExecutionResult, error) { + // Do not panic if the gas pool is nil. This is allowed when executing + // a single message via RPC invocation. + if gp == nil { + gp = NewGasPool(msg.GasLimit) + } evm.SetTxContext(NewEVMTxContext(msg)) return newStateTransition(evm, msg, gp).execute() } @@ -476,8 +481,8 @@ func (st *stateTransition) buyGas() error { st.evm.Config.Tracer.OnGasChange(0, st.msg.GasLimit, tracing.GasChangeTxInitialBalance) } st.gasRemaining = st.msg.GasLimit - st.initialGas = st.msg.GasLimit + mgvalU256, _ := uint256.FromBig(mgval) st.state.SubBalance(st.msg.From, mgvalU256, tracing.BalanceDecreaseGasBuy) @@ -607,6 +612,18 @@ func (st *stateTransition) execute() (*ExecutionResult, error) { startHookUsedSingleGas := startHookUsedMultiGas.SingleGas() if endTxNow { + // Arbitrum endTxNow paths (deposits, retryable submission, internal txs) + // bypass the normal SubGas/ReturnGas flow but still report gas used. + // Bump gp.cumulativeUsed so receipt.CumulativeGasUsed (and the GasUsed + // derived from it via DeriveFields) reflects this tx's contribution. + if startHookUsedSingleGas > 0 { + if subErr := st.gp.SubGas(startHookUsedSingleGas); subErr != nil { + return nil, subErr + } + if retErr := st.gp.ReturnGas(startHookUsedSingleGas, startHookUsedSingleGas); retErr != nil { + return nil, retErr + } + } return &ExecutionResult{ UsedGas: startHookUsedSingleGas, MaxUsedGas: startHookUsedSingleGas, @@ -697,8 +714,10 @@ func (st *stateTransition) execute() (*ExecutionResult, error) { } // Check whether the init code size has been exceeded. - if rules.IsShanghai && contractCreation && len(msg.Data) > int(st.evm.ChainConfig().MaxInitCodeSize()) { - return nil, fmt.Errorf("%w: code size %v limit %v", ErrMaxInitCodeSizeExceeded, len(msg.Data), int(st.evm.ChainConfig().MaxInitCodeSize())) + if contractCreation { + if err := vm.CheckMaxInitCodeSize(&rules, uint64(len(msg.Data))); err != nil { + return nil, err + } } // Execute the preparatory steps for state transition which includes: @@ -777,8 +796,20 @@ func (st *stateTransition) execute() (*ExecutionResult, error) { peakGasUsed = floorDataGas } } + // Return gas to the user st.returnGas() + // Return gas to the gas pool + if rules.IsAmsterdam { + // Refund is excluded for returning + err = st.gp.ReturnGas(st.initialGas-peakGasUsed, st.gasUsed()) + } else { + // Refund is included for returning + err = st.gp.ReturnGas(st.gasRemaining, st.gasUsed()) + } + if err != nil { + return nil, err + } effectiveTip := msg.GasPrice if rules.IsLondon { effectiveTip = new(big.Int).Sub(msg.GasPrice, st.evm.Context.BaseFee) @@ -806,7 +837,6 @@ func (st *stateTransition) execute() (*ExecutionResult, error) { st.evm.AccessEvents.AddAccount(st.evm.Context.Coinbase, true, math.MaxUint64) } } - // Arbitrum: record the tip if tracer := st.evm.Config.Tracer; tracer != nil && st.evm.ProcessingHook.CollectTips() && tracer.CaptureArbitrumTransfer != nil { tracer.CaptureArbitrumTransfer(nil, &tipReceipient, tipAmount, false, tracing.BalanceIncreaseRewardTransactionFee) @@ -823,6 +853,9 @@ func (st *stateTransition) execute() (*ExecutionResult, error) { } } + if rules.IsAmsterdam { + st.evm.StateDB.EmitLogsForBurnAccounts() + } return &ExecutionResult{ UsedGas: gasUsed, MaxUsedGas: peakGasUsed, @@ -933,10 +966,6 @@ func (st *stateTransition) returnGas() { if tracer := st.evm.Config.Tracer; tracer != nil && tracer.CaptureArbitrumTransfer != nil { tracer.CaptureArbitrumTransfer(nil, &st.msg.From, remaining.ToBig(), false, tracing.BalanceIncreaseGasReturn) } - - // Also return remaining gas to the block gas counter so it is - // available for the next transaction. - st.gp.AddGas(st.gasRemaining) } // gasUsed returns the amount of gas used up by the state transition. diff --git a/core/state_transition_test.go b/core/state_transition_test.go index 32fc0322e7..d832ae859c 100644 --- a/core/state_transition_test.go +++ b/core/state_transition_test.go @@ -2,7 +2,7 @@ package core import ( "bytes" - "errors" + "fmt" "testing" "github.com/ethereum/go-ethereum/arbitrum/multigas" @@ -46,8 +46,7 @@ func TestApplyMessageReturnsMultiGas(t *testing.T) { To: &contract, } - gp := new(GasPool) - gp.SetGas(30_000_000) + gp := NewGasPool(30_000_000) res, err := ApplyMessage(evm, msg, gp) if err != nil { @@ -94,7 +93,7 @@ func TestApplyMessageCalldataReturnsMultiGas(t *testing.T) { To: &contract, } - gp := new(GasPool).AddGas(30_000_000) + gp := NewGasPool(30_000_000) res, err := ApplyMessage(evm, msg, gp) if err != nil { @@ -339,12 +338,8 @@ func TestCallVariantsMultiGas(t *testing.T) { callFn: func() ([]byte, uint64, multigas.MultiGas, error) { return evm.StaticCall(caller, contractAddr, nil, gasLimit) }, - expectErr: vm.ErrWriteProtection, - expectedMultiGas: multigas.MultiGasFromPairs( - multigas.Pair{Kind: multigas.ResourceKindComputation, Amount: refinedComputationGas}, - multigas.Pair{Kind: multigas.ResourceKindStorageAccessRead, Amount: params.ColdSloadCostEIP2929}, - multigas.Pair{Kind: multigas.ResourceKindStorageGrowth, Amount: params.SstoreSetGasEIP2200}, - ), + expectErr: fmt.Errorf("%w: %v", vm.ErrOutOfGas, vm.ErrWriteProtection), + expectedMultiGas: multigas.ComputationGas(refinedComputationGas + params.ColdSloadCostEIP2929 + params.SstoreSetGasEIP2200), }, } @@ -352,8 +347,15 @@ func TestCallVariantsMultiGas(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ret, leftover, usedMultiGas, err := tt.callFn() - if !errors.Is(err, tt.expectErr) { - t.Fatalf("unexpected error: got %v, want %v", err, tt.expectErr) + if tt.expectErr != nil { + if err == nil { + t.Fatalf("unexpected error, got: nil, want: %v", tt.expectErr) + } + if err.Error() != tt.expectErr.Error() { + t.Fatalf("unexpected error, got: %v, want: %v", err, tt.expectErr) + } + } else if err != nil { + t.Fatalf("unexpected error, got: %v, want: nil", err) } if gasLimit-leftover != usedMultiGas.SingleGas() { diff --git a/core/stateless.go b/core/stateless.go index b20c909da6..86d4dc304b 100644 --- a/core/stateless.go +++ b/core/stateless.go @@ -17,6 +17,8 @@ package core import ( + "context" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/consensus/beacon" @@ -40,7 +42,7 @@ import ( // - It cannot be placed outside of core, because it needs to construct a dud headerchain // // TODO(karalabe): Would be nice to resolve both issues above somehow and move it. -func ExecuteStateless(config *params.ChainConfig, vmconfig vm.Config, block *types.Block, witness *stateless.Witness) (common.Hash, common.Hash, error) { +func ExecuteStateless(ctx context.Context, config *params.ChainConfig, vmconfig vm.Config, block *types.Block, witness *stateless.Witness) (common.Hash, common.Hash, error) { // Sanity check if the supplied block accidentally contains a set root or // receipt hash. If so, be very loud, but still continue. if block.Root() != (common.Hash{}) { @@ -51,7 +53,7 @@ func ExecuteStateless(config *params.ChainConfig, vmconfig vm.Config, block *typ } // Create and populate the state database to serve as the stateless backend memdb := witness.MakeHashDB() - db, err := state.New(witness.Root(), state.NewDatabase(triedb.NewDatabase(memdb, triedb.HashDefaults), nil)) + db, err := state.New(witness.Root(), state.NewDatabase(triedb.NewDatabase(memdb, triedb.HashDefaults), state.NewCodeDB(memdb))) if err != nil { return common.Hash{}, common.Hash{}, err } @@ -66,7 +68,7 @@ func ExecuteStateless(config *params.ChainConfig, vmconfig vm.Config, block *typ validator := NewBlockValidator(config, nil) // No chain, we only validate the state, not the block // Run the stateless blocks processing and self-validate certain fields - res, err := processor.Process(block, db, vmconfig) + res, err := processor.Process(ctx, block, db, vmconfig) if err != nil { return common.Hash{}, common.Hash{}, err } diff --git a/core/stateless/encoding.go b/core/stateless/encoding.go index 5c43159e66..d559178892 100644 --- a/core/stateless/encoding.go +++ b/core/stateless/encoding.go @@ -40,8 +40,8 @@ func (w *Witness) ToExtWitness() *ExtWitness { return ext } -// fromExtWitness converts the consensus witness format into our internal one. -func (w *Witness) fromExtWitness(ext *ExtWitness) error { +// FromExtWitness converts the consensus witness format into our internal one. +func (w *Witness) FromExtWitness(ext *ExtWitness) error { w.Headers = ext.Headers w.Codes = make(map[string]struct{}, len(ext.Codes)) @@ -66,7 +66,7 @@ func (w *Witness) DecodeRLP(s *rlp.Stream) error { if err := s.Decode(&ext); err != nil { return err } - return w.fromExtWitness(&ext) + return w.FromExtWitness(&ext) } // ExtWitness is a witness RLP encoding for transferring across clients. diff --git a/core/stateless/stats.go b/core/stateless/stats.go index 94f5587f99..8c05b23d37 100644 --- a/core/stateless/stats.go +++ b/core/stateless/stats.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/trie" ) var accountTrieLeavesAtDepth [16]*metrics.Counter @@ -41,52 +42,75 @@ func init() { // WitnessStats aggregates statistics for account and storage trie accesses. type WitnessStats struct { - accountTrieLeaves [16]int64 - storageTrieLeaves [16]int64 + accountTrie *trie.LevelStats + storageTrie *trie.LevelStats } // NewWitnessStats creates a new WitnessStats collector. func NewWitnessStats() *WitnessStats { - return &WitnessStats{} + return &WitnessStats{ + accountTrie: trie.NewLevelStats(), + storageTrie: trie.NewLevelStats(), + } +} + +func (s *WitnessStats) copy() *WitnessStats { + return &WitnessStats{ + accountTrie: s.accountTrie.Copy(), + storageTrie: s.storageTrie.Copy(), + } +} + +func (s *WitnessStats) init() { + if s.accountTrie == nil { + s.accountTrie = trie.NewLevelStats() + } + if s.storageTrie == nil { + s.storageTrie = trie.NewLevelStats() + } } // Add records trie access depths from the given node paths. // If `owner` is the zero hash, accesses are attributed to the account trie; // otherwise, they are attributed to the storage trie of that account. func (s *WitnessStats) Add(nodes map[string][]byte, owner common.Hash) { - // Extract paths from the nodes map + s.init() + + // Extract paths from the nodes map. paths := slices.Collect(maps.Keys(nodes)) sort.Strings(paths) + ownerStat := s.accountTrie + if owner != (common.Hash{}) { + ownerStat = s.storageTrie + } + for i, path := range paths { // If current path is a prefix of the next path, it's not a leaf. // The last path is always a leaf. if i == len(paths)-1 || !strings.HasPrefix(paths[i+1], paths[i]) { - if owner == (common.Hash{}) { - s.accountTrieLeaves[len(path)] += 1 - } else { - s.storageTrieLeaves[len(path)] += 1 - } + ownerStat.AddLeaf(len(path)) } } } // ReportMetrics reports the collected statistics to the global metrics registry. func (s *WitnessStats) ReportMetrics(blockNumber uint64) { - // Encode the metrics as JSON for easier consumption - accountLeavesJson, _ := json.Marshal(s.accountTrieLeaves) - storageLeavesJson, _ := json.Marshal(s.storageTrieLeaves) - - // Log account trie depth statistics - log.Info("Account trie depth stats", - "block", blockNumber, - "leavesAtDepth", string(accountLeavesJson)) - log.Info("Storage trie depth stats", - "block", blockNumber, - "leavesAtDepth", string(storageLeavesJson)) + s.init() - for i := 0; i < 16; i++ { - accountTrieLeavesAtDepth[i].Inc(s.accountTrieLeaves[i]) - storageTrieLeavesAtDepth[i].Inc(s.storageTrieLeaves[i]) + accountTrieLeaves := s.accountTrie.LeafDepths() + storageTrieLeaves := s.storageTrie.LeafDepths() + + // Encode the metrics as JSON for easier consumption. + accountLeavesJSON, _ := json.Marshal(accountTrieLeaves) + storageLeavesJSON, _ := json.Marshal(storageTrieLeaves) + + // Log account trie depth statistics. + log.Info("Account trie depth stats", "block", blockNumber, "leavesAtDepth", string(accountLeavesJSON)) + log.Info("Storage trie depth stats", "block", blockNumber, "leavesAtDepth", string(storageLeavesJSON)) + + for i := 0; i < len(accountTrieLeavesAtDepth); i++ { + accountTrieLeavesAtDepth[i].Inc(accountTrieLeaves[i]) + storageTrieLeavesAtDepth[i].Inc(storageTrieLeaves[i]) } } diff --git a/core/stateless/stats_test.go b/core/stateless/stats_test.go index e77084df5d..6dc1fa0844 100644 --- a/core/stateless/stats_test.go +++ b/core/stateless/stats_test.go @@ -17,18 +17,27 @@ package stateless import ( + "strings" "testing" "github.com/ethereum/go-ethereum/common" ) +func expectedLeaves(counts map[int]int64) [16]int64 { + var leaves [16]int64 + for depth, count := range counts { + leaves[depth] = count + } + return leaves +} + func TestWitnessStatsAdd(t *testing.T) { tests := []struct { name string nodes map[string][]byte owner common.Hash - expectedAccountLeaves map[int64]int64 - expectedStorageLeaves map[int64]int64 + expectedAccountLeaves map[int]int64 + expectedStorageLeaves map[int]int64 }{ { name: "empty nodes", @@ -41,7 +50,7 @@ func TestWitnessStatsAdd(t *testing.T) { "": []byte("data"), }, owner: common.Hash{}, - expectedAccountLeaves: map[int64]int64{0: 1}, + expectedAccountLeaves: map[int]int64{0: 1}, }, { name: "single account trie leaf", @@ -49,7 +58,7 @@ func TestWitnessStatsAdd(t *testing.T) { "abc": []byte("data"), }, owner: common.Hash{}, - expectedAccountLeaves: map[int64]int64{3: 1}, + expectedAccountLeaves: map[int]int64{3: 1}, }, { name: "account trie with internal nodes", @@ -59,7 +68,7 @@ func TestWitnessStatsAdd(t *testing.T) { "abc": []byte("data3"), }, owner: common.Hash{}, - expectedAccountLeaves: map[int64]int64{3: 1}, // Only "abc" is a leaf + expectedAccountLeaves: map[int]int64{3: 1}, // Only "abc" is a leaf }, { name: "multiple account trie branches", @@ -72,7 +81,7 @@ func TestWitnessStatsAdd(t *testing.T) { "bcd": []byte("data6"), }, owner: common.Hash{}, - expectedAccountLeaves: map[int64]int64{3: 2}, // "abc" (3) + "bcd" (3) + expectedAccountLeaves: map[int]int64{3: 2}, // "abc" (3) + "bcd" (3) }, { name: "siblings are all leaves", @@ -82,7 +91,7 @@ func TestWitnessStatsAdd(t *testing.T) { "ac": []byte("data3"), }, owner: common.Hash{}, - expectedAccountLeaves: map[int64]int64{2: 3}, + expectedAccountLeaves: map[int]int64{2: 3}, }, { name: "storage trie leaves", @@ -93,7 +102,7 @@ func TestWitnessStatsAdd(t *testing.T) { "124": []byte("data4"), }, owner: common.HexToHash("0x1234"), - expectedStorageLeaves: map[int64]int64{3: 2}, // "123" (3) + "124" (3) + expectedStorageLeaves: map[int]int64{3: 2}, // "123" (3) + "124" (3) }, { name: "complex trie structure", @@ -109,7 +118,7 @@ func TestWitnessStatsAdd(t *testing.T) { "3": []byte("data9"), }, owner: common.Hash{}, - expectedAccountLeaves: map[int64]int64{1: 1, 3: 4}, // "123"(3) + "124"(3) + "234"(3) + "235"(3) + "3"(1) + expectedAccountLeaves: map[int]int64{1: 1, 3: 4}, // "123"(3) + "124"(3) + "234"(3) + "235"(3) + "3"(1) }, } @@ -118,32 +127,59 @@ func TestWitnessStatsAdd(t *testing.T) { stats := NewWitnessStats() stats.Add(tt.nodes, tt.owner) - var expectedAccountTrieLeaves [16]int64 - for depth, count := range tt.expectedAccountLeaves { - expectedAccountTrieLeaves[depth] = count + if got, want := stats.accountTrie.LeafDepths(), expectedLeaves(tt.expectedAccountLeaves); got != want { + t.Errorf("account trie leaves = %v, want %v", got, want) } - var expectedStorageTrieLeaves [16]int64 - for depth, count := range tt.expectedStorageLeaves { - expectedStorageTrieLeaves[depth] = count + if got, want := stats.storageTrie.LeafDepths(), expectedLeaves(tt.expectedStorageLeaves); got != want { + t.Errorf("storage trie leaves = %v, want %v", got, want) } + }) + } +} - // Check account trie depth - if stats.accountTrieLeaves != expectedAccountTrieLeaves { - t.Errorf("Account trie total depth = %v, want %v", stats.accountTrieLeaves, expectedAccountTrieLeaves) - } +func TestWitnessStatsStorageTrieAggregation(t *testing.T) { + stats := NewWitnessStats() + ownerA := common.HexToHash("0xa") + ownerB := common.HexToHash("0xb") - // Check storage trie depth - if stats.storageTrieLeaves != expectedStorageTrieLeaves { - t.Errorf("Storage trie total depth = %v, want %v", stats.storageTrieLeaves, expectedStorageTrieLeaves) - } - }) + stats.Add(map[string][]byte{ + "a": []byte("data1"), + "ab": []byte("data2"), + "abc": []byte("data3"), + }, ownerA) + stats.Add(map[string][]byte{ + "xy": []byte("data4"), + }, ownerA) + stats.Add(map[string][]byte{ + "1": []byte("data5"), + "12": []byte("data6"), + "123": []byte("data7"), + "124": []byte("data8"), + }, ownerB) + + if got, want := stats.storageTrie.LeafDepths(), expectedLeaves(map[int]int64{2: 1, 3: 3}); got != want { + t.Errorf("storage leaves = %v, want %v", got, want) } + if got, want := stats.accountTrie.LeafDepths(), expectedLeaves(nil); got != want { + t.Errorf("account leaves = %v, want %v", got, want) + } +} + +func TestWitnessStatsPanicsOnDeepLeaf(t *testing.T) { + stats := NewWitnessStats() + + defer func() { + if r := recover(); r == nil { + t.Fatal("expected panic for depth >= 16") + } + }() + stats.Add(map[string][]byte{strings.Repeat("a", 16): []byte("data")}, common.Hash{}) } func TestWitnessStatsMinMax(t *testing.T) { stats := NewWitnessStats() - // Add some account trie nodes with varying depths + // Add some account trie nodes with varying depths. stats.Add(map[string][]byte{ "a": []byte("data1"), "ab": []byte("data2"), @@ -152,21 +188,21 @@ func TestWitnessStatsMinMax(t *testing.T) { "abcde": []byte("data5"), }, common.Hash{}) - // Only "abcde" is a leaf (depth 5) - for i, v := range stats.accountTrieLeaves { + // Only "abcde" is a leaf (depth 5). + for i, v := range stats.accountTrie.LeafDepths() { if v != 0 && i != 5 { t.Errorf("leaf found at invalid depth %d", i) } } - // Add more leaves with different depths + // Add more leaves with different depths. stats.Add(map[string][]byte{ "x": []byte("data6"), "yz": []byte("data7"), }, common.Hash{}) - // Now we have leaves at depths 1, 2, and 5 - for i, v := range stats.accountTrieLeaves { + // Now we have leaves at depths 1, 2, and 5. + for i, v := range stats.accountTrie.LeafDepths() { if v != 0 && (i != 5 && i != 2 && i != 1) { t.Errorf("leaf found at invalid depth %d", i) } @@ -176,7 +212,7 @@ func TestWitnessStatsMinMax(t *testing.T) { func TestWitnessStatsAverage(t *testing.T) { stats := NewWitnessStats() - // Add nodes that will create leaves at depths 2, 3, and 4 + // Add nodes that will create leaves at depths 2, 3, and 4. stats.Add(map[string][]byte{ "aa": []byte("data1"), "bb": []byte("data2"), @@ -184,22 +220,22 @@ func TestWitnessStatsAverage(t *testing.T) { "dddd": []byte("data4"), }, common.Hash{}) - // All are leaves: 2 + 2 + 3 + 4 = 11 total, 4 samples + // All are leaves: 2 + 2 + 3 + 4 = 11 total, 4 samples. expectedAvg := int64(11) / int64(4) var actualAvg, totalSamples int64 - for i, c := range stats.accountTrieLeaves { + for i, c := range stats.accountTrie.LeafDepths() { actualAvg += c * int64(i) totalSamples += c } actualAvg = actualAvg / totalSamples if actualAvg != expectedAvg { - t.Errorf("Account trie average depth = %d, want %d", actualAvg, expectedAvg) + t.Errorf("account trie average depth = %d, want %d", actualAvg, expectedAvg) } } func BenchmarkWitnessStatsAdd(b *testing.B) { - // Create a realistic trie node structure + // Create a realistic trie node structure. nodes := make(map[string][]byte) for i := 0; i < 100; i++ { base := string(rune('a' + i%26)) diff --git a/core/stateless/witness.go b/core/stateless/witness.go index 588c895a2f..f1321699c1 100644 --- a/core/stateless/witness.go +++ b/core/stateless/witness.go @@ -42,12 +42,13 @@ type Witness struct { Codes map[string]struct{} // Set of bytecodes ran or accessed State map[string]struct{} // Set of MPT state trie nodes (account and storage together) - chain HeaderReader // Chain reader to convert block hash ops to header proofs - lock sync.Mutex // Lock to allow concurrent state insertions + chain HeaderReader // Chain reader to convert block hash ops to header proofs + stats *WitnessStats // Optional statistics collector + lock sync.Mutex // Lock to allow concurrent state insertions } // NewWitness creates an empty witness ready for population. -func NewWitness(context *types.Header, chain HeaderReader) (*Witness, error) { +func NewWitness(context *types.Header, chain HeaderReader, enableStats bool) (*Witness, error) { // When building witnesses, retrieve the parent header, which will *always* // be included to act as a trustless pre-root hash container var headers []*types.Header @@ -59,13 +60,17 @@ func NewWitness(context *types.Header, chain HeaderReader) (*Witness, error) { headers = append(headers, parent) } // Create the witness with a reconstructed gutted out block - return &Witness{ + w := &Witness{ context: context, Headers: headers, Codes: make(map[string]struct{}), State: make(map[string]struct{}), chain: chain, - }, nil + } + if enableStats { + w.stats = NewWitnessStats() + } + return w, nil } // AddBlockHash adds a "blockhash" to the witness with the designated offset from @@ -87,8 +92,11 @@ func (w *Witness) AddCode(code []byte) { w.Codes[string(code)] = struct{}{} } -// AddState inserts a batch of MPT trie nodes into the witness. -func (w *Witness) AddState(nodes map[string][]byte) { +// AddState inserts a batch of MPT trie nodes into the witness. The owner +// identifies which trie the nodes belong to: the zero hash for the account +// trie, or the hashed address for a storage trie. This is used for optional +// statistics collection. +func (w *Witness) AddState(nodes map[string][]byte, owner common.Hash) { if len(nodes) == 0 { return } @@ -98,6 +106,17 @@ func (w *Witness) AddState(nodes map[string][]byte) { for _, value := range nodes { w.State[string(value)] = struct{}{} } + if w.stats != nil { + w.stats.Add(nodes, owner) + } +} + +// ReportMetrics reports the collected statistics to the global metrics registry. +func (w *Witness) ReportMetrics(blockNumber uint64) { + if w.stats == nil { + return + } + w.stats.ReportMetrics(blockNumber) } func (w *Witness) AddKey() { @@ -113,6 +132,9 @@ func (w *Witness) Copy() *Witness { State: maps.Clone(w.State), chain: w.chain, } + if w.stats != nil { + cpy.stats = w.stats.copy() + } if w.context != nil { cpy.context = types.CopyHeader(w.context) } diff --git a/core/tracing/gen_code_change_reason_stringer.go b/core/tracing/gen_code_change_reason_stringer.go index 9372954063..2531b10471 100644 --- a/core/tracing/gen_code_change_reason_stringer.go +++ b/core/tracing/gen_code_change_reason_stringer.go @@ -22,8 +22,9 @@ const _CodeChangeReason_name = "UnspecifiedContractCreationGenesisAuthorizationA var _CodeChangeReason_index = [...]uint8{0, 11, 27, 34, 47, 65, 77, 83} func (i CodeChangeReason) String() string { - if i >= CodeChangeReason(len(_CodeChangeReason_index)-1) { + idx := int(i) - 0 + if i < 0 || idx >= len(_CodeChangeReason_index)-1 { return "CodeChangeReason(" + strconv.FormatInt(int64(i), 10) + ")" } - return _CodeChangeReason_name[_CodeChangeReason_index[i]:_CodeChangeReason_index[i+1]] + return _CodeChangeReason_name[_CodeChangeReason_index[idx]:_CodeChangeReason_index[idx+1]] } diff --git a/core/tracing/gen_nonce_change_reason_stringer.go b/core/tracing/gen_nonce_change_reason_stringer.go index f775c1f3a6..8c9099b7ce 100644 --- a/core/tracing/gen_nonce_change_reason_stringer.go +++ b/core/tracing/gen_nonce_change_reason_stringer.go @@ -15,15 +15,17 @@ func _() { _ = x[NonceChangeNewContract-4] _ = x[NonceChangeAuthorization-5] _ = x[NonceChangeRevert-6] + _ = x[NonceChangeSelfdestruct-7] } -const _NonceChangeReason_name = "UnspecifiedGenesisEoACallContractCreatorNewContractAuthorizationRevert" +const _NonceChangeReason_name = "UnspecifiedGenesisEoACallContractCreatorNewContractAuthorizationRevertSelfdestruct" -var _NonceChangeReason_index = [...]uint8{0, 11, 18, 25, 40, 51, 64, 70} +var _NonceChangeReason_index = [...]uint8{0, 11, 18, 25, 40, 51, 64, 70, 82} func (i NonceChangeReason) String() string { - if i >= NonceChangeReason(len(_NonceChangeReason_index)-1) { + idx := int(i) - 0 + if i < 0 || idx >= len(_NonceChangeReason_index)-1 { return "NonceChangeReason(" + strconv.FormatInt(int64(i), 10) + ")" } - return _NonceChangeReason_name[_NonceChangeReason_index[i]:_NonceChangeReason_index[i+1]] + return _NonceChangeReason_name[_NonceChangeReason_index[idx]:_NonceChangeReason_index[idx+1]] } diff --git a/core/tracing/hooks.go b/core/tracing/hooks.go index 24aeae76d0..75ce432677 100644 --- a/core/tracing/hooks.go +++ b/core/tracing/hooks.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie/trienode" "github.com/holiman/uint256" ) @@ -80,6 +81,56 @@ type BlockEvent struct { Safe *types.Header } +// StateUpdate represents the state mutations resulting from block execution. +// It provides access to account changes, storage changes, and contract code +// deployments with both previous and new values. +type StateUpdate struct { + OriginRoot common.Hash // State root before the update + Root common.Hash // State root after the update + BlockNumber uint64 + + // AccountChanges contains all account state changes keyed by address. + AccountChanges map[common.Address]*AccountChange + + // StorageChanges contains all storage slot changes keyed by address and storage slot key. + StorageChanges map[common.Address]map[common.Hash]*StorageChange + + // CodeChanges contains all contract code changes keyed by address. + CodeChanges map[common.Address]*CodeChange + + // TrieChanges contains trie node mutations keyed by address hash and trie node path. + TrieChanges map[common.Hash]map[string]*TrieNodeChange +} + +// AccountChange represents a change to an account's state. +type AccountChange struct { + Prev *types.StateAccount // nil if account was created + New *types.StateAccount // nil if account was deleted +} + +// StorageChange represents a change to a storage slot. +type StorageChange struct { + Prev common.Hash // previous value (zero if slot was created) + New common.Hash // new value (zero if slot was deleted) +} + +type ContractCode struct { + Hash common.Hash + Code []byte + Exists bool // true if the code was existent +} + +// CodeChange represents a change in contract code of an account. +type CodeChange struct { + Prev *ContractCode // nil if no code existed before + New *ContractCode +} + +type TrieNodeChange struct { + Prev *trienode.Node + New *trienode.Node +} + type ( /* - VM events - @@ -169,6 +220,11 @@ type ( // beacon block root. OnSystemCallEndHook = func() + // StateUpdateHook is called after state is committed for a block. + // It provides access to the complete state mutations including account changes, + // storage changes, trie node mutations, and contract code deployments. + StateUpdateHook = func(update *StateUpdate) + /* - State events - */ @@ -224,6 +280,7 @@ type Hooks struct { OnSystemCallStart OnSystemCallStartHook OnSystemCallStartV2 OnSystemCallStartHookV2 OnSystemCallEnd OnSystemCallEndHook + OnStateUpdate StateUpdateHook // State events OnBalanceChange BalanceChangeHook OnNonceChange NonceChangeHook @@ -392,7 +449,7 @@ const ( // this generates an increase in gas. There is at most one of such gas change per transaction. GasChangeTxRefunds GasChangeReason = 3 // GasChangeTxLeftOverReturned is the amount of gas left over at the end of transaction's execution that will be returned - // to the chain. This change will always be a negative change as we "drain" left over gas towards 0. If there was no gas + // to the account. This change will always be a negative change as we "drain" left over gas towards 0. If there was no gas // left at the end of execution, no such even will be emitted. The returned gas's value in Wei is returned to caller. // There is at most one of such gas change per transaction. GasChangeTxLeftOverReturned GasChangeReason = 4 @@ -460,12 +517,15 @@ const ( // NonceChangeNewContract is the nonce change of a newly created contract. NonceChangeNewContract NonceChangeReason = 4 - // NonceChangeTransaction is the nonce change due to a EIP-7702 authorization. + // NonceChangeAuthorization is the nonce change due to a EIP-7702 authorization. NonceChangeAuthorization NonceChangeReason = 5 // NonceChangeRevert is emitted when the nonce is reverted back to a previous value due to call failure. // It is only emitted when the tracer has opted in to use the journaling wrapper (WrapWithJournal). NonceChangeRevert NonceChangeReason = 6 + + // NonceChangeSelfdestruct is emitted when the nonce is reset to zero due to a self-destruct + NonceChangeSelfdestruct NonceChangeReason = 7 ) // CodeChangeReason is used to indicate the reason for a code change. diff --git a/core/tracing/journal.go b/core/tracing/journal.go index 62a70d6c27..560c937115 100644 --- a/core/tracing/journal.go +++ b/core/tracing/journal.go @@ -155,10 +155,18 @@ func (j *journal) OnBalanceChange(addr common.Address, prev, new *big.Int, reaso } func (j *journal) OnNonceChangeV2(addr common.Address, prev, new uint64, reason NonceChangeReason) { - // When a contract is created, the nonce of the creator is incremented. - // This change is not reverted when the creation fails. - if reason != NonceChangeContractCreator { - j.entries = append(j.entries, nonceChange{addr: addr, prev: prev, new: new}) + j.entries = append(j.entries, nonceChange{addr: addr, prev: prev, new: new}) + if reason == NonceChangeContractCreator { + // When a contract is created via CREATE/CREATE2, the creator's nonce is + // incremented. The EVM does not revert this when the CREATE frame itself + // fails (the nonce change happens before the EVM snapshot). However, if + // a parent frame reverts, the nonce must be reverted along with everything + // else. + // + // To achieve this, advance the current frame's revision point past this + // entry. The CREATE frame's revert won't touch it (it's below the revision), + // but a parent frame's revert will (it's above the parent's revision). + j.revisions[len(j.revisions)-1] = len(j.entries) } if j.hooks.OnNonceChangeV2 != nil { j.hooks.OnNonceChangeV2(addr, prev, new, reason) diff --git a/core/tracing/journal_test.go b/core/tracing/journal_test.go index e00447f5f3..488d192502 100644 --- a/core/tracing/journal_test.go +++ b/core/tracing/journal_test.go @@ -219,6 +219,42 @@ func TestNonceIncOnCreate(t *testing.T) { } } +// TestNonceIncOnCreateParentReverts checks that the creator's nonce increment +// from CREATE survives the CREATE frame's own revert but is properly reverted +// when the parent call frame reverts. +func TestNonceIncOnCreateParentReverts(t *testing.T) { + const opCREATE = 0xf0 + + tr := &testTracer{t: t} + wr, err := WrapWithJournal(&Hooks{OnNonceChange: tr.OnNonceChange}) + if err != nil { + t.Fatalf("failed to wrap test tracer: %v", err) + } + + addr := common.HexToAddress("0x1234") + { + // Parent call frame + wr.OnEnter(0, 0, addr, addr, nil, 1000, big.NewInt(0)) + { + // CREATE frame — creator nonce incremented, then CREATE reverts + wr.OnEnter(1, opCREATE, addr, addr, nil, 1000, big.NewInt(0)) + wr.OnNonceChangeV2(addr, 0, 1, NonceChangeContractCreator) + wr.OnExit(1, nil, 100, errors.New("revert"), true) + } + // After CREATE reverts, nonce should still be 1 + if tr.nonce != 1 { + t.Fatalf("nonce after CREATE revert: got %v, want 1", tr.nonce) + } + // Parent frame also reverts + wr.OnExit(0, nil, 150, errors.New("revert"), true) + } + + // After parent reverts, nonce should be back to 0 + if tr.nonce != 0 { + t.Fatalf("nonce after parent revert: got %v, want 0", tr.nonce) + } +} + func TestOnNonceChangeV2(t *testing.T) { tr := &testTracer{t: t} wr, err := WrapWithJournal(&Hooks{OnNonceChangeV2: tr.OnNonceChangeV2}) diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index 11a1b8b866..8d416ab918 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -21,12 +21,10 @@ import ( "container/heap" "errors" "fmt" - "maps" "math" "math/big" "os" "path/filepath" - "slices" "sort" "sync" "sync/atomic" @@ -97,10 +95,25 @@ const ( // store. storeVersion = 1 - // conversionTimeWindow defines the period after the Osaka fork during which - // the pool will still accept and convert legacy blob transactions. After this - // window, all legacy blob transactions will be rejected. - conversionTimeWindow = time.Hour * 2 + // gappedLifetime is the approximate duration for which nonce-gapped transactions + // are kept before being dropped. Since gapped is only a reorder buffer and it + // is expected that the original transactions were inserted in the mempool in + // nonce order, the duration is kept short to avoid DoS vectors. + gappedLifetime = 1 * time.Minute + + // maxGappedTxs is the maximum number of gapped transactions kept overall. + // This is a safety limit to avoid DoS vectors. + maxGapped = 128 + + // notifyThreshold is the eviction priority threshold above which a transaction + // is considered close enough to being includable to be announced to peers. + // Setting this to zero will disable announcements for anyting not immediately + // includable. Setting it to -1 allows transactions that are close to being + // includable, maybe already in the next block if fees go down, to be announced. + + // Note, this threshold is in the abstract eviction priority space, so its + // meaning depends on the current basefee/blobfee and the transaction's fees. + announceThreshold = -1 ) // blobTxMeta is the minimal subset of types.BlobTx necessary to validate and @@ -112,6 +125,8 @@ type blobTxMeta struct { vhashes []common.Hash // Blob versioned hashes to maintain the lookup table version byte // Blob transaction version to determine proof type + announced bool // Whether the tx has been announced to listeners + id uint64 // Storage ID in the pool's persistent store storageSize uint32 // Byte size in the pool's persistent store size uint64 // RLP-encoded size of transaction including the attached blob @@ -156,7 +171,7 @@ func newBlobTxMeta(id uint64, size uint64, storageSize uint32, tx *types.Transac blobGas: tx.BlobGas(), } meta.basefeeJumps = dynamicFeeJumps(meta.execFeeCap) - meta.blobfeeJumps = dynamicFeeJumps(meta.blobFeeCap) + meta.blobfeeJumps = dynamicBlobFeeJumps(meta.blobFeeCap) return meta } @@ -207,6 +222,14 @@ func newBlobTxMeta(id uint64, size uint64, storageSize uint32, tx *types.Transac // via a normal transaction. It should nonetheless be high enough to support // resurrecting reorged transactions. Perhaps 4-16. // +// - It is not the role of the blobpool to serve as a storage for limit orders +// below market: blob transactions with fee caps way below base fee or blob fee. +// Therefore, the propagation of blob transactions that are far from being +// includable is suppressed. The pool will only announce blob transactions that +// are close to being includable (based on the current fees and the transaction's +// fee caps), and will delay the announcement of blob transactions that are far +// from being includable until base fee and/or blob fee is reduced. +// // - Local txs are meaningless. Mining pools historically used local transactions // for payouts or for backdoor deals. With 1559 in place, the basefee usually // dominates the final price, so 0 or non-0 tip doesn't change much. Blob txs @@ -278,47 +301,54 @@ func newBlobTxMeta(id uint64, size uint64, storageSize uint32, tx *types.Transac // solve after every block. // // - The first observation is that comparing 1559 base fees or 4844 blob fees -// needs to happen in the context of their dynamism. Since these fees jump -// up or down in ~1.125 multipliers (at max) across blocks, comparing fees -// in two transactions should be based on log1.125(fee) to eliminate noise. +// needs to happen in the context of their dynamism. Since base fees are +// adjusted continuously and fluctuate, and we want to optimize for effective +// miner fees, it is better to disregard small base fee cap differences. +// Instead of considering the exact fee cap values, we should group +// transactions into buckets based on fee cap values, allowing us to use +// the miner tip meaningfully as a splitter inside a bucket. // -// - The second observation is that the basefee and blobfee move independently, -// so there's no way to split mixed txs on their own (A has higher base fee, -// B has higher blob fee). Rather than look at the absolute fees, the useful -// metric is the max time it can take to exceed the transaction's fee caps. +// To create these buckets, rather than looking at the absolute fee +// differences, the useful metric is the max time it can take to exceed the +// transaction's fee caps. Base fee changes are multiplicative, so we use a +// logarithmic scale. Fees jumps up or down in ~1.125 multipliers at max +// across blocks, so we use log1.125(fee) and rounding to eliminate noise. // Specifically, we're interested in the number of jumps needed to go from // the current fee to the transaction's cap: // -// jumps = log1.125(txfee) - log1.125(basefee) +// jumps = floor(log1.125(txfee) - log1.125(basefee)) +// +// For blob fees, EIP-7892 changed the ratio of target to max blobs, and +// with that also the maximum blob fee decrease in a slot from 1.125 to +// approx 1.17. therefore, we use: +// +// blobfeeJumps = floor(log1.17(txBlobfee) - log1.17(blobfee)) // -// - The third observation is that the base fee tends to hover around rather -// than swing wildly. The number of jumps needed from the current fee starts -// to get less relevant the higher it is. To remove the noise here too, the -// pool will use log(jumps) as the delta for comparing transactions. +// - The second observation is that when ranking executable blob txs, it +// does not make sense to grant a later eviction priority to txs with high +// fee caps since it could enable pool wars. As such, any positive priority +// will be grouped together. // -// delta = sign(jumps) * log(abs(jumps)) +// priority = min(jumps, 0) // -// - To establish a total order, we need to reduce the dimensionality of the +// - The third observation is that the basefee and blobfee move independently, +// so there's no way to split mixed txs on their own (A has higher base fee, +// B has higher blob fee). +// +// To establish a total order, we need to reduce the dimensionality of the // two base fees (log jumps) to a single value. The interesting aspect from // the pool's perspective is how fast will a tx get executable (fees going // down, crossing the smaller negative jump counter) or non-executable (fees // going up, crossing the smaller positive jump counter). As such, the pool // cares only about the min of the two delta values for eviction priority. // -// priority = min(deltaBasefee, deltaBlobfee) +// priority = min(deltaBasefee, deltaBlobfee, 0) // // - The above very aggressive dimensionality and noise reduction should result // in transaction being grouped into a small number of buckets, the further // the fees the larger the buckets. This is good because it allows us to use // the miner tip meaningfully as a splitter. // -// - For the scenario where the pool does not contain non-executable blob txs -// anymore, it does not make sense to grant a later eviction priority to txs -// with high fee caps since it could enable pool wars. As such, any positive -// priority will be grouped together. -// -// priority = min(deltaBasefee, deltaBlobfee, 0) -// // Optimisation tradeoffs: // // - Eviction relies on 3 fee minimums per account (exec tip, exec cap and blob @@ -337,9 +367,11 @@ type BlobPool struct { stored uint64 // Useful data size of all transactions on disk limbo *limbo // Persistent data store for the non-finalized blobs - signer types.Signer // Transaction signer to use for sender recovery - chain BlockChain // Chain object to access the state through - cQueue *conversionQueue // The queue for performing legacy sidecar conversion (TODO: remove after Osaka) + gapped map[common.Address][]*types.Transaction // Transactions that are currently gapped (nonce too high) + gappedSource map[common.Hash]common.Address // Source of gapped transactions to allow rechecking on inclusion + + signer types.Signer // Transaction signer to use for sender recovery + chain BlockChain // Chain object to access the state through head atomic.Pointer[types.Header] // Current head of the chain state *state.StateDB // Current state at the head of the chain @@ -368,16 +400,22 @@ func New(config Config, chain BlockChain, hasPendingAuth func(common.Address) bo hasPendingAuth: hasPendingAuth, signer: types.LatestSigner(chain.Config()), chain: chain, - cQueue: newConversionQueue(), // Deprecate it after the osaka fork lookup: newLookup(), index: make(map[common.Address][]*blobTxMeta), spent: make(map[common.Address]*uint256.Int), + gapped: make(map[common.Address][]*types.Transaction), + gappedSource: make(map[common.Hash]common.Address), } } // Filter returns whether the given transaction can be consumed by the blob pool. func (p *BlobPool) Filter(tx *types.Transaction) bool { - return tx.Type() == types.BlobTxType + return p.FilterType(tx.Type()) +} + +// FilterType returns whether the blob pool supports the given transaction type. +func (p *BlobPool) FilterType(kind byte) bool { + return kind == types.BlobTxType } // Init sets the gas price needed to keep a transaction in the pool and the chain @@ -414,7 +452,7 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserver txpool.Reser p.state = state // Create new slotter for pre-Osaka blob configuration. - slotter := newSlotter(eip4844.LatestMaxBlobsPerBlock(p.chain.Config())) + slotter := newSlotter(params.BlobTxMaxBlobs) // See if we need to migrate the queue blob store after fusaka slotter, err = tryMigrate(p.chain.Config(), slotter, queuedir) @@ -459,6 +497,20 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserver txpool.Reser } p.evict = newPriceHeap(basefee, blobfee, p.index) + // Guess what was announced. This is needed because we don't want to + // participate in the diffusion of transactions where inclusion is blocked by + // a low base fee transaction. Since we don't persist that info, the best + // we can do is to assume that anything that could have been announced + // at current prices, actually was. + for addr := range p.index { + for _, tx := range p.index[addr] { + tx.announced = p.isAnnouncable(tx) + if !tx.announced { + break + } + } + } + // Pool initialized, attach the blob limbo to it to track blobs included // recently but not yet finalized p.limbo, err = newLimbo(p.chain.Config(), limbodir) @@ -485,9 +537,6 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserver txpool.Reser // Close closes down the underlying persistent store. func (p *BlobPool) Close() error { - // Terminate the conversion queue - p.cQueue.close() - var errs []error if p.limbo != nil { // Close might be invoked due to error in constructor, before p,limbo is set if err := p.limbo.Close(); err != nil { @@ -509,6 +558,7 @@ func (p *BlobPool) Close() error { // parseTransaction is a callback method on pool creation that gets called for // each transaction on disk to create the in-memory metadata index. +// Announced state is not initialized here, it needs to be iniitalized seprately. func (p *BlobPool) parseTransaction(id uint64, size uint32, blob []byte) error { tx := new(types.Transaction) if err := rlp.DecodeBytes(blob, tx); err != nil { @@ -841,6 +891,9 @@ func (p *BlobPool) Reset(oldHead, newHead *types.Header) { resettimeHist.Update(time.Since(start).Nanoseconds()) }(time.Now()) + // Handle reorg buffer timeouts evicting old gapped transactions + p.evictGapped() + statedb, err := p.chain.StateAt(newHead.Root) if err != nil { log.Error("Failed to reset blobpool state", "err", err) @@ -884,175 +937,40 @@ func (p *BlobPool) Reset(oldHead, newHead *types.Header) { } p.evict.reinit(basefee, blobfee, false) - basefeeGauge.Update(int64(basefee.Uint64())) - blobfeeGauge.Update(int64(blobfee.Uint64())) - p.updateStorageMetrics() - - // Perform the conversion logic at the fork boundary - if !p.chain.Config().IsOsaka(oldHead.Number, oldHead.Time, types.DeserializeHeaderExtraInformation(oldHead).ArbOSFormatVersion) && p.chain.Config().IsOsaka(newHead.Number, newHead.Time, types.DeserializeHeaderExtraInformation(newHead).ArbOSFormatVersion) { - // Deep copy all indexed transaction metadata. - var ( - ids = make(map[common.Address]map[uint64]uint64) - txs = make(map[common.Address]map[uint64]common.Hash) - ) - for sender, list := range p.index { - ids[sender] = make(map[uint64]uint64) - txs[sender] = make(map[uint64]common.Hash) - for _, m := range list { - ids[sender][m.nonce] = m.id - txs[sender][m.nonce] = m.hash + // Announce transactions that became announcable due to fee changes + var announcable []*types.Transaction + for addr, txs := range p.index { + for i, meta := range txs { + if !meta.announced && (i == 0 || txs[i-1].announced) && p.isAnnouncable(meta) { + // Load the full transaction and strip the sidecar before announcing + // TODO: this is a bit ugly, as we have everything needed in meta already + data, err := p.store.Get(meta.id) + // Technically, we are supposed to set announced only if Get is successful. + // However, Get failing here indicates a more serious issue (data loss), + // so we set announced anyway to avoid repeated attempts. + meta.announced = true + if err != nil { + log.Error("Blobs missing for announcable transaction", "from", addr, "nonce", meta.nonce, "id", meta.id, "err", err) + continue + } + var tx types.Transaction + if err = rlp.DecodeBytes(data, &tx); err != nil { + log.Error("Blobs corrupted for announcable transaction", "from", addr, "nonce", meta.nonce, "id", meta.id, "err", err) + continue + } + announcable = append(announcable, tx.WithoutBlobTxSidecar()) + log.Trace("Blob transaction now announcable", "from", addr, "nonce", meta.nonce, "id", meta.id, "hash", tx.Hash()) } } - // Initiate the background conversion thread. - p.cQueue.launchBillyConversion(func() { - p.convertLegacySidecars(ids, txs) - }) - } -} - -// compareAndSwap checks if the specified transaction is still tracked in the pool -// and replace the metadata accordingly. It should only be used in the fork boundary -// bulk conversion. If it fails for some reason, the subsequent txs won't be dropped -// for simplicity which we assume it's very likely to happen. -// -// The returned flag indicates whether the replacement succeeded. -func (p *BlobPool) compareAndSwap(address common.Address, hash common.Hash, blob []byte, oldID uint64, oldStorageSize uint32) bool { - p.lock.Lock() - defer p.lock.Unlock() - - newId, err := p.store.Put(blob) - if err != nil { - log.Error("Failed to store transaction", "hash", hash, "err", err) - return false - } - newSize := uint64(len(blob)) - newStorageSize := p.store.Size(newId) - - // Terminate the procedure if the transaction was already evicted. The - // newly added blob should be removed before return. - if !p.lookup.update(hash, newId, newSize) { - if derr := p.store.Delete(newId); derr != nil { - log.Error("Failed to delete the dangling blob tx", "err", derr) - } else { - log.Warn("Deleted the dangling blob tx", "id", newId) - } - return false } - // Update the metadata of blob transaction - for _, meta := range p.index[address] { - if meta.hash == hash { - meta.id = newId - meta.version = types.BlobSidecarVersion1 - meta.storageSize = newStorageSize - meta.size = newSize - - p.stored += uint64(newStorageSize) - p.stored -= uint64(oldStorageSize) - break - } + if len(announcable) > 0 { + p.discoverFeed.Send(core.NewTxsEvent{Txs: announcable}) } - if err := p.store.Delete(oldID); err != nil { - log.Error("Failed to delete the legacy transaction", "hash", hash, "id", oldID, "err", err) - } - return true -} -// convertLegacySidecar fetches transaction data from the store, performs an -// on-the-fly conversion. This function is intended for use only during the -// Osaka fork transition period. -// -// The returned flag indicates whether the replacement succeeds or not. -func (p *BlobPool) convertLegacySidecar(sender common.Address, hash common.Hash, id uint64) bool { - start := time.Now() - - // Retrieves the legacy blob transaction from the underlying store with - // read lock held, preventing any potential data race around the slot - // specified by the id. - p.lock.RLock() - data, err := p.store.Get(id) - if err != nil { - p.lock.RUnlock() - // The transaction may have been evicted simultaneously, safe to skip conversion. - log.Debug("Blob transaction is missing", "hash", hash, "id", id, "err", err) - return false - } - oldStorageSize := p.store.Size(id) - p.lock.RUnlock() - - // Decode the transaction, the failure is not expected and report the error - // loudly if possible. If the blob transaction in this slot is corrupted, - // leave it in the store, it will be dropped during the next pool - // initialization. - var tx types.Transaction - if err = rlp.DecodeBytes(data, &tx); err != nil { - log.Error("Blob transaction is corrupted", "hash", hash, "id", id, "err", err) - return false - } - - // Skip conversion if the transaction does not match the expected hash, or if it was - // already converted. This can occur if the original transaction was evicted from the - // pool and the slot was reused by a new one. - if tx.Hash() != hash { - log.Warn("Blob transaction was replaced", "hash", hash, "id", id, "stored", tx.Hash()) - return false - } - sc := tx.BlobTxSidecar() - if sc.Version >= types.BlobSidecarVersion1 { - log.Debug("Skipping conversion of blob tx", "hash", hash, "id", id) - return false - } - - // Perform the sidecar conversion, the failure is not expected and report the error - // loudly if possible. - if err := tx.BlobTxSidecar().ToV1(); err != nil { - log.Error("Failed to convert blob transaction", "hash", hash, "err", err) - return false - } - - // Encode the converted transaction, the failure is not expected and report - // the error loudly if possible. - blob, err := rlp.EncodeToBytes(&tx) - if err != nil { - log.Error("Failed to encode blob transaction", "hash", tx.Hash(), "err", err) - return false - } - - // Replace the legacy blob transaction with the converted format. - if !p.compareAndSwap(sender, hash, blob, id, oldStorageSize) { - log.Error("Failed to replace the legacy transaction", "hash", hash) - return false - } - log.Debug("Converted legacy blob transaction", "hash", hash, "elapsed", common.PrettyDuration(time.Since(start))) - return true -} - -// convertLegacySidecars converts all given transactions to sidecar version 1. -// -// If any of them fails to be converted, the subsequent transactions will still -// be processed, as we assume the failure is very unlikely to happen. If happens, -// these transactions will be stuck in the pool until eviction. -func (p *BlobPool) convertLegacySidecars(ids map[common.Address]map[uint64]uint64, txs map[common.Address]map[uint64]common.Hash) { - var ( - start = time.Now() - success int - failure int - ) - for addr, list := range txs { - // Transactions evicted from the pool must be contiguous, if in any case, - // the transactions are gapped with each other, they will be discarded. - nonces := slices.Collect(maps.Keys(list)) - slices.Sort(nonces) - - // Convert the txs with nonce order - for _, nonce := range nonces { - if p.convertLegacySidecar(addr, list[nonce], ids[addr][nonce]) { - success++ - } else { - failure++ - } - } - } - log.Info("Completed blob transaction conversion", "discarded", failure, "injected", success, "elapsed", common.PrettyDuration(time.Since(start))) + // Update the basefee and blobfee metrics + basefeeGauge.Update(int64(basefee.Uint64())) + blobfeeGauge.Update(int64(blobfee.Uint64())) + p.updateStorageMetrics() } // reorg assembles all the transactors and missing transactions between an old @@ -1371,7 +1289,9 @@ func (p *BlobPool) validateTx(tx *types.Transaction) error { State: p.state, FirstNonceGap: func(addr common.Address) uint64 { - // Nonce gaps are not permitted in the blob pool, the first gap will + // Nonce gaps are permitted in the blob pool, but only as part of the + // in-memory 'gapped' buffer. We expose the gap here to validateTx, + // then handle the error by adding to the buffer. The first gap will // be the next nonce shifted by however many transactions we already // have pooled. return p.state.GetNonce(addr) + uint64(len(p.index[addr])) @@ -1450,7 +1370,9 @@ func (p *BlobPool) Has(hash common.Hash) bool { p.lock.RLock() defer p.lock.RUnlock() - return p.lookup.exists(hash) + poolHas := p.lookup.exists(hash) + _, gapped := p.gappedSource[hash] + return poolHas || gapped } func (p *BlobPool) getRLP(hash common.Hash) []byte { @@ -1532,8 +1454,8 @@ func (p *BlobPool) GetMetadata(hash common.Hash) *txpool.TxMetadata { // // The version argument specifies the type of proofs to return, either the // blob proofs (version 0) or the cell proofs (version 1). Proofs conversion is -// CPU intensive, so only done if explicitly requested with the convert flag. -func (p *BlobPool) GetBlobs(vhashes []common.Hash, version byte, convert bool) ([]*kzg4844.Blob, []kzg4844.Commitment, [][]kzg4844.Proof, error) { +// CPU intensive and prohibited in the blobpool explicitly. +func (p *BlobPool) GetBlobs(vhashes []common.Hash, version byte) ([]*kzg4844.Blob, []kzg4844.Commitment, [][]kzg4844.Proof, error) { var ( blobs = make([]*kzg4844.Blob, len(vhashes)) commitments = make([]kzg4844.Commitment, len(vhashes)) @@ -1584,7 +1506,7 @@ func (p *BlobPool) GetBlobs(vhashes []common.Hash, version byte, convert bool) ( } // Mark hash as seen. filled[hash] = struct{}{} - if sidecar.Version != version && !convert { + if sidecar.Version != version { // Skip blobs with incompatible version. Note we still track the blob hash // in `filled` here, ensuring that we do not resolve this tx another time. continue @@ -1593,29 +1515,13 @@ func (p *BlobPool) GetBlobs(vhashes []common.Hash, version byte, convert bool) ( var pf []kzg4844.Proof switch version { case types.BlobSidecarVersion0: - if sidecar.Version == types.BlobSidecarVersion0 { - pf = []kzg4844.Proof{sidecar.Proofs[i]} - } else { - proof, err := kzg4844.ComputeBlobProof(&sidecar.Blobs[i], sidecar.Commitments[i]) - if err != nil { - return nil, nil, nil, err - } - pf = []kzg4844.Proof{proof} - } + pf = []kzg4844.Proof{sidecar.Proofs[i]} case types.BlobSidecarVersion1: - if sidecar.Version == types.BlobSidecarVersion0 { - cellProofs, err := kzg4844.ComputeCellProofs(&sidecar.Blobs[i]) - if err != nil { - return nil, nil, nil, err - } - pf = cellProofs - } else { - cellProofs, err := sidecar.CellProofsAt(i) - if err != nil { - return nil, nil, nil, err - } - pf = cellProofs + cellProofs, err := sidecar.CellProofsAt(i) + if err != nil { + return nil, nil, nil, err } + pf = cellProofs } for _, index := range list { blobs[index] = &sidecar.Blobs[i] @@ -1642,65 +1548,15 @@ func (p *BlobPool) AvailableBlobs(vhashes []common.Hash) int { return available } -// preCheck performs the static validation upon the provided tx and converts -// the legacy sidecars if Osaka fork has been activated with a short time window. -// -// This function is pure static and lock free. -func (p *BlobPool) preCheck(tx *types.Transaction) error { - var ( - head = p.head.Load() - isOsaka = p.chain.Config().IsOsaka(head.Number, head.Time, types.DeserializeHeaderExtraInformation(head).ArbOSFormatVersion) - deadline time.Time - ) - if isOsaka { - deadline = time.Unix(int64(*p.chain.Config().OsakaTime), 0).Add(conversionTimeWindow) - } - // Validate the transaction statically at first to avoid unnecessary - // conversion. This step doesn't require lock protection. - if err := p.ValidateTxBasics(tx); err != nil { - return err - } - // Before the Osaka fork, reject the blob txs with cell proofs - if !isOsaka { - if tx.BlobTxSidecar().Version == types.BlobSidecarVersion0 { - return nil - } else { - return errors.New("cell proof is not supported yet") - } - } - // After the Osaka fork, reject the legacy blob txs if the conversion - // time window is passed. - if tx.BlobTxSidecar().Version == types.BlobSidecarVersion1 { - return nil - } - if head.Time > uint64(deadline.Unix()) { - return errors.New("legacy blob tx is not supported") - } - // Convert the legacy sidecar after Osaka fork. This could be a long - // procedure which takes a few seconds, even minutes if there is a long - // queue. Fortunately it will only block the routine of the source peer - // announcing the tx, without affecting other parts. - return p.cQueue.convert(tx) -} - // Add inserts a set of blob transactions into the pool if they pass validation (both // consensus validity and pool restrictions). func (p *BlobPool) Add(txs []*types.Transaction, sync bool) []error { - var ( - errs []error = make([]error, len(txs)) - adds = make([]*types.Transaction, 0, len(txs)) - ) + errs := make([]error, len(txs)) for i, tx := range txs { - if errs[i] = p.preCheck(tx); errs[i] != nil { + if errs[i] = p.ValidateTxBasics(tx); errs[i] != nil { continue } - if errs[i] = p.add(tx); errs[i] == nil { - adds = append(adds, tx.WithoutBlobTxSidecar()) - } - } - if len(adds) > 0 { - p.discoverFeed.Send(core.NewTxsEvent{Txs: adds}) - p.insertFeed.Send(core.NewTxsEvent{Txs: adds}) + errs[i] = p.add(tx) } return errs } @@ -1720,6 +1576,13 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { addtimeHist.Update(time.Since(start).Nanoseconds()) }(time.Now()) + return p.addLocked(tx, true) +} + +// addLocked inserts a new blob transaction into the pool if it passes validation (both +// consensus validity and pool restrictions). It must be called with the pool lock held. +// Only for internal use. +func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error) { // Ensure the transaction is valid from all perspectives if err := p.validateTx(tx); err != nil { log.Trace("Transaction validation failed", "hash", tx.Hash(), "err", err) @@ -1732,6 +1595,21 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { addStaleMeter.Mark(1) case errors.Is(err, core.ErrNonceTooHigh): addGappedMeter.Mark(1) + // Store the tx in memory, and revalidate later + from, _ := types.Sender(p.signer, tx) + allowance := p.gappedAllowance(from) + if allowance >= 1 && len(p.gapped) < maxGapped { + p.gapped[from] = append(p.gapped[from], tx) + p.gappedSource[tx.Hash()] = from + log.Trace("added tx to gapped blob queue", "allowance", allowance, "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "qlen", len(p.gapped[from])) + return nil + } else { + // if maxGapped is reached, it is better to give time to gapped + // transactions by keeping the old and dropping this one. + // Thus replacing a gapped transaction with another gapped transaction + // is discouraged. + log.Trace("no gapped blob queue allowance", "allowance", allowance, "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "qlen", len(p.gapped[from])) + } case errors.Is(err, core.ErrInsufficientFunds): addOverdraftedMeter.Mark(1) case errors.Is(err, txpool.ErrAccountLimitExceeded): @@ -1869,6 +1747,64 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { p.updateStorageMetrics() addValidMeter.Mark(1) + + // Transaction was addded successfully, but we only announce if it is (close to being) + // includable and the previous one was already announced. + if p.isAnnouncable(meta) && (meta.nonce == next || (len(txs) > 1 && txs[offset-1].announced)) { + meta.announced = true + p.discoverFeed.Send(core.NewTxsEvent{Txs: []*types.Transaction{tx.WithoutBlobTxSidecar()}}) + p.insertFeed.Send(core.NewTxsEvent{Txs: []*types.Transaction{tx.WithoutBlobTxSidecar()}}) + } else { + log.Trace("Blob transaction not announcable yet", "hash", tx.Hash(), "nonce", tx.Nonce()) + } + + //check the gapped queue for this account and try to promote + if gtxs, ok := p.gapped[from]; checkGapped && ok && len(gtxs) > 0 { + // We have to add in nonce order, but we want to stable sort to cater for situations + // where transactions are replaced, keeping the original receive order for same nonce + sort.SliceStable(gtxs, func(i, j int) bool { + return gtxs[i].Nonce() < gtxs[j].Nonce() + }) + for len(gtxs) > 0 { + stateNonce := p.state.GetNonce(from) + firstgap := stateNonce + uint64(len(p.index[from])) + + if gtxs[0].Nonce() > firstgap { + // Anything beyond the first gap is not addable yet + break + } + + // Drop any buffered transactions that became stale in the meantime (included in chain or replaced) + // If we arrive to the transaction in the pending range (between the state Nonce and first gap, we + // try to add them now while removing from here. + tx := gtxs[0] + gtxs[0] = nil + gtxs = gtxs[1:] + delete(p.gappedSource, tx.Hash()) + + if tx.Nonce() < stateNonce { + // Stale, drop it. Eventually we could add to limbo here if hash matches. + log.Trace("Gapped blob transaction became stale", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "state", stateNonce, "qlen", len(p.gapped[from])) + continue + } + + if tx.Nonce() <= firstgap { + // If we hit the pending range, including the first gap, add it and continue to try to add more. + // We do not recurse here, but continue to loop instead. + // We are under lock, so we can add the transaction directly. + if err := p.addLocked(tx, false); err == nil { + log.Trace("Gapped blob transaction added to pool", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "qlen", len(p.gapped[from])) + } else { + log.Trace("Gapped blob transaction not accepted", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "err", err) + } + } + } + if len(gtxs) == 0 { + delete(p.gapped, from) + } else { + p.gapped[from] = gtxs + } + } return nil } @@ -1931,11 +1867,11 @@ func (p *BlobPool) drop() { // // The transactions can also be pre-filtered by the dynamic fee components to // reduce allocations and load on downstream subsystems. -func (p *BlobPool) Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction { +func (p *BlobPool) Pending(filter txpool.PendingFilter) (map[common.Address][]*txpool.LazyTransaction, int) { // If only plain transactions are requested, this pool is unsuitable as it // contains none, don't even bother. if !filter.BlobTxs { - return nil + return nil, 0 } // Track the amount of time waiting to retrieve the list of pending blob txs // from the pool and the amount of time actually spent on assembling the data. @@ -1951,6 +1887,7 @@ func (p *BlobPool) Pending(filter txpool.PendingFilter) map[common.Address][]*tx pendtimeHist.Update(time.Since(execStart).Nanoseconds()) }() + var count int pending := make(map[common.Address][]*txpool.LazyTransaction, len(p.index)) for addr, txs := range p.index { lazies := make([]*txpool.LazyTransaction, 0, len(txs)) @@ -1996,9 +1933,10 @@ func (p *BlobPool) Pending(filter txpool.PendingFilter) map[common.Address][]*tx } if len(lazies) > 0 { pending[addr] = lazies + count += len(lazies) } } - return pending + return pending, count } // updateStorageMetrics retrieves a bunch of stats from the data store and pushes @@ -2100,6 +2038,56 @@ func (p *BlobPool) Nonce(addr common.Address) uint64 { return p.state.GetNonce(addr) } +// gappedAllowance returns the number of gapped transactions still +// allowed for the given account. Allowance is based on a slow-start +// logic, allowing more gaps (resource usage) to accounts with a +// higher nonce. Can also return negative values. +func (p *BlobPool) gappedAllowance(addr common.Address) int { + // Gaps happen, but we don't want to allow too many. + // Use log10(nonce+1) as the allowance, with a minimum of 0. + nonce := p.state.GetNonce(addr) + allowance := int(math.Log10(float64(nonce + 1))) + // Cap the allowance to the remaining pool space + return min(allowance, maxTxsPerAccount-len(p.index[addr])) - len(p.gapped[addr]) +} + +// evictGapped removes the old transactions from the gapped reorder buffer. +// Concurrency: The caller must hold the pool lock before calling this function. +func (p *BlobPool) evictGapped() { + cutoff := time.Now().Add(-gappedLifetime) + for from, txs := range p.gapped { + nonce := p.state.GetNonce(from) + // Reuse the original slice to avoid extra allocations. + // This is safe because we only keep references to the original gappedTx objects, + // and we overwrite the slice for this account after filtering. + keep := txs[:0] + for i, gtx := range txs { + if gtx.Time().Before(cutoff) || gtx.Nonce() < nonce { + // Evict old or stale transactions + // Should we add stale to limbo here if it would belong? + delete(p.gappedSource, gtx.Hash()) + txs[i] = nil // Explicitly nil out evicted element + } else { + keep = append(keep, gtx) + } + } + if len(keep) < len(txs) { + log.Trace("Evicting old gapped blob transactions", "count", len(txs)-len(keep), "from", from) + } + if len(keep) == 0 { + delete(p.gapped, from) + } else { + p.gapped[from] = keep + } + } +} + +// isAnnouncable checks whether a transaction is announcable based on its +// fee parameters and announceThreshold. +func (p *BlobPool) isAnnouncable(meta *blobTxMeta) bool { + return evictionPriority(p.evict.basefeeJumps, meta.basefeeJumps, p.evict.blobfeeJumps, meta.blobfeeJumps) >= announceThreshold +} + // Stats retrieves the current pool stats, namely the number of pending and the // number of queued (non-executable) transactions. func (p *BlobPool) Stats() (int, int) { @@ -2134,9 +2122,15 @@ func (p *BlobPool) ContentFrom(addr common.Address) ([]*types.Transaction, []*ty // Status returns the known status (unknown/pending/queued) of a transaction // identified by their hashes. func (p *BlobPool) Status(hash common.Hash) txpool.TxStatus { - if p.Has(hash) { + p.lock.RLock() + defer p.lock.RUnlock() + + if p.lookup.exists(hash) { return txpool.TxStatusPending } + if _, gapped := p.gappedSource[hash]; gapped { + return txpool.TxStatusQueued + } return txpool.TxStatusUnknown } @@ -2186,6 +2180,11 @@ func (p *BlobPool) Clear() { p.index = make(map[common.Address][]*blobTxMeta) p.spent = make(map[common.Address]*uint256.Int) + // Reset counters and the gapped buffer + p.stored = 0 + p.gapped = make(map[common.Address][]*types.Transaction) + p.gappedSource = make(map[common.Hash]common.Address) + var ( basefee = uint256.MustFromBig(eip1559.CalcBaseFee(p.chain.Config(), p.head.Load())) blobfee = uint256.NewInt(params.BlobTxMinBlobGasprice) diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index f7d8ca209b..ba96bea8ed 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -92,10 +92,6 @@ type testBlockChain struct { blockTime *uint64 } -func (bc *testBlockChain) setHeadTime(time uint64) { - bc.blockTime = &time -} - func (bc *testBlockChain) Config() *params.ChainConfig { return bc.config } @@ -433,11 +429,11 @@ func verifyBlobRetrievals(t *testing.T, pool *BlobPool) { hashes = append(hashes, tx.vhashes...) } } - blobs1, _, proofs1, err := pool.GetBlobs(hashes, types.BlobSidecarVersion0, false) + blobs1, _, proofs1, err := pool.GetBlobs(hashes, types.BlobSidecarVersion0) if err != nil { t.Fatal(err) } - blobs2, _, proofs2, err := pool.GetBlobs(hashes, types.BlobSidecarVersion1, false) + blobs2, _, proofs2, err := pool.GetBlobs(hashes, types.BlobSidecarVersion1) if err != nil { t.Fatal(err) } @@ -834,8 +830,8 @@ func TestOpenIndex(t *testing.T) { //blobfeeJumps = []float64{34.023, 35.570, 36.879, 29.686, 26.243, 20.358} // log 1.125 (blob fee cap) evictExecTipCaps = []uint64{10, 10, 5, 5, 1, 1} - evictExecFeeJumps = []float64{39.098, 38.204, 38.204, 19.549, 19.549, 19.549} // min(log 1.125 (exec fee cap)) - evictBlobFeeJumps = []float64{34.023, 34.023, 34.023, 29.686, 26.243, 20.358} // min(log 1.125 (blob fee cap)) + evictExecFeeJumps = []float64{39.098, 38.204, 38.204, 19.549, 19.549, 19.549} // min(log 1.125 (exec fee cap)) + evictBlobFeeJumps = []float64{25.517256, 25.517256, 25.517256, 22.264502, 19.682646, 15.268934} // min(log 1.17 (blob fee cap)) totalSpent = uint256.NewInt(21000*(100+90+200+10+80+300) + blobSize*(55+66+77+33+22+11) + 100*6) // 21000 gas x price + 128KB x blobprice + value ) @@ -1329,7 +1325,7 @@ func TestBlobCountLimit(t *testing.T) { // Check that first succeeds second fails. if errs[0] != nil { - t.Fatalf("expected tx with 7 blobs to succeed") + t.Fatalf("expected tx with 7 blobs to succeed, got %v", errs[0]) } if !errors.Is(errs[1], txpool.ErrTxBlobLimitExceeded) { t.Fatalf("expected tx with 8 blobs to fail, got: %v", errs[1]) @@ -1356,9 +1352,10 @@ func TestAdd(t *testing.T) { } // addtx is a helper sender/tx tuple to represent a new tx addition type addtx struct { - from string - tx *types.BlobTx - err error + from string + tx *types.BlobTx + err error + check func(*BlobPool, *types.Transaction) bool } tests := []struct { @@ -1375,6 +1372,7 @@ func TestAdd(t *testing.T) { "bob": {balance: 21100 + blobSize, nonce: 1}, "claire": {balance: 21100 + blobSize}, "dave": {balance: 21100 + blobSize, nonce: 1}, + "eve": {balance: 21100 + blobSize, nonce: 10}, // High nonce to test gapped acceptance }, adds: []addtx{ { // New account, no previous txs: accept nonce 0 @@ -1402,6 +1400,14 @@ func TestAdd(t *testing.T) { tx: makeUnsignedTx(2, 1, 1, 1), err: core.ErrNonceTooHigh, }, + { // Old account, 10 txs in chain: 0 pending: accept nonce 11 as gapped + from: "eve", + tx: makeUnsignedTx(11, 1, 1, 1), + err: nil, + check: func(pool *BlobPool, tx *types.Transaction) bool { + return pool.Status(tx.Hash()) == txpool.TxStatusQueued + }, + }, }, }, // Transactions from already pooled accounts should only be accepted if @@ -1745,8 +1751,8 @@ func TestAdd(t *testing.T) { // Create a blob pool out of the pre-seeded dats chain := &testBlockChain{ config: params.MainnetChainConfig, - basefee: uint256.NewInt(1050), - blobfee: uint256.NewInt(105), + basefee: uint256.NewInt(1), + blobfee: uint256.NewInt(1), statedb: statedb, } pool := New(Config{Datadir: storage}, chain, nil) @@ -1762,15 +1768,28 @@ func TestAdd(t *testing.T) { t.Errorf("test %d, tx %d: adding transaction error mismatch: have %v, want %v", i, j, errs[0], add.err) } if add.err == nil { - size, exist := pool.lookup.sizeOfTx(signed.Hash()) - if !exist { - t.Errorf("test %d, tx %d: failed to lookup transaction's size", i, j) + // first check if tx is in the pool (reorder queue or pending) + if !pool.Has(signed.Hash()) { + t.Errorf("test %d, tx %d: added transaction not found in pool", i, j) } - if size != signed.Size() { - t.Errorf("test %d, tx %d: transaction's size mismatches: have %v, want %v", - i, j, size, signed.Size()) + // if it is pending, check if size matches + if pool.Status(signed.Hash()) == txpool.TxStatusPending { + size, exist := pool.lookup.sizeOfTx(signed.Hash()) + if !exist { + t.Errorf("test %d, tx %d: failed to lookup transaction's size", i, j) + } + if size != signed.Size() { + t.Errorf("test %d, tx %d: transaction's size mismatches: have %v, want %v", + i, j, size, signed.Size()) + } } } + if add.check != nil { + if !add.check(pool, signed) { + t.Errorf("test %d, tx %d: custom check failed", i, j) + } + } + // Verify the pool internals after each addition verifyPoolInternals(t, pool) } verifyPoolInternals(t, pool) @@ -1806,66 +1825,6 @@ func TestAdd(t *testing.T) { } } -// Tests that transactions with legacy sidecars are accepted within the -// conversion window but rejected after it has passed. -func TestAddLegacyBlobTx(t *testing.T) { - testAddLegacyBlobTx(t, true) // conversion window has not yet passed - testAddLegacyBlobTx(t, false) // conversion window passed -} - -func testAddLegacyBlobTx(t *testing.T, accept bool) { - var ( - key1, _ = crypto.GenerateKey() - key2, _ = crypto.GenerateKey() - - addr1 = crypto.PubkeyToAddress(key1.PublicKey) - addr2 = crypto.PubkeyToAddress(key2.PublicKey) - ) - - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) - statedb.AddBalance(addr1, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) - statedb.AddBalance(addr2, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) - statedb.Commit(0, true, false) - - chain := &testBlockChain{ - config: params.MergedTestChainConfig, - basefee: uint256.NewInt(1050), - blobfee: uint256.NewInt(105), - statedb: statedb, - } - var timeDiff uint64 - if accept { - timeDiff = uint64(conversionTimeWindow.Seconds()) - 1 - } else { - timeDiff = uint64(conversionTimeWindow.Seconds()) + 1 - } - time := *params.MergedTestChainConfig.OsakaTime + timeDiff - chain.setHeadTime(time) - - pool := New(Config{Datadir: t.TempDir()}, chain, nil) - if err := pool.Init(1, chain.CurrentBlock(), newReserver()); err != nil { - t.Fatalf("failed to create blob pool: %v", err) - } - - // Attempt to add legacy blob transactions. - var ( - tx1 = makeMultiBlobTx(0, 1, 1000, 100, 6, 0, key1, types.BlobSidecarVersion0) - tx2 = makeMultiBlobTx(0, 1, 800, 70, 6, 6, key2, types.BlobSidecarVersion0) - txs = []*types.Transaction{tx1, tx2} - ) - errs := pool.Add(txs, true) - for _, err := range errs { - if accept && err != nil { - t.Fatalf("expected tx add to succeed, %v", err) - } - if !accept && err == nil { - t.Fatal("expected tx add to fail") - } - } - verifyPoolInternals(t, pool) - pool.Close() -} - func TestGetBlobs(t *testing.T) { //log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, true))) @@ -1952,7 +1911,6 @@ func TestGetBlobs(t *testing.T) { limit int fillRandom bool // Whether to randomly fill some of the requested blobs with unknowns version byte // Blob sidecar version to request - convert bool // Whether to convert version on retrieval }{ { start: 0, limit: 6, @@ -2018,11 +1976,6 @@ func TestGetBlobs(t *testing.T) { start: 0, limit: 18, fillRandom: true, version: types.BlobSidecarVersion1, }, - { - start: 0, limit: 18, fillRandom: true, - version: types.BlobSidecarVersion1, - convert: true, // Convert some version 0 blobs to version 1 while retrieving - }, } for i, c := range cases { var ( @@ -2044,7 +1997,7 @@ func TestGetBlobs(t *testing.T) { filled[len(vhashes)] = struct{}{} vhashes = append(vhashes, testrand.Hash()) } - blobs, _, proofs, err := pool.GetBlobs(vhashes, c.version, c.convert) + blobs, _, proofs, err := pool.GetBlobs(vhashes, c.version) if err != nil { t.Errorf("Unexpected error for case %d, %v", i, err) } @@ -2070,8 +2023,7 @@ func TestGetBlobs(t *testing.T) { // If an item is missing, but shouldn't, error if blobs[j] == nil || proofs[j] == nil { // This is only an error if there was no version mismatch - if c.convert || - (c.version == types.BlobSidecarVersion1 && 6 <= testBlobIndex && testBlobIndex < 12) || + if (c.version == types.BlobSidecarVersion1 && 6 <= testBlobIndex && testBlobIndex < 12) || (c.version == types.BlobSidecarVersion0 && (testBlobIndex < 6 || 12 <= testBlobIndex)) { t.Errorf("tracked blob retrieval failed: item %d, hash %x", j, vhashes[j]) } @@ -2098,185 +2050,6 @@ func TestGetBlobs(t *testing.T) { pool.Close() } -// TestSidecarConversion will verify that after the Osaka fork, all legacy -// sidecars in the pool are successfully convert to v1 sidecars. -func TestSidecarConversion(t *testing.T) { - // log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, true))) - - // Create a temporary folder for the persistent backend - storage := t.TempDir() - os.MkdirAll(filepath.Join(storage, pendingTransactionStore), 0700) - - var ( - preOsakaTxs = make(types.Transactions, 10) - postOsakaTxs = make(types.Transactions, 3) - keys = make([]*ecdsa.PrivateKey, len(preOsakaTxs)+len(postOsakaTxs)) - addrs = make([]common.Address, len(preOsakaTxs)+len(postOsakaTxs)) - statedb, _ = state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) - ) - for i := range keys { - keys[i], _ = crypto.GenerateKey() - addrs[i] = crypto.PubkeyToAddress(keys[i].PublicKey) - statedb.AddBalance(addrs[i], uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) - } - for i := range preOsakaTxs { - preOsakaTxs[i] = makeMultiBlobTx(0, 1, 1000, 100, 2, 0, keys[i], types.BlobSidecarVersion0) - } - for i := range postOsakaTxs { - if i == 0 { - // First has a v0 sidecar. - postOsakaTxs[i] = makeMultiBlobTx(0, 1, 1000, 100, 1, 0, keys[len(preOsakaTxs)+i], types.BlobSidecarVersion0) - } - postOsakaTxs[i] = makeMultiBlobTx(0, 1, 1000, 100, 1, 0, keys[len(preOsakaTxs)+i], types.BlobSidecarVersion1) - } - statedb.Commit(0, true, false) - - // Test plan: - // 1) Create a bunch v0 sidecar txs and add to pool before Osaka. - // 2) Pass in new Osaka header to activate the conversion thread. - // 3) Continue adding both v0 and v1 transactions to the pool. - // 4) Verify that as additional blocks come in, transactions involved in the - // migration are correctly discarded. - - config := ¶ms.ChainConfig{ - ChainID: big.NewInt(1), - LondonBlock: big.NewInt(0), - BerlinBlock: big.NewInt(0), - CancunTime: newUint64(0), - PragueTime: newUint64(0), - OsakaTime: newUint64(1), - BlobScheduleConfig: params.DefaultBlobSchedule, - } - chain := &testBlockChain{ - config: config, - basefee: uint256.NewInt(1050), - blobfee: uint256.NewInt(105), - statedb: statedb, - blocks: make(map[uint64]*types.Block), - } - - // Create 3 blocks: - // - the current block, before Osaka - // - the first block after Osaka - // - another post-Osaka block with several transactions in it - header0 := chain.CurrentBlock() - header0.Time = 0 - chain.blocks[0] = types.NewBlockWithHeader(header0) - - header1 := chain.CurrentBlock() - header1.Number = big.NewInt(1) - header1.Time = 1 - chain.blocks[1] = types.NewBlockWithHeader(header1) - - header2 := chain.CurrentBlock() - header2.Time = 2 - header2.Number = big.NewInt(2) - - // Make a copy of one of the pre-Osaka transactions and convert it to v1 here - // so that we can add it to the pool later and ensure a duplicate is not added - // by the conversion queue. - tx := preOsakaTxs[len(preOsakaTxs)-1] - sc := *tx.BlobTxSidecar() // copy sidecar - sc.ToV1() - tx.WithBlobTxSidecar(&sc) - - block2 := types.NewBlockWithHeader(header2).WithBody(types.Body{Transactions: append(postOsakaTxs, tx)}) - chain.blocks[2] = block2 - - pool := New(Config{Datadir: storage}, chain, nil) - if err := pool.Init(1, header0, newReserver()); err != nil { - t.Fatalf("failed to create blob pool: %v", err) - } - - errs := pool.Add(preOsakaTxs, true) - for i, err := range errs { - if err != nil { - t.Errorf("failed to insert blob tx from %s: %s", addrs[i], errs[i]) - } - } - - // Kick off migration. - pool.Reset(header0, header1) - - // Add the v0 sidecar tx, but don't block so we can keep doing other stuff - // while it converts the sidecar. - addDone := make(chan struct{}) - go func() { - pool.Add(types.Transactions{postOsakaTxs[0]}, false) - close(addDone) - }() - - // Add the post-Osaka v1 sidecar txs. - errs = pool.Add(postOsakaTxs[1:], false) - for _, err := range errs { - if err != nil { - t.Fatalf("expected tx add to succeed: %v", err) - } - } - - // Wait for the first tx's conversion to complete, then check that all - // transactions added after Osaka can be accounted for in the pool. - <-addDone - pending := pool.Pending(txpool.PendingFilter{BlobTxs: true, BlobVersion: types.BlobSidecarVersion1}) - for _, tx := range postOsakaTxs { - from, _ := pool.signer.Sender(tx) - if len(pending[from]) != 1 || pending[from][0].Hash != tx.Hash() { - t.Fatalf("expected post-Osaka txs to be pending") - } - } - - // Now update the pool with the next block. This should cause the pool to - // clear out the post-Osaka txs since they were included in block 2. Since the - // test blockchain doesn't manage nonces, we'll just do that manually before - // the reset is called. Don't forget about the pre-Osaka transaction we also - // added to block 2! - for i := range postOsakaTxs { - statedb.SetNonce(addrs[len(preOsakaTxs)+i], 1, tracing.NonceChangeEoACall) - } - statedb.SetNonce(addrs[len(preOsakaTxs)-1], 1, tracing.NonceChangeEoACall) - pool.Reset(header1, block2.Header()) - - // Now verify no post-Osaka transactions are tracked by the pool. - for i, tx := range postOsakaTxs { - if pool.Get(tx.Hash()) != nil { - t.Fatalf("expected txs added post-osaka to have been placed in limbo due to inclusion in a block: index %d, hash %s", i, tx.Hash()) - } - } - - // Wait for the pool migration to complete. - <-pool.cQueue.anyBillyConversionDone - - // Verify all transactions in the pool were converted and verify the - // subsequent cell proofs. - count, _ := pool.Stats() - if count != len(preOsakaTxs)-1 { - t.Errorf("expected pending count to match initial tx count: pending=%d, expected=%d", count, len(preOsakaTxs)-1) - } - for addr, acc := range pool.index { - for _, m := range acc { - if m.version != types.BlobSidecarVersion1 { - t.Errorf("expected sidecar to have been converted: from %s, hash %s", addr, m.hash) - } - tx := pool.Get(m.hash) - if tx == nil { - t.Errorf("failed to get tx by hash: %s", m.hash) - } - sc := tx.BlobTxSidecar() - if err := kzg4844.VerifyCellProofs(sc.Blobs, sc.Commitments, sc.Proofs); err != nil { - t.Errorf("failed to verify cell proofs for tx %s after conversion: %s", m.hash, err) - } - } - } - - verifyPoolInternals(t, pool) - - // Launch conversion a second time. - // This is just a sanity check to ensure we can handle it. - pool.Reset(header0, header1) - - pool.Close() -} - // fakeBilly is a billy.Database implementation which just drops data on the floor. type fakeBilly struct { billy.Database @@ -2349,15 +2122,14 @@ func benchmarkPoolPending(b *testing.B, datacap uint64) { b.ReportAllocs() for i := 0; i < b.N; i++ { - p := pool.Pending(txpool.PendingFilter{ + p, _ := pool.Pending(txpool.PendingFilter{ MinTip: uint256.NewInt(1), BaseFee: chain.basefee, BlobFee: chain.blobfee, + BlobTxs: true, }) if len(p) != int(capacity) { b.Fatalf("have %d want %d", len(p), capacity) } } } - -func newUint64(val uint64) *uint64 { return &val } diff --git a/core/txpool/blobpool/conversion.go b/core/txpool/blobpool/conversion.go deleted file mode 100644 index 95828d83b2..0000000000 --- a/core/txpool/blobpool/conversion.go +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright 2025 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package blobpool - -import ( - "errors" - "slices" - "sync/atomic" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/log" -) - -// maxPendingConversionTasks caps the number of pending conversion tasks. This -// prevents excessive memory usage; the worst-case scenario (2k transactions -// with 6 blobs each) would consume approximately 1.5GB of memory. -const maxPendingConversionTasks = 2048 - -// txConvert represents a conversion task with an attached legacy blob transaction. -type txConvert struct { - tx *types.Transaction // Legacy blob transaction - done chan error // Channel for signaling back if the conversion succeeds -} - -// conversionQueue is a dedicated queue for converting legacy blob transactions -// received from the network after the Osaka fork. Since conversion is expensive, -// it is performed in the background by a single thread, ensuring the main Geth -// process is not overloaded. -type conversionQueue struct { - tasks chan *txConvert - startBilly chan func() - quit chan struct{} - closed chan struct{} - - billyQueue []func() - billyTaskDone chan struct{} - - // This channel will be closed when the first billy conversion finishes. - // It's added for unit tests to synchronize with the conversion progress. - anyBillyConversionDone chan struct{} -} - -// newConversionQueue constructs the conversion queue. -func newConversionQueue() *conversionQueue { - q := &conversionQueue{ - tasks: make(chan *txConvert), - startBilly: make(chan func()), - quit: make(chan struct{}), - closed: make(chan struct{}), - anyBillyConversionDone: make(chan struct{}), - } - go q.loop() - return q -} - -// convert accepts a legacy blob transaction with version-0 blobs and queues it -// for conversion. -// -// This function may block for a long time until the transaction is processed. -func (q *conversionQueue) convert(tx *types.Transaction) error { - done := make(chan error, 1) - select { - case q.tasks <- &txConvert{tx: tx, done: done}: - return <-done - case <-q.closed: - return errors.New("conversion queue closed") - } -} - -// launchBillyConversion starts a conversion task in the background. -func (q *conversionQueue) launchBillyConversion(fn func()) error { - select { - case q.startBilly <- fn: - return nil - case <-q.closed: - return errors.New("conversion queue closed") - } -} - -// close terminates the conversion queue. -func (q *conversionQueue) close() { - select { - case <-q.closed: - return - default: - close(q.quit) - <-q.closed - } -} - -// run converts a batch of legacy blob txs to the new cell proof format. -func (q *conversionQueue) run(tasks []*txConvert, done chan struct{}, interrupt *atomic.Int32) { - defer close(done) - - for _, t := range tasks { - if interrupt != nil && interrupt.Load() != 0 { - t.done <- errors.New("conversion is interrupted") - continue - } - sidecar := t.tx.BlobTxSidecar() - if sidecar == nil { - t.done <- errors.New("tx without sidecar") - continue - } - // Run the conversion, the original sidecar will be mutated in place - start := time.Now() - err := sidecar.ToV1() - t.done <- err - log.Trace("Converted legacy blob tx", "hash", t.tx.Hash(), "err", err, "elapsed", common.PrettyDuration(time.Since(start))) - } -} - -func (q *conversionQueue) loop() { - defer close(q.closed) - - var ( - done chan struct{} // Non-nil if background routine is active - interrupt *atomic.Int32 // Flag to signal conversion interruption - - // The pending tasks for sidecar conversion. We assume the number of legacy - // blob transactions requiring conversion will not be excessive. However, - // a hard cap is applied as a protective measure. - txTasks []*txConvert - - firstBilly = true - ) - - for { - select { - case t := <-q.tasks: - if len(txTasks) >= maxPendingConversionTasks { - t.done <- errors.New("conversion queue is overloaded") - continue - } - txTasks = append(txTasks, t) - - // Launch the background conversion thread if it's idle - if done == nil { - done, interrupt = make(chan struct{}), new(atomic.Int32) - - tasks := slices.Clone(txTasks) - txTasks = txTasks[:0] - go q.run(tasks, done, interrupt) - } - - case <-done: - done, interrupt = nil, nil - - case fn := <-q.startBilly: - q.billyQueue = append(q.billyQueue, fn) - q.runNextBillyTask() - - case <-q.billyTaskDone: - if firstBilly { - close(q.anyBillyConversionDone) - firstBilly = false - } - q.runNextBillyTask() - - case <-q.quit: - if done != nil { - log.Debug("Waiting for blob proof conversion to exit") - interrupt.Store(1) - <-done - } - if q.billyTaskDone != nil { - log.Debug("Waiting for blobpool billy conversion to exit") - <-q.billyTaskDone - } - return - } - } -} - -func (q *conversionQueue) runNextBillyTask() { - if len(q.billyQueue) == 0 { - q.billyTaskDone = nil - return - } - - fn := q.billyQueue[0] - q.billyQueue = append(q.billyQueue[:0], q.billyQueue[1:]...) - - done := make(chan struct{}) - go func() { defer close(done); fn() }() - q.billyTaskDone = done -} diff --git a/core/txpool/blobpool/conversion_test.go b/core/txpool/blobpool/conversion_test.go deleted file mode 100644 index a9fd26dbaf..0000000000 --- a/core/txpool/blobpool/conversion_test.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2025 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package blobpool - -import ( - "crypto/ecdsa" - "crypto/sha256" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/crypto/kzg4844" - "github.com/ethereum/go-ethereum/params" - "github.com/holiman/uint256" -) - -// createV1BlobTx creates a blob transaction with version 1 sidecar for testing. -func createV1BlobTx(nonce uint64, key *ecdsa.PrivateKey) *types.Transaction { - blob := &kzg4844.Blob{byte(nonce)} - commitment, _ := kzg4844.BlobToCommitment(blob) - cellProofs, _ := kzg4844.ComputeCellProofs(blob) - - blobtx := &types.BlobTx{ - ChainID: uint256.MustFromBig(params.MainnetChainConfig.ChainID), - Nonce: nonce, - GasTipCap: uint256.NewInt(1), - GasFeeCap: uint256.NewInt(1000), - Gas: 21000, - BlobFeeCap: uint256.NewInt(100), - BlobHashes: []common.Hash{kzg4844.CalcBlobHashV1(sha256.New(), &commitment)}, - Value: uint256.NewInt(100), - Sidecar: types.NewBlobTxSidecar(types.BlobSidecarVersion1, []kzg4844.Blob{*blob}, []kzg4844.Commitment{commitment}, cellProofs), - } - return types.MustSignNewTx(key, types.LatestSigner(params.MainnetChainConfig), blobtx) -} - -func TestConversionQueueBasic(t *testing.T) { - queue := newConversionQueue() - defer queue.close() - - key, _ := crypto.GenerateKey() - tx := makeTx(0, 1, 1, 1, key) - if err := queue.convert(tx); err != nil { - t.Fatalf("Expected successful conversion, got error: %v", err) - } - if tx.BlobTxSidecar().Version != types.BlobSidecarVersion1 { - t.Errorf("Expected sidecar version to be %d, got %d", types.BlobSidecarVersion1, tx.BlobTxSidecar().Version) - } -} - -func TestConversionQueueV1BlobTx(t *testing.T) { - queue := newConversionQueue() - defer queue.close() - - key, _ := crypto.GenerateKey() - tx := createV1BlobTx(0, key) - version := tx.BlobTxSidecar().Version - - err := queue.convert(tx) - if err != nil { - t.Fatalf("Expected successful conversion, got error: %v", err) - } - if tx.BlobTxSidecar().Version != version { - t.Errorf("Expected sidecar version to remain %d, got %d", version, tx.BlobTxSidecar().Version) - } -} - -func TestConversionQueueClosed(t *testing.T) { - queue := newConversionQueue() - - // Close the queue first - queue.close() - key, _ := crypto.GenerateKey() - tx := makeTx(0, 1, 1, 1, key) - - err := queue.convert(tx) - if err == nil { - t.Fatal("Expected error when converting on closed queue, got nil") - } -} - -func TestConversionQueueDoubleClose(t *testing.T) { - queue := newConversionQueue() - queue.close() - queue.close() // Should not panic -} diff --git a/core/txpool/blobpool/evictheap.go b/core/txpool/blobpool/evictheap.go index 722a71bc9b..a46b8e9a6f 100644 --- a/core/txpool/blobpool/evictheap.go +++ b/core/txpool/blobpool/evictheap.go @@ -67,7 +67,7 @@ func newPriceHeap(basefee *uint256.Int, blobfee *uint256.Int, index map[common.A func (h *evictHeap) reinit(basefee *uint256.Int, blobfee *uint256.Int, force bool) { // If the update is mostly the same as the old, don't sort pointlessly basefeeJumps := dynamicFeeJumps(basefee) - blobfeeJumps := dynamicFeeJumps(blobfee) + blobfeeJumps := dynamicBlobFeeJumps(blobfee) if !force && math.Abs(h.basefeeJumps-basefeeJumps) < 0.01 && math.Abs(h.blobfeeJumps-blobfeeJumps) < 0.01 { // TODO(karalabe): 0.01 enough, maybe should be smaller? Maybe this optimization is moot? return @@ -95,13 +95,7 @@ func (h *evictHeap) Less(i, j int) bool { lastJ := txsJ[len(txsJ)-1] prioI := evictionPriority(h.basefeeJumps, lastI.evictionExecFeeJumps, h.blobfeeJumps, lastI.evictionBlobFeeJumps) - if prioI > 0 { - prioI = 0 - } prioJ := evictionPriority(h.basefeeJumps, lastJ.evictionExecFeeJumps, h.blobfeeJumps, lastJ.evictionBlobFeeJumps) - if prioJ > 0 { - prioJ = 0 - } if prioI == prioJ { return lastI.evictionExecTip.Lt(lastJ.evictionExecTip) } diff --git a/core/txpool/blobpool/evictheap_test.go b/core/txpool/blobpool/evictheap_test.go index de4076e298..112fa77a01 100644 --- a/core/txpool/blobpool/evictheap_test.go +++ b/core/txpool/blobpool/evictheap_test.go @@ -109,22 +109,22 @@ func TestPriceHeapSorting(t *testing.T) { order: []int{3, 2, 1, 0, 4, 5, 6}, }, // If both basefee and blobfee is specified, sort by the larger distance - // of the two from the current network conditions, splitting same (loglog) + // of the two from the current network conditions, splitting same // ones via the tip. // - // Basefee: 1000 - // Blobfee: 100 + // Basefee: 1000 , jumps: 888, 790, 702, 624 + // Blobfee: 100 , jumps: 85, 73, 62, 53 // - // Tx #0: (800, 80) - 2 jumps below both => priority -1 - // Tx #1: (630, 63) - 4 jumps below both => priority -2 - // Tx #2: (800, 63) - 2 jumps below basefee, 4 jumps below blobfee => priority -2 (blob penalty dominates) - // Tx #3: (630, 80) - 4 jumps below basefee, 2 jumps below blobfee => priority -2 (base penalty dominates) + // Tx #0: (800, 80) - 2 jumps below both => priority -2 + // Tx #1: (630, 55) - 4 jumps below both => priority -4 + // Tx #2: (800, 55) - 2 jumps below basefee, 4 jumps below blobfee => priority -4 (blob penalty dominates) + // Tx #3: (630, 80) - 4 jumps below basefee, 2 jumps below blobfee => priority -4 (base penalty dominates) // // Txs 1, 2, 3 share the same priority, split via tip, prefer 0 as the best { execTips: []uint64{1, 2, 3, 4}, execFees: []uint64{800, 630, 800, 630}, - blobFees: []uint64{80, 63, 63, 80}, + blobFees: []uint64{80, 55, 55, 80}, basefee: 1000, blobfee: 100, order: []int{1, 2, 3, 0}, @@ -142,7 +142,7 @@ func TestPriceHeapSorting(t *testing.T) { blobFee = uint256.NewInt(tt.blobFees[j]) basefeeJumps = dynamicFeeJumps(execFee) - blobfeeJumps = dynamicFeeJumps(blobFee) + blobfeeJumps = dynamicBlobFeeJumps(blobFee) ) index[addr] = []*blobTxMeta{{ id: uint64(j), @@ -201,7 +201,7 @@ func benchmarkPriceHeapReinit(b *testing.B, datacap uint64) { blobFee = uint256.NewInt(rnd.Uint64()) basefeeJumps = dynamicFeeJumps(execFee) - blobfeeJumps = dynamicFeeJumps(blobFee) + blobfeeJumps = dynamicBlobFeeJumps(blobFee) ) index[addr] = []*blobTxMeta{{ id: uint64(i), @@ -277,7 +277,7 @@ func benchmarkPriceHeapOverflow(b *testing.B, datacap uint64) { blobFee = uint256.NewInt(rnd.Uint64()) basefeeJumps = dynamicFeeJumps(execFee) - blobfeeJumps = dynamicFeeJumps(blobFee) + blobfeeJumps = dynamicBlobFeeJumps(blobFee) ) index[addr] = []*blobTxMeta{{ id: uint64(i), @@ -308,7 +308,7 @@ func benchmarkPriceHeapOverflow(b *testing.B, datacap uint64) { blobFee = uint256.NewInt(rnd.Uint64()) basefeeJumps = dynamicFeeJumps(execFee) - blobfeeJumps = dynamicFeeJumps(blobFee) + blobfeeJumps = dynamicBlobFeeJumps(blobFee) ) metas[i] = &blobTxMeta{ id: uint64(int(blobs) + i), diff --git a/core/txpool/blobpool/limbo.go b/core/txpool/blobpool/limbo.go index 50c40c9d83..36284d6a03 100644 --- a/core/txpool/blobpool/limbo.go +++ b/core/txpool/blobpool/limbo.go @@ -20,7 +20,6 @@ import ( "errors" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" @@ -57,7 +56,7 @@ func newLimbo(config *params.ChainConfig, datadir string) (*limbo, error) { } // Create new slotter for pre-Osaka blob configuration. - slotter := newSlotter(eip4844.LatestMaxBlobsPerBlock(config)) + slotter := newSlotter(params.BlobTxMaxBlobs) // See if we need to migrate the limbo after fusaka. slotter, err := tryMigrate(config, slotter, datadir) diff --git a/core/txpool/blobpool/lookup.go b/core/txpool/blobpool/lookup.go index 874ca85b8c..7607cd487a 100644 --- a/core/txpool/blobpool/lookup.go +++ b/core/txpool/blobpool/lookup.go @@ -110,13 +110,3 @@ func (l *lookup) untrack(tx *blobTxMeta) { } } } - -// update updates the transaction index. It should only be used in the conversion. -func (l *lookup) update(hash common.Hash, id uint64, size uint64) bool { - meta, exists := l.txIndex[hash] - if !exists { - return false - } - meta.id, meta.size = id, size - return true -} diff --git a/core/txpool/blobpool/priority.go b/core/txpool/blobpool/priority.go index 7ae7f92def..d7e8789ce7 100644 --- a/core/txpool/blobpool/priority.go +++ b/core/txpool/blobpool/priority.go @@ -18,7 +18,6 @@ package blobpool import ( "math" - "math/bits" "github.com/holiman/uint256" ) @@ -26,6 +25,13 @@ import ( // log1_125 is used in the eviction priority calculation. var log1_125 = math.Log(1.125) +// log1_17 is used in the eviction priority calculation for blob fees. +// EIP-7892 (BPO) changed the ratio of target to max blobs, and with that +// also the maximum blob fee decrease in a slot from 1.125 to approx 1.17 . +// Since we want priorities to approximate time, we should change our log +// calculation for blob fees. +var log1_17 = log1_125 * 4 / 3 + // evictionPriority calculates the eviction priority based on the algorithm // described in the BlobPool docs for both fee components. // @@ -36,23 +42,20 @@ func evictionPriority(basefeeJumps float64, txBasefeeJumps, blobfeeJumps, txBlob basefeePriority = evictionPriority1D(basefeeJumps, txBasefeeJumps) blobfeePriority = evictionPriority1D(blobfeeJumps, txBlobfeeJumps) ) - if basefeePriority < blobfeePriority { - return basefeePriority - } - return blobfeePriority + return min(0, basefeePriority, blobfeePriority) } // evictionPriority1D calculates the eviction priority based on the algorithm // described in the BlobPool docs for a single fee component. func evictionPriority1D(basefeeJumps float64, txfeeJumps float64) int { jumps := txfeeJumps - basefeeJumps - if int(jumps) == 0 { - return 0 // can't log2 0 + if jumps <= 0 { + return int(math.Floor(jumps)) } - if jumps < 0 { - return -intLog2(uint(-math.Floor(jumps))) - } - return intLog2(uint(math.Ceil(jumps))) + // We only use the negative part for ordering. The positive part is only used + // for threshold comparison (with a negative threshold), so the value is almost + // irrelevant, as long as it's positive. + return int((math.Ceil(jumps))) } // dynamicFeeJumps calculates the log1.125(fee), namely the number of fee jumps @@ -70,21 +73,9 @@ func dynamicFeeJumps(fee *uint256.Int) float64 { return math.Log(fee.Float64()) / log1_125 } -// intLog2 is a helper to calculate the integral part of a log2 of an unsigned -// integer. It is a very specific calculation that's not particularly useful in -// general, but it's what we need here (it's fast). -func intLog2(n uint) int { - switch { - case n == 0: - panic("log2(0) is undefined") - - case n < 2048: - return bits.UintSize - bits.LeadingZeros(n) - 1 - - default: - // The input is log1.125(uint256) = log2(uint256) / log2(1.125). At the - // most extreme, log2(uint256) will be a bit below 257, and the constant - // log2(1.125) ~= 0.17. The larges input thus is ~257 / ~0.17 ~= ~1511. - panic("dynamic fee jump diffs cannot reach this") +func dynamicBlobFeeJumps(fee *uint256.Int) float64 { + if fee.IsZero() { + return 0 // can't log2 zero, should never happen outside tests, but don't choke } + return math.Log(fee.Float64()) / log1_17 } diff --git a/core/txpool/blobpool/priority_test.go b/core/txpool/blobpool/priority_test.go index 1eaee6d7df..47b3a5375f 100644 --- a/core/txpool/blobpool/priority_test.go +++ b/core/txpool/blobpool/priority_test.go @@ -30,12 +30,12 @@ func TestPriorityCalculation(t *testing.T) { txfee uint64 result int }{ - {basefee: 7, txfee: 10, result: 2}, // 3.02 jumps, 4 ceil, 2 log2 - {basefee: 17_200_000_000, txfee: 17_200_000_000, result: 0}, // 0 jumps, special case 0 log2 - {basefee: 9_853_941_692, txfee: 11_085_092_510, result: 0}, // 0.99 jumps, 1 ceil, 0 log2 - {basefee: 11_544_106_391, txfee: 10_356_781_100, result: 0}, // -0.92 jumps, -1 floor, 0 log2 - {basefee: 17_200_000_000, txfee: 7, result: -7}, // -183.57 jumps, -184 floor, -7 log2 - {basefee: 7, txfee: 17_200_000_000, result: 7}, // 183.57 jumps, 184 ceil, 7 log2 + {basefee: 7, txfee: 10, result: 4}, // 3.02 jumps, 4 ceil + {basefee: 17_200_000_000, txfee: 17_200_000_000, result: 0}, // 0 jumps, special case 0 + {basefee: 9_853_941_692, txfee: 11_085_092_510, result: 1}, // 0.99 jumps, 1 ceil + {basefee: 11_544_106_391, txfee: 10_356_781_100, result: -1}, // -0.92 jumps, -1 floor + {basefee: 17_200_000_000, txfee: 7, result: -184}, // -183.57 jumps, -184 floor + {basefee: 7, txfee: 17_200_000_000, result: 184}, // 183.57 jumps, 184 ceil } for i, tt := range tests { var ( @@ -69,7 +69,7 @@ func BenchmarkPriorityCalculation(b *testing.B) { blobfee := uint256.NewInt(123_456_789_000) // Completely random, no idea what this will be basefeeJumps := dynamicFeeJumps(basefee) - blobfeeJumps := dynamicFeeJumps(blobfee) + blobfeeJumps := dynamicBlobFeeJumps(blobfee) // The transaction's fee cap and blob fee cap are constant across the life // of the transaction, so we can pre-calculate and cache them. @@ -77,7 +77,7 @@ func BenchmarkPriorityCalculation(b *testing.B) { txBlobfeeJumps := make([]float64, b.N) for i := 0; i < b.N; i++ { txBasefeeJumps[i] = dynamicFeeJumps(uint256.NewInt(rnd.Uint64())) - txBlobfeeJumps[i] = dynamicFeeJumps(uint256.NewInt(rnd.Uint64())) + txBlobfeeJumps[i] = dynamicBlobFeeJumps(uint256.NewInt(rnd.Uint64())) } b.ResetTimer() b.ReportAllocs() diff --git a/core/txpool/blobpool/slotter.go b/core/txpool/blobpool/slotter.go index 9b793e366c..3399361e55 100644 --- a/core/txpool/blobpool/slotter.go +++ b/core/txpool/blobpool/slotter.go @@ -17,7 +17,6 @@ package blobpool import ( - "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/params" "github.com/holiman/billy" ) @@ -42,7 +41,7 @@ func tryMigrate(config *params.ChainConfig, slotter billy.SlotSizeFn, datadir st // If the version found is less than the currently configured store version, // perform a migration then write the updated version of the store. if version < storeVersion { - newSlotter := newSlotterEIP7594(eip4844.LatestMaxBlobsPerBlock(config)) + newSlotter := newSlotterEIP7594(params.BlobTxMaxBlobs) if err := billy.Migrate(billy.Options{Path: datadir, Repair: true}, slotter, newSlotter); err != nil { return nil, err } @@ -54,7 +53,7 @@ func tryMigrate(config *params.ChainConfig, slotter billy.SlotSizeFn, datadir st store.Close() } // Set the slotter to the format now that the Osaka is active. - slotter = newSlotterEIP7594(eip4844.LatestMaxBlobsPerBlock(config)) + slotter = newSlotterEIP7594(params.BlobTxMaxBlobs) } return slotter, nil } diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index 9da0bc2121..2e92905714 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -114,6 +114,9 @@ var ( queuedGauge = metrics.NewRegisteredGauge("txpool/queued", nil) slotsGauge = metrics.NewRegisteredGauge("txpool/slots", nil) + pendingAddrsGauge = metrics.NewRegisteredGauge("txpool/pending/accounts", nil) + queuedAddrsGauge = metrics.NewRegisteredGauge("txpool/queued/accounts", nil) + reheapTimer = metrics.NewRegisteredTimer("txpool/reheap", nil) ) @@ -148,7 +151,7 @@ type Config struct { AccountQueue uint64 // Maximum number of non-executable transaction slots permitted per account GlobalQueue uint64 // Maximum number of non-executable transaction slots for all accounts - Lifetime time.Duration // Maximum amount of time non-executable transaction are queued + Lifetime time.Duration // Maximum amount of time an account can remain stale in the non-executable pool } // DefaultConfig contains the default configurations for the transaction pool. @@ -288,7 +291,12 @@ func New(config Config, chain BlockChain) *LegacyPool { // Filter returns whether the given transaction can be consumed by the legacy // pool, specifically, whether it is a Legacy, AccessList or Dynamic transaction. func (pool *LegacyPool) Filter(tx *types.Transaction) bool { - switch tx.Type() { + return pool.FilterType(tx.Type()) +} + +// FilterType returns whether the legacy pool supports the given transaction type. +func (pool *LegacyPool) FilterType(kind byte) bool { + switch kind { case types.LegacyTxType, types.AccessListTxType, types.DynamicFeeTxType, types.SetCodeTxType: return true default: @@ -486,15 +494,16 @@ func (pool *LegacyPool) ContentFrom(addr common.Address) ([]*types.Transaction, // // The transactions can also be pre-filtered by the dynamic fee components to // reduce allocations and load on downstream subsystems. -func (pool *LegacyPool) Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction { +func (pool *LegacyPool) Pending(filter txpool.PendingFilter) (map[common.Address][]*txpool.LazyTransaction, int) { // If only blob transactions are requested, this pool is unsuitable as it // contains none, don't even bother. if filter.BlobTxs { - return nil + return nil, 0 } pool.mu.Lock() defer pool.mu.Unlock() + var count int pending := make(map[common.Address][]*txpool.LazyTransaction, len(pool.pending)) for addr, list := range pool.pending { txs := list.Flatten() @@ -531,9 +540,10 @@ func (pool *LegacyPool) Pending(filter txpool.PendingFilter) map[common.Address] } } pending[addr] = lazies + count += len(lazies) } } - return pending + return pending, count } // ValidateTxBasics checks whether a transaction is valid according to the consensus @@ -770,7 +780,7 @@ func (pool *LegacyPool) add(tx *types.Transaction) (replaced bool, err error) { pool.queueTxEvent(tx) log.Trace("Pooled new executable transaction", "hash", hash, "from", from, "to", tx.To()) - // Successful promotion, bump the heartbeat + // Successful replacement. If needed, bump the heartbeat giving more time to queued txs. pool.queue.bump(from) return old != nil, nil } @@ -839,6 +849,7 @@ func (pool *LegacyPool) promoteTx(addr common.Address, hash common.Hash, tx *typ // Try to insert the transaction into the pending queue if pool.pending[addr] == nil { pool.pending[addr] = newList(true) + pendingAddrsGauge.Inc(1) } list := pool.pending[addr] @@ -862,7 +873,7 @@ func (pool *LegacyPool) promoteTx(addr common.Address, hash common.Hash, tx *typ // Set the potentially new pending nonce and notify any subsystems of the new tx pool.pendingNonces.set(addr, tx.Nonce()+1) - // Successful promotion, bump the heartbeat + // Successful promotion, bump the heartbeat, giving more time to queued txs. pool.queue.bump(addr) return true } @@ -899,8 +910,8 @@ func (pool *LegacyPool) addRemoteSync(tx *types.Transaction) error { func (pool *LegacyPool) Add(txs []*types.Transaction, sync bool) []error { // Filter out known ones without obtaining the pool lock or recovering signatures var ( - errs = make([]error, len(txs)) - news = make([]*types.Transaction, 0, len(txs)) + hasValid bool + errs = make([]error, len(txs)) ) for i, tx := range txs { // If the transaction is known, pre-set the error slot @@ -918,26 +929,17 @@ func (pool *LegacyPool) Add(txs []*types.Transaction, sync bool) []error { invalidTxMeter.Mark(1) continue } - // Accumulate all unknown transactions for deeper processing - news = append(news, tx) + hasValid = true } - if len(news) == 0 { + if !hasValid { return errs } // Process all the new transaction and merge any errors into the original slice pool.mu.Lock() - newErrs, dirtyAddrs := pool.addTxsLocked(news) + dirtyAddrs := pool.addTxsLocked(txs, errs) pool.mu.Unlock() - var nilSlot = 0 - for _, err := range newErrs { - for errs[nilSlot] != nil { - nilSlot++ - } - errs[nilSlot] = err - nilSlot++ - } // Reorg the pool internals if needed and return done := pool.requestPromoteExecutables(dirtyAddrs) if sync { @@ -948,14 +950,19 @@ func (pool *LegacyPool) Add(txs []*types.Transaction, sync bool) []error { // addTxsLocked attempts to queue a batch of transactions if they are valid. // The transaction pool lock must be held. -// Returns the error for each tx, and the set of accounts that might became promotable. -func (pool *LegacyPool) addTxsLocked(txs []*types.Transaction) ([]error, *accountSet) { +// Sets the error for each tx, and the set of accounts that might became promotable. +// We only try to add txs that have no error set in the errs slice. +// If adding the transaction returns an error, we set the error in the errs slice. +// Requires len(txs) == len(errs). +func (pool *LegacyPool) addTxsLocked(txs []*types.Transaction, errs []error) *accountSet { var ( dirty = newAccountSet(pool.signer) - errs = make([]error, len(txs)) valid int64 ) for i, tx := range txs { + if errs[i] != nil { + continue + } replaced, err := pool.add(tx) errs[i] = err if err == nil { @@ -966,7 +973,7 @@ func (pool *LegacyPool) addTxsLocked(txs []*types.Transaction) ([]error, *accoun } } validTxMeter.Mark(valid) - return errs, dirty + return dirty } // Status returns the status (unknown/pending/queued) of a batch of transactions @@ -991,11 +998,7 @@ func (pool *LegacyPool) Status(hash common.Hash) txpool.TxStatus { // Get returns a transaction if it is contained in the pool and nil otherwise. func (pool *LegacyPool) Get(hash common.Hash) *types.Transaction { - tx := pool.get(hash) - if tx == nil { - return nil - } - return tx + return pool.get(hash) } // get returns a transaction if it is contained in the pool and nil otherwise. @@ -1078,6 +1081,7 @@ func (pool *LegacyPool) removeTx(hash common.Hash, outofbound bool, unreserve bo // If no more pending transactions are left, remove the list if pending.Empty() { delete(pool.pending, addr) + pendingAddrsGauge.Dec(1) } // Postpone any invalidated transactions for _, tx := range invalids { @@ -1387,7 +1391,7 @@ func (pool *LegacyPool) reset(oldHead, newHead *types.Header) { // Inject any transactions discarded due to reorgs log.Debug("Reinjecting stale transactions", "count", len(reinject)) core.SenderCacher().Recover(pool.signer, reinject) - pool.addTxsLocked(reinject) + pool.addTxsLocked(reinject, make([]error, len(reinject))) } // promoteExecutables moves transactions that have become processable from the @@ -1400,7 +1404,7 @@ func (pool *LegacyPool) promoteExecutables(accounts []common.Address) []*types.T // promote all promotable transactions promoted := make([]*types.Transaction, 0, len(promotable)) for _, tx := range promotable { - from, _ := pool.signer.Sender(tx) + from, _ := types.Sender(pool.signer, tx) // already validated if pool.promoteTx(from, tx.Hash(), tx) { promoted = append(promoted, tx) } @@ -1558,6 +1562,7 @@ func (pool *LegacyPool) demoteUnexecutables() { // Internal shuffle shouldn't touch the lookup set. pool.enqueueTx(hash, tx, false) } + pool.priced.Removed(len(olds) + len(drops)) pendingGauge.Dec(int64(len(olds) + len(drops) + len(invalids))) // If there's a gap in front, alert (should never happen) and postpone all transactions @@ -1575,6 +1580,7 @@ func (pool *LegacyPool) demoteUnexecutables() { // Delete the entire pending entry if it became empty. if list.Empty() { delete(pool.pending, addr) + pendingAddrsGauge.Dec(1) if _, ok := pool.queue.get(addr); !ok { pool.reserver.Release(addr) } @@ -1834,6 +1840,13 @@ func (pool *LegacyPool) Clear() { pool.pending = make(map[common.Address]*list) pool.queue = newQueue(pool.config, pool.signer) pool.pendingNonces = newNoncer(pool.currentState) + + // Reset gauges + pendingGauge.Update(0) + queuedGauge.Update(0) + slotsGauge.Update(0) + pendingAddrsGauge.Update(0) + queuedAddrsGauge.Update(0) } // HasPendingAuth returns a flag indicating whether there are pending diff --git a/core/txpool/legacypool/list_test.go b/core/txpool/legacypool/list_test.go index 8587c66f7d..dc03def26c 100644 --- a/core/txpool/legacypool/list_test.go +++ b/core/txpool/legacypool/list_test.go @@ -68,6 +68,43 @@ func TestListAddVeryExpensive(t *testing.T) { } } +// TestPriceHeapCmp tests that the price heap comparison function works as intended. +// It also tests combinations where the basefee is higher than the gas fee cap, which +// are useful to sort in the mempool to support basefee changes. +func TestPriceHeapCmp(t *testing.T) { + key, _ := crypto.GenerateKey() + txs := []*types.Transaction{ + // nonce, gaslimit, gasfee, gastip + dynamicFeeTx(0, 1000, big.NewInt(2), big.NewInt(1), key), + dynamicFeeTx(0, 1000, big.NewInt(1), big.NewInt(2), key), + dynamicFeeTx(0, 1000, big.NewInt(1), big.NewInt(1), key), + dynamicFeeTx(0, 1000, big.NewInt(1), big.NewInt(0), key), + } + + // create priceHeap + ph := &priceHeap{} + + // now set the basefee on the heap + for _, basefee := range []uint64{0, 1, 2, 3} { + ph.baseFee = uint256.NewInt(basefee) + + for i := 0; i < len(txs); i++ { + for j := 0; j < len(txs); j++ { + switch { + case i == j: + if c := ph.cmp(txs[i], txs[j]); c != 0 { + t.Errorf("tx %d should be equal priority to tx %d with basefee %d (cmp=%d)", i, j, basefee, c) + } + case i < j: + if c := ph.cmp(txs[i], txs[j]); c != 1 { + t.Errorf("tx %d vs tx %d comparison inconsistent with basefee %d (cmp=%d)", i, j, basefee, c) + } + } + } + } + } +} + func BenchmarkListAdd(b *testing.B) { // Generate a list of transactions to insert key, _ := crypto.GenerateKey() diff --git a/core/txpool/legacypool/queue.go b/core/txpool/legacypool/queue.go index a889debe37..35beaded12 100644 --- a/core/txpool/legacypool/queue.go +++ b/core/txpool/legacypool/queue.go @@ -88,8 +88,12 @@ func (q *queue) get(addr common.Address) (*list, bool) { return l, ok } +// bump updates the heartbeat for the given account address. +// If the address is unknown, the call is a no-op. func (q *queue) bump(addr common.Address) { - q.beats[addr] = time.Now() + if _, ok := q.beats[addr]; ok { + q.beats[addr] = time.Now() + } } func (q *queue) addresses() []common.Address { @@ -114,6 +118,7 @@ func (q *queue) remove(addr common.Address, tx *types.Transaction) { if future.Empty() { delete(q.queued, addr) delete(q.beats, addr) + queuedAddrsGauge.Dec(1) } } } @@ -123,6 +128,7 @@ func (q *queue) add(tx *types.Transaction) (*common.Hash, error) { from, _ := types.Sender(q.signer, tx) // already validated if q.queued[from] == nil { q.queued[from] = newList(false) + queuedAddrsGauge.Inc(1) } inserted, old := q.queued[from].Add(tx, q.config.PriceBump) if !inserted { @@ -200,6 +206,7 @@ func (q *queue) promoteExecutables(accounts []common.Address, gasLimit uint64, c if list.Empty() { delete(q.queued, addr) delete(q.beats, addr) + queuedAddrsGauge.Dec(1) removedAddresses = append(removedAddresses, addr) } } diff --git a/core/txpool/subpool.go b/core/txpool/subpool.go index 519ae7b989..4cc1b193d6 100644 --- a/core/txpool/subpool.go +++ b/core/txpool/subpool.go @@ -100,6 +100,9 @@ type SubPool interface { // to this particular subpool. Filter(tx *types.Transaction) bool + // FilterType returns whether the subpool supports the given transaction type. + FilterType(kind byte) bool + // Init sets the base parameters of the subpool, allowing it to load any saved // transactions from disk and also permitting internal maintenance routines to // start up. @@ -151,7 +154,7 @@ type SubPool interface { // // The transactions can also be pre-filtered by the dynamic fee components to // reduce allocations and load on downstream subsystems. - Pending(filter PendingFilter) map[common.Address][]*LazyTransaction + Pending(filter PendingFilter) (map[common.Address][]*LazyTransaction, int) // SubscribeTransactions subscribes to new transaction events. The subscriber // can decide whether to receive notifications only for newly seen transactions diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index 437861efca..25647e0cce 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -359,14 +359,17 @@ func (p *TxPool) Add(txs []*types.Transaction, sync bool) []error { // // The transactions can also be pre-filtered by the dynamic fee components to // reduce allocations and load on downstream subsystems. -func (p *TxPool) Pending(filter PendingFilter) map[common.Address][]*LazyTransaction { +func (p *TxPool) Pending(filter PendingFilter) (map[common.Address][]*LazyTransaction, int) { + var count int txs := make(map[common.Address][]*LazyTransaction) for _, subpool := range p.subpools { - for addr, set := range subpool.Pending(filter) { - txs[addr] = set + set, n := subpool.Pending(filter) + for addr, list := range set { + txs[addr] = list } + count += n } - return txs + return txs, count } // SubscribeTransactions registers a subscription for new transaction events, @@ -489,3 +492,14 @@ func (p *TxPool) Clear() { subpool.Clear() } } + +// FilterType returns whether a transaction with the given type is supported +// (can be added) by the pool. +func (p *TxPool) FilterType(kind byte) bool { + for _, subpool := range p.subpools { + if subpool.FilterType(kind) { + return true + } + } + return false +} diff --git a/core/txpool/validation.go b/core/txpool/validation.go index 773de530e9..fd37294fea 100644 --- a/core/txpool/validation.go +++ b/core/txpool/validation.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" @@ -87,8 +88,10 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types return fmt.Errorf("%w: type %d rejected, pool not yet in Prague", core.ErrTxTypeNotSupported, tx.Type()) } // Check whether the init code size has been exceeded - if rules.IsShanghai && tx.To() == nil && len(tx.Data()) > int(opts.Config.MaxInitCodeSize()) { - return fmt.Errorf("%w: code size %v, limit %v", core.ErrMaxInitCodeSizeExceeded, len(tx.Data()), int(opts.Config.MaxInitCodeSize())) + if tx.To() == nil { + if err := vm.CheckMaxInitCodeSize(&rules, uint64(len(tx.Data()))); err != nil { + return err + } } if !opts.Config.IsArbitrum() && rules.IsOsaka && tx.Gas() > params.MaxTxGasRenamedForNitroMerges { return fmt.Errorf("%w (cap: %d, tx: %d)", core.ErrGasLimitTooHigh, params.MaxTxGasRenamedForNitroMerges, tx.Gas()) @@ -131,7 +134,7 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types return fmt.Errorf("%w: gas %v, minimum needed %v", core.ErrIntrinsicGas, tx.Gas(), intrGas) } // Ensure the transaction can cover floor data gas. - if opts.Config.IsPrague(head.Number, head.Time, arbosVersion) { + if rules.IsPrague { floorDataGas, err := core.FloorDataGas(tx.Data()) if err != nil { return err @@ -161,6 +164,15 @@ func validateBlobTx(tx *types.Transaction, head *types.Header, opts *ValidationO if sidecar == nil { return errors.New("missing sidecar in blob transaction") } + // Ensure the sidecar is constructed with the correct version, consistent + // with the current fork. + version := types.BlobSidecarVersion0 + if opts.Config.IsOsaka(head.Number, head.Time, types.DeserializeHeaderExtraInformation(head).ArbOSFormatVersion) { + version = types.BlobSidecarVersion1 + } + if sidecar.Version != version { + return fmt.Errorf("unexpected sidecar version, want: %d, got: %d", version, sidecar.Version) + } // Ensure the blob fee cap satisfies the minimum blob gas price if tx.BlobGasFeeCapIntCmp(blobTxMinBlobGasPrice) < 0 { return fmt.Errorf("%w: blob fee cap %v, minimum needed %v", ErrTxGasPriceTooLow, tx.BlobGasFeeCap(), blobTxMinBlobGasPrice) diff --git a/core/types.go b/core/types.go index bed20802ab..87bbfcff58 100644 --- a/core/types.go +++ b/core/types.go @@ -17,6 +17,7 @@ package core import ( + "context" "sync/atomic" "github.com/ethereum/go-ethereum/core/state" @@ -48,7 +49,7 @@ type Processor interface { // Process processes the state changes according to the Ethereum rules by running // the transaction messages using the statedb and applying any rewards to both // the processor (coinbase) and any included uncles. - Process(block *types.Block, statedb *state.StateDB, cfg vm.Config) (*ProcessResult, error) + Process(ctx context.Context, block *types.Block, statedb *state.StateDB, cfg vm.Config) (*ProcessResult, error) } // ProcessResult contains the values computed by Process. diff --git a/core/types/bal/bal.go b/core/types/bal/bal.go index fca54f7681..86dc8e5426 100644 --- a/core/types/bal/bal.go +++ b/core/types/bal/bal.go @@ -24,13 +24,6 @@ import ( "github.com/holiman/uint256" ) -// CodeChange contains the runtime bytecode deployed at an address and the -// transaction index where the deployment took place. -type CodeChange struct { - TxIndex uint16 - Code []byte `json:"code,omitempty"` -} - // ConstructionAccountAccess contains post-block account state for mutations as well as // all storage keys that were read during execution. It is used when building block // access list during execution. @@ -55,9 +48,9 @@ type ConstructionAccountAccess struct { // by tx index. NonceChanges map[uint16]uint64 `json:"nonceChanges,omitempty"` - // CodeChange is only set for contract accounts which were deployed in - // the block. - CodeChange *CodeChange `json:"codeChange,omitempty"` + // CodeChange contains the post-state contract code of an account keyed + // by tx index. + CodeChange map[uint16][]byte `json:"codeChange,omitempty"` } // NewConstructionAccountAccess initializes the account access object. @@ -67,6 +60,7 @@ func NewConstructionAccountAccess() *ConstructionAccountAccess { StorageReads: make(map[common.Hash]struct{}), BalanceChanges: make(map[uint16]*uint256.Int), NonceChanges: make(map[uint16]uint64), + CodeChange: make(map[uint16][]byte), } } @@ -120,10 +114,8 @@ func (b *ConstructionBlockAccessList) CodeChange(address common.Address, txIndex if _, ok := b.Accounts[address]; !ok { b.Accounts[address] = NewConstructionAccountAccess() } - b.Accounts[address].CodeChange = &CodeChange{ - TxIndex: txIndex, - Code: bytes.Clone(code), - } + // TODO(rjl493456442) is it essential to deep-copy the code? + b.Accounts[address].CodeChange[txIndex] = bytes.Clone(code) } // NonceChange records tx post-state nonce of any contract-like accounts whose @@ -170,12 +162,11 @@ func (b *ConstructionBlockAccessList) Copy() *ConstructionBlockAccessList { aaCopy.BalanceChanges = balances aaCopy.NonceChanges = maps.Clone(aa.NonceChanges) - if aa.CodeChange != nil { - aaCopy.CodeChange = &CodeChange{ - TxIndex: aa.CodeChange.TxIndex, - Code: bytes.Clone(aa.CodeChange.Code), - } + codes := make(map[uint16][]byte, len(aa.CodeChange)) + for index, code := range aa.CodeChange { + codes[index] = bytes.Clone(code) } + aaCopy.CodeChange = codes res.Accounts[addr] = &aaCopy } return &res diff --git a/core/types/bal/bal_encoding.go b/core/types/bal/bal_encoding.go index 35b6887739..915c8749fe 100644 --- a/core/types/bal/bal_encoding.go +++ b/core/types/bal/bal_encoding.go @@ -119,6 +119,13 @@ func (e *encodingSlotWrites) validate() error { return errors.New("storage write tx indices not in order") } +// encodingCodeChange contains the runtime bytecode deployed at an address +// and the transaction index where the deployment took place. +type encodingCodeChange struct { + TxIndex uint16 `ssz-size:"2"` + Code []byte `ssz-max:"300000"` // TODO(rjl493456442) shall we put the limit here? The limit will be increased gradually +} + // AccountAccess is the encoding format of ConstructionAccountAccess. type AccountAccess struct { Address [20]byte `ssz-size:"20"` // 20-byte Ethereum address @@ -126,7 +133,7 @@ type AccountAccess struct { StorageReads [][32]byte `ssz-max:"300000"` // Read-only storage keys BalanceChanges []encodingBalanceChange `ssz-max:"300000"` // Balance changes ([tx_index -> post_balance]) NonceChanges []encodingAccountNonce `ssz-max:"300000"` // Nonce changes ([tx_index -> new_nonce]) - Code []CodeChange `ssz-max:"1"` // Code changes ([tx_index -> new_code]) + CodeChanges []encodingCodeChange `ssz-max:"300000"` // Code changes ([tx_index -> new_code]) } // validate converts the account accesses out of encoding format. @@ -166,9 +173,16 @@ func (e *AccountAccess) validate() error { return errors.New("nonce changes not in ascending order by tx index") } - // Convert code change - if len(e.Code) == 1 { - if len(e.Code[0].Code) > params.DefaultMaxCodeSize { + // Check the code changes are sorted in order + if !slices.IsSortedFunc(e.CodeChanges, func(a, b encodingCodeChange) int { + return cmp.Compare[uint16](a.TxIndex, b.TxIndex) + }) { + return errors.New("code changes not in ascending order by tx index") + } + for _, change := range e.CodeChanges { + // TODO(rjl493456442): This check should be fork-aware, since the limit may + // differ across forks. + if len(change.Code) > params.DefaultMaxCodeSize { return errors.New("code change contained oversized code") } } @@ -182,6 +196,8 @@ func (e *AccountAccess) Copy() AccountAccess { StorageReads: slices.Clone(e.StorageReads), BalanceChanges: slices.Clone(e.BalanceChanges), NonceChanges: slices.Clone(e.NonceChanges), + StorageWrites: make([]encodingSlotWrites, 0, len(e.StorageWrites)), + CodeChanges: make([]encodingCodeChange, 0, len(e.CodeChanges)), } for _, storageWrite := range e.StorageWrites { res.StorageWrites = append(res.StorageWrites, encodingSlotWrites{ @@ -189,13 +205,11 @@ func (e *AccountAccess) Copy() AccountAccess { Accesses: slices.Clone(storageWrite.Accesses), }) } - if len(e.Code) == 1 { - res.Code = []CodeChange{ - { - e.Code[0].TxIndex, - bytes.Clone(e.Code[0].Code), - }, - } + for _, codeChange := range e.CodeChanges { + res.CodeChanges = append(res.CodeChanges, encodingCodeChange{ + TxIndex: codeChange.TxIndex, + Code: bytes.Clone(codeChange.Code), + }) } return res } @@ -212,11 +226,11 @@ var _ rlp.Encoder = &ConstructionBlockAccessList{} func (a *ConstructionAccountAccess) toEncodingObj(addr common.Address) AccountAccess { res := AccountAccess{ Address: addr, - StorageWrites: make([]encodingSlotWrites, 0), - StorageReads: make([][32]byte, 0), - BalanceChanges: make([]encodingBalanceChange, 0), - NonceChanges: make([]encodingAccountNonce, 0), - Code: nil, + StorageWrites: make([]encodingSlotWrites, 0, len(a.StorageWrites)), + StorageReads: make([][32]byte, 0, len(a.StorageReads)), + BalanceChanges: make([]encodingBalanceChange, 0, len(a.BalanceChanges)), + NonceChanges: make([]encodingAccountNonce, 0, len(a.NonceChanges)), + CodeChanges: make([]encodingCodeChange, 0, len(a.CodeChange)), } // Convert write slots @@ -268,13 +282,13 @@ func (a *ConstructionAccountAccess) toEncodingObj(addr common.Address) AccountAc } // Convert code change - if a.CodeChange != nil { - res.Code = []CodeChange{ - { - a.CodeChange.TxIndex, - bytes.Clone(a.CodeChange.Code), - }, - } + codeIndices := slices.Collect(maps.Keys(a.CodeChange)) + slices.SortFunc(codeIndices, cmp.Compare[uint16]) + for _, idx := range codeIndices { + res.CodeChanges = append(res.CodeChanges, encodingCodeChange{ + TxIndex: idx, + Code: a.CodeChange[idx], + }) } return res } @@ -327,9 +341,9 @@ func (e *BlockAccessList) PrettyPrint() string { printWithIndent(2, fmt.Sprintf("%d: %d", change.TxIdx, change.Nonce)) } - if len(accountDiff.Code) > 0 { - printWithIndent(1, "code:") - printWithIndent(2, fmt.Sprintf("%d: %x", accountDiff.Code[0].TxIndex, accountDiff.Code[0].Code)) + printWithIndent(1, "code changes:") + for _, change := range accountDiff.CodeChanges { + printWithIndent(2, fmt.Sprintf("%d: %x", change.TxIndex, change.Code)) } } return res.String() diff --git a/core/types/bal/bal_encoding_rlp_generated.go b/core/types/bal/bal_encoding_rlp_generated.go index 0d52395329..640035e30e 100644 --- a/core/types/bal/bal_encoding_rlp_generated.go +++ b/core/types/bal/bal_encoding_rlp_generated.go @@ -49,7 +49,7 @@ func (obj *BlockAccessList) EncodeRLP(_w io.Writer) error { } w.ListEnd(_tmp15) _tmp18 := w.List() - for _, _tmp19 := range _tmp2.Code { + for _, _tmp19 := range _tmp2.CodeChanges { _tmp20 := w.List() w.WriteUint64(uint64(_tmp19.TxIndex)) w.WriteBytes(_tmp19.Code) @@ -228,13 +228,13 @@ func (obj *BlockAccessList) DecodeRLP(dec *rlp.Stream) error { return err } _tmp2.NonceChanges = _tmp17 - // Code: - var _tmp21 []CodeChange + // CodeChanges: + var _tmp21 []encodingCodeChange if _, err := dec.List(); err != nil { return err } for dec.MoreDataInList() { - var _tmp22 CodeChange + var _tmp22 encodingCodeChange { if _, err := dec.List(); err != nil { return err @@ -260,7 +260,7 @@ func (obj *BlockAccessList) DecodeRLP(dec *rlp.Stream) error { if err := dec.ListEnd(); err != nil { return err } - _tmp2.Code = _tmp21 + _tmp2.CodeChanges = _tmp21 if err := dec.ListEnd(); err != nil { return err } diff --git a/core/types/bal/bal_test.go b/core/types/bal/bal_test.go index 29414e414e..52c0de825e 100644 --- a/core/types/bal/bal_test.go +++ b/core/types/bal/bal_test.go @@ -60,9 +60,8 @@ func makeTestConstructionBAL() *ConstructionBlockAccessList { 1: 2, 2: 6, }, - CodeChange: &CodeChange{ - TxIndex: 0, - Code: common.Hex2Bytes("deadbeef"), + CodeChange: map[uint16][]byte{ + 0: common.Hex2Bytes("deadbeef"), }, }, common.BytesToAddress([]byte{0xff, 0xff, 0xff}): { @@ -85,6 +84,9 @@ func makeTestConstructionBAL() *ConstructionBlockAccessList { NonceChanges: map[uint16]uint64{ 1: 2, }, + CodeChange: map[uint16][]byte{ + 0: common.Hex2Bytes("deadbeef"), + }, }, }, } @@ -179,7 +181,7 @@ func makeTestAccountAccess(sort bool) AccountAccess { StorageReads: storageReads, BalanceChanges: balances, NonceChanges: nonces, - Code: []CodeChange{ + CodeChanges: []encodingCodeChange{ { TxIndex: 100, Code: testrand.Bytes(256), diff --git a/core/types/block.go b/core/types/block.go index b5b6468a13..d092351b58 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -31,7 +31,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-verkle" ) // A BlockNonce is a 64-bit hash which proves (combined with the @@ -61,13 +60,6 @@ func (n *BlockNonce) UnmarshalText(input []byte) error { return hexutil.UnmarshalFixedText("BlockNonce", input, n[:]) } -// ExecutionWitness represents the witness + proof used in a verkle context, -// to provide the ability to execute a block statelessly. -type ExecutionWitness struct { - StateDiff verkle.StateDiff `json:"stateDiff"` - VerkleProof *verkle.VerkleProof `json:"verkleProof"` -} - //go:generate go run github.com/fjl/gencodec -type Header -field-override headerMarshaling -out gen_header_json.go //go:generate go run ../../rlp/rlpgen -type Header -out gen_header_rlp.go @@ -106,6 +98,9 @@ type Header struct { // RequestsHash was added by EIP-7685 and is ignored in legacy headers. RequestsHash *common.Hash `json:"requestsHash" rlp:"optional"` + + // SlotNumber was added by EIP-7843 and is ignored in legacy headers. + SlotNumber *uint64 `json:"slotNumber" rlp:"optional"` } // field type overrides for gencodec @@ -120,6 +115,7 @@ type headerMarshaling struct { Hash common.Hash `json:"hash"` // adds call to Hash() in MarshalJSON BlobGasUsed *hexutil.Uint64 ExcessBlobGas *hexutil.Uint64 + SlotNumber *hexutil.Uint64 } // Hash returns the block hash of the header, which is simply the keccak256 hash of its @@ -209,11 +205,6 @@ type Block struct { transactions Transactions withdrawals Withdrawals - // witness is not an encoded part of the block body. - // It is held in Block in order for easy relaying to the places - // that process it. - witness *ExecutionWitness - // caches hash atomic.Pointer[common.Hash] size atomic.Uint64 @@ -329,6 +320,10 @@ func CopyHeader(h *Header) *Header { cpy.RequestsHash = new(common.Hash) *cpy.RequestsHash = *h.RequestsHash } + if h.SlotNumber != nil { + cpy.SlotNumber = new(uint64) + *cpy.SlotNumber = *h.SlotNumber + } return &cpy } @@ -429,8 +424,14 @@ func (b *Block) BlobGasUsed() *uint64 { return blobGasUsed } -// ExecutionWitness returns the verkle execution witneess + proof for a block -func (b *Block) ExecutionWitness() *ExecutionWitness { return b.witness } +func (b *Block) SlotNumber() *uint64 { + var slotNum *uint64 + if b.header.SlotNumber != nil { + slotNum = new(uint64) + *slotNum = *b.header.SlotNumber + } + return slotNum +} // Size returns the true RLP encoded storage size of the block, either by encoding // and returning it, or returning a previously cached value. @@ -494,7 +495,6 @@ func (b *Block) WithSeal(header *Header) *Block { transactions: b.transactions, uncles: b.uncles, withdrawals: b.withdrawals, - witness: b.witness, } } @@ -506,7 +506,6 @@ func (b *Block) WithBody(body Body) *Block { transactions: slices.Clone(body.Transactions), uncles: make([]*Header, len(body.Uncles)), withdrawals: slices.Clone(body.Withdrawals), - witness: b.witness, } for i := range body.Uncles { block.uncles[i] = CopyHeader(body.Uncles[i]) @@ -514,16 +513,6 @@ func (b *Block) WithBody(body Body) *Block { return block } -func (b *Block) WithWitness(witness *ExecutionWitness) *Block { - return &Block{ - header: b.header, - transactions: b.transactions, - uncles: b.uncles, - withdrawals: b.withdrawals, - witness: witness, - } -} - // Hash returns the keccak256 hash of b's header. // The hash is computed on the first call and cached thereafter. func (b *Block) Hash() common.Hash { diff --git a/core/types/gen_header_json.go b/core/types/gen_header_json.go index 0af12500bd..16fb03f612 100644 --- a/core/types/gen_header_json.go +++ b/core/types/gen_header_json.go @@ -37,6 +37,7 @@ func (h Header) MarshalJSON() ([]byte, error) { ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas" rlp:"optional"` ParentBeaconRoot *common.Hash `json:"parentBeaconBlockRoot" rlp:"optional"` RequestsHash *common.Hash `json:"requestsHash" rlp:"optional"` + SlotNumber *hexutil.Uint64 `json:"slotNumber" rlp:"optional"` Hash common.Hash `json:"hash"` } var enc Header @@ -61,6 +62,7 @@ func (h Header) MarshalJSON() ([]byte, error) { enc.ExcessBlobGas = (*hexutil.Uint64)(h.ExcessBlobGas) enc.ParentBeaconRoot = h.ParentBeaconRoot enc.RequestsHash = h.RequestsHash + enc.SlotNumber = (*hexutil.Uint64)(h.SlotNumber) enc.Hash = h.Hash() return json.Marshal(&enc) } @@ -89,6 +91,7 @@ func (h *Header) UnmarshalJSON(input []byte) error { ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas" rlp:"optional"` ParentBeaconRoot *common.Hash `json:"parentBeaconBlockRoot" rlp:"optional"` RequestsHash *common.Hash `json:"requestsHash" rlp:"optional"` + SlotNumber *hexutil.Uint64 `json:"slotNumber" rlp:"optional"` } var dec Header if err := json.Unmarshal(input, &dec); err != nil { @@ -169,5 +172,8 @@ func (h *Header) UnmarshalJSON(input []byte) error { if dec.RequestsHash != nil { h.RequestsHash = dec.RequestsHash } + if dec.SlotNumber != nil { + h.SlotNumber = (*uint64)(dec.SlotNumber) + } return nil } diff --git a/core/types/gen_header_rlp.go b/core/types/gen_header_rlp.go index c79aa8a250..cfbd57ab8a 100644 --- a/core/types/gen_header_rlp.go +++ b/core/types/gen_header_rlp.go @@ -43,7 +43,8 @@ func (obj *Header) EncodeRLP(_w io.Writer) error { _tmp4 := obj.ExcessBlobGas != nil _tmp5 := obj.ParentBeaconRoot != nil _tmp6 := obj.RequestsHash != nil - if _tmp1 || _tmp2 || _tmp3 || _tmp4 || _tmp5 || _tmp6 { + _tmp7 := obj.SlotNumber != nil + if _tmp1 || _tmp2 || _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 { if obj.BaseFee == nil { w.Write(rlp.EmptyString) } else { @@ -53,41 +54,48 @@ func (obj *Header) EncodeRLP(_w io.Writer) error { w.WriteBigInt(obj.BaseFee) } } - if _tmp2 || _tmp3 || _tmp4 || _tmp5 || _tmp6 { + if _tmp2 || _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 { if obj.WithdrawalsHash == nil { w.Write([]byte{0x80}) } else { w.WriteBytes(obj.WithdrawalsHash[:]) } } - if _tmp3 || _tmp4 || _tmp5 || _tmp6 { + if _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 { if obj.BlobGasUsed == nil { w.Write([]byte{0x80}) } else { w.WriteUint64((*obj.BlobGasUsed)) } } - if _tmp4 || _tmp5 || _tmp6 { + if _tmp4 || _tmp5 || _tmp6 || _tmp7 { if obj.ExcessBlobGas == nil { w.Write([]byte{0x80}) } else { w.WriteUint64((*obj.ExcessBlobGas)) } } - if _tmp5 || _tmp6 { + if _tmp5 || _tmp6 || _tmp7 { if obj.ParentBeaconRoot == nil { w.Write([]byte{0x80}) } else { w.WriteBytes(obj.ParentBeaconRoot[:]) } } - if _tmp6 { + if _tmp6 || _tmp7 { if obj.RequestsHash == nil { w.Write([]byte{0x80}) } else { w.WriteBytes(obj.RequestsHash[:]) } } + if _tmp7 { + if obj.SlotNumber == nil { + w.Write([]byte{0x80}) + } else { + w.WriteUint64((*obj.SlotNumber)) + } + } w.ListEnd(_tmp0) return w.Flush() } diff --git a/core/types/log.go b/core/types/log.go index 84dc051f82..fd8f6b1428 100644 --- a/core/types/log.go +++ b/core/types/log.go @@ -22,6 +22,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" ) //go:generate go run ../../rlp/rlpgen -type Log -out gen_log_rlp.go @@ -68,7 +70,7 @@ type logMarshaling struct { // FilterLogs creates a slice of logs matching the given criteria. func FilterLogs(logs []*Log, fromBlock, toBlock *big.Int, addresses []common.Address, topics [][]common.Hash) []*Log { - var check = func(log *Log) bool { + check := func(log *Log) bool { if fromBlock != nil && fromBlock.Int64() >= 0 && fromBlock.Uint64() > log.BlockNumber { return false } @@ -100,3 +102,32 @@ func FilterLogs(logs []*Log, fromBlock, toBlock *big.Int, addresses []common.Add } return ret } + +// EthTransferLog creates and ETH transfer log according to EIP-7708. +// Specification: https://eips.ethereum.org/EIPS/eip-7708 +func EthTransferLog(from, to common.Address, amount *uint256.Int) *Log { + amount32 := amount.Bytes32() + return &Log{ + Address: params.SystemAddress, + Topics: []common.Hash{ + params.EthTransferLogEvent, + common.BytesToHash(from.Bytes()), + common.BytesToHash(to.Bytes()), + }, + Data: amount32[:], + } +} + +// EthBurnLog creates an ETH burn log according to EIP-7708. +// Specification: https://eips.ethereum.org/EIPS/eip-7708 +func EthBurnLog(from common.Address, amount *uint256.Int) *Log { + amount32 := amount.Bytes32() + return &Log{ + Address: params.SystemAddress, + Topics: []common.Hash{ + params.EthBurnLogEvent, + common.BytesToHash(from.Bytes()), + }, + Data: amount32[:], + } +} diff --git a/core/types/receipt.go b/core/types/receipt.go index 0ba5034c68..9ef45b566f 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -539,3 +539,33 @@ func EncodeBlockReceiptLists(receipts []Receipts) []rlp.RawValue { } return result } + +// SlimReceipt is a wrapper around a Receipt with RLP serialization that omits +// the Bloom field and includes the tx type. Used for era files. +type SlimReceipt Receipt + +type slimReceiptRLP struct { + Type uint8 + StatusEncoding []byte + CumulativeGasUsed uint64 + Logs []*Log +} + +// EncodeRLP implements rlp.Encoder, encoding the receipt as +// [tx-type, post-state-or-status, cumulative-gas, logs]. +func (r *SlimReceipt) EncodeRLP(w io.Writer) error { + data := &slimReceiptRLP{r.Type, (*Receipt)(r).statusEncoding(), r.CumulativeGasUsed, r.Logs} + return rlp.Encode(w, data) +} + +// DecodeRLP implements rlp.Decoder. +func (r *SlimReceipt) DecodeRLP(s *rlp.Stream) error { + var data slimReceiptRLP + if err := s.Decode(&data); err != nil { + return err + } + r.Type = data.Type + r.CumulativeGasUsed = data.CumulativeGasUsed + r.Logs = data.Logs + return (*Receipt)(r).setStatus(data.StatusEncoding) +} diff --git a/core/types/receipt_test.go b/core/types/receipt_test.go index 8408d05ce4..87342ad27f 100644 --- a/core/types/receipt_test.go +++ b/core/types/receipt_test.go @@ -515,6 +515,45 @@ func TestReceiptUnmarshalBinary(t *testing.T) { } } +func TestSlimReceiptEncodingDecoding(t *testing.T) { + tests := []*Receipt{ + legacyReceipt, + accessListReceipt, + eip1559Receipt, + { + Type: BlobTxType, + Status: ReceiptStatusSuccessful, + CumulativeGasUsed: 100, + Logs: []*Log{}, + }, + } + for i, want := range tests { + enc, err := rlp.EncodeToBytes((*SlimReceipt)(want)) + if err != nil { + t.Fatalf("test %d: encode error: %v", i, err) + } + got := new(SlimReceipt) + if err := rlp.DecodeBytes(enc, got); err != nil { + t.Fatalf("test %d: decode error: %v", i, err) + } + if got.Type != want.Type { + t.Errorf("test %d: Type mismatch: got %d, want %d", i, got.Type, want.Type) + } + if got.Status != want.Status { + t.Errorf("test %d: Status mismatch: got %d, want %d", i, got.Status, want.Status) + } + if !bytes.Equal(got.PostState, want.PostState) { + t.Errorf("test %d: PostState mismatch: got %x, want %x", i, got.PostState, want.PostState) + } + if got.CumulativeGasUsed != want.CumulativeGasUsed { + t.Errorf("test %d: CumulativeGasUsed mismatch: got %d, want %d", i, got.CumulativeGasUsed, want.CumulativeGasUsed) + } + if len(got.Logs) != len(want.Logs) { + t.Errorf("test %d: Logs length mismatch: got %d, want %d", i, len(got.Logs), len(want.Logs)) + } + } +} + func clearComputedFieldsOnReceipts(receipts []*Receipt) []*Receipt { r := make([]*Receipt, len(receipts)) for i, receipt := range receipts { diff --git a/core/types/transaction.go b/core/types/transaction.go index 337ab4666d..24f94edef3 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -396,11 +396,15 @@ func (tx *Transaction) To() *common.Address { // Cost returns (gas * gasPrice) + (blobGas * blobGasPrice) + value. func (tx *Transaction) Cost() *big.Int { - total := new(big.Int).Mul(tx.GasPrice(), new(big.Int).SetUint64(tx.Gas())) - if tx.Type() == BlobTxType { - total.Add(total, new(big.Int).Mul(tx.BlobGasFeeCap(), new(big.Int).SetUint64(tx.BlobGas()))) + // Avoid allocating copies via tx.GasPrice()/tx.Value(); use inner values directly. + total := new(big.Int).SetUint64(tx.inner.gas()) + total.Mul(total, tx.inner.gasPrice()) + if blobtx, ok := tx.inner.(*BlobTx); ok { + tmp := new(big.Int).SetUint64(blobtx.blobGas()) + tmp.Mul(tmp, blobtx.BlobFeeCap.ToBig()) + total.Add(total, tmp) } - total.Add(total, tx.Value()) + total.Add(total, tx.inner.value()) return total } @@ -475,14 +479,37 @@ func (tx *Transaction) calcEffectiveGasTip(dst *uint256.Int, baseFee *uint256.In return err } +// EffectiveGasTipValue returns the effective gasTip value for the given base fee, +// even if it would be negative. This can be used for sorting purposes. +func (tx *Transaction) EffectiveGasTipValue(baseFee *big.Int) *big.Int { + // min(gasTipCap, gasFeeCap - baseFee) + dst := new(big.Int) + if baseFee == nil { + dst.Set(tx.inner.gasTipCap()) + return dst + } + + dst.Sub(tx.inner.gasFeeCap(), baseFee) // gasFeeCap - baseFee + gasTipCap := tx.inner.gasTipCap() + if gasTipCap.Cmp(dst) < 0 { // gasTipCap < (gasFeeCap - baseFee) + dst.Set(gasTipCap) + } + return dst +} + func (tx *Transaction) EffectiveGasTipCmp(other *Transaction, baseFee *uint256.Int) int { if baseFee == nil { return tx.GasTipCapCmp(other) } // Use more efficient internal method. txTip, otherTip := new(uint256.Int), new(uint256.Int) - tx.calcEffectiveGasTip(txTip, baseFee) - other.calcEffectiveGasTip(otherTip, baseFee) + err1 := tx.calcEffectiveGasTip(txTip, baseFee) + err2 := other.calcEffectiveGasTip(otherTip, baseFee) + if err1 != nil || err2 != nil { + // fall back to big int comparison in case of error + base := baseFee.ToBig() + return tx.EffectiveGasTipValue(base).Cmp(other.EffectiveGasTipValue(base)) + } return txTip.Cmp(otherTip) } diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index cbe12139eb..893a200d5c 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -299,7 +299,10 @@ func (s *modernSigner) SignatureValues(tx *Transaction, sig []byte) (R, S, V *bi if tx.inner.chainID().Sign() != 0 && tx.inner.chainID().Cmp(s.chainID) != 0 { return nil, nil, nil, fmt.Errorf("%w: have %d want %d", ErrInvalidChainId, tx.inner.chainID(), s.chainID) } - R, S, _ = decodeSignature(sig) + R, S, _, err = decodeSignature(sig) + if err != nil { + return nil, nil, nil, err + } V = big.NewInt(int64(sig[64])) return R, S, V, nil } @@ -344,7 +347,7 @@ func NewEIP2930Signer(chainId *big.Int) Signer { // are replay-protected as well as unprotected homestead transactions. // Deprecated: always use the Signer interface type type EIP155Signer struct { - chainId, chainIdMul *big.Int + chainId *big.Int } func NewEIP155Signer(chainId *big.Int) EIP155Signer { @@ -352,8 +355,7 @@ func NewEIP155Signer(chainId *big.Int) EIP155Signer { chainId = new(big.Int) } return EIP155Signer{ - chainId: chainId, - chainIdMul: new(big.Int).Mul(chainId, big.NewInt(2)), + chainId: chainId, } } @@ -379,7 +381,8 @@ func (s EIP155Signer) Sender(tx *Transaction) (common.Address, error) { return common.Address{}, fmt.Errorf("%w: have %d want %d", ErrInvalidChainId, tx.ChainId(), s.chainId) } V, R, S := tx.RawSignatureValues() - V = new(big.Int).Sub(V, s.chainIdMul) + V = new(big.Int).Sub(V, s.chainId) + V = new(big.Int).Sub(V, s.chainId) V.Sub(V, big8) return recoverPlain(s.Hash(tx), R, S, V, true) } @@ -390,10 +393,14 @@ func (s EIP155Signer) SignatureValues(tx *Transaction, sig []byte) (R, S, V *big if tx.Type() != LegacyTxType { return nil, nil, nil, ErrTxTypeNotSupported } - R, S, V = decodeSignature(sig) + R, S, V, err = decodeSignature(sig) + if err != nil { + return nil, nil, nil, err + } if s.chainId.Sign() != 0 { V = big.NewInt(int64(sig[64] + 35)) - V.Add(V, s.chainIdMul) + V.Add(V, s.chainId) + V.Add(V, s.chainId) } return R, S, V, nil } @@ -459,8 +466,8 @@ func (fs FrontierSigner) SignatureValues(tx *Transaction, sig []byte) (r, s, v * if tx.Type() != LegacyTxType { return nil, nil, nil, ErrTxTypeNotSupported } - r, s, v = decodeSignature(sig) - return r, s, v, nil + r, s, v, err = decodeSignature(sig) + return r, s, v, err } // Hash returns the hash to be signed by the sender. @@ -476,14 +483,14 @@ func (fs FrontierSigner) Hash(tx *Transaction) common.Hash { }) } -func decodeSignature(sig []byte) (r, s, v *big.Int) { +func decodeSignature(sig []byte) (r, s, v *big.Int, err error) { if len(sig) != crypto.SignatureLength { - panic(fmt.Sprintf("wrong size for signature: got %d, want %d", len(sig), crypto.SignatureLength)) + return nil, nil, nil, fmt.Errorf("wrong size for signature: got %d, want %d", len(sig), crypto.SignatureLength) } r = new(big.Int).SetBytes(sig[:32]) s = new(big.Int).SetBytes(sig[32:64]) v = new(big.Int).SetBytes([]byte{sig[64] + 27}) - return r, s, v + return r, s, v, nil } func recoverPlain(sighash common.Hash, R, S, Vb *big.Int, homestead bool) (common.Address, error) { diff --git a/core/types/transaction_signing_test.go b/core/types/transaction_signing_test.go index 02a65fda13..252593e87b 100644 --- a/core/types/transaction_signing_test.go +++ b/core/types/transaction_signing_test.go @@ -200,3 +200,28 @@ func Benchmark_modernSigner_Equal(b *testing.B) { } } } + +func TestSignatureValuesError(t *testing.T) { + // 1. Setup a valid transaction + tx := NewTransaction(0, common.Address{}, big.NewInt(0), 0, big.NewInt(0), nil) + signer := HomesteadSigner{} + + // 2. Call WithSignature with invalid length sig (not 65 bytes) + invalidSig := make([]byte, 64) + + func() { + defer func() { + if r := recover(); r != nil { + t.Fatalf("Panicked for invalid signature length, expected error: %v", r) + } + }() + _, err := tx.WithSignature(signer, invalidSig) + if err == nil { + t.Fatal("Expected error for invalid signature length, got nil") + } else { + // This is just a sanity check to ensure we got an error, + // the exact error message is verified in unit tests elsewhere if needed. + t.Logf("Got expected error: %v", err) + } + }() +} diff --git a/core/types/tx_blob.go b/core/types/tx_blob.go index dee4d1fc2d..e7a2b91601 100644 --- a/core/types/tx_blob.go +++ b/core/types/tx_blob.go @@ -118,8 +118,8 @@ func (sc *BlobTxSidecar) ToV1() error { } if sc.Version == BlobSidecarVersion0 { proofs := make([]kzg4844.Proof, 0, len(sc.Blobs)*kzg4844.CellProofsPerBlob) - for _, blob := range sc.Blobs { - cellProofs, err := kzg4844.ComputeCellProofs(&blob) + for i := range sc.Blobs { + cellProofs, err := kzg4844.ComputeCellProofs(&sc.Blobs[i]) if err != nil { return err } diff --git a/core/types/tx_setcode.go b/core/types/tx_setcode.go index f2281d4ae7..9487c9cc81 100644 --- a/core/types/tx_setcode.go +++ b/core/types/tx_setcode.go @@ -94,7 +94,10 @@ func SignSetCode(prv *ecdsa.PrivateKey, auth SetCodeAuthorization) (SetCodeAutho if err != nil { return SetCodeAuthorization{}, err } - r, s, _ := decodeSignature(sig) + r, s, _, err := decodeSignature(sig) + if err != nil { + return SetCodeAuthorization{}, err + } return SetCodeAuthorization{ ChainID: auth.ChainID, Address: auth.Address, diff --git a/core/verkle_witness_test.go b/core/verkle_witness_test.go deleted file mode 100644 index d8486e2dc9..0000000000 --- a/core/verkle_witness_test.go +++ /dev/null @@ -1,1107 +0,0 @@ -// Copyright 2024 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package core - -import ( - "bytes" - "encoding/binary" - "encoding/hex" - "fmt" - "math/big" - "slices" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus/beacon" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/tracing" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie/utils" - "github.com/ethereum/go-ethereum/triedb" - "github.com/ethereum/go-verkle" - "github.com/holiman/uint256" -) - -var ( - testVerkleChainConfig = ¶ms.ChainConfig{ - ChainID: big.NewInt(1), - HomesteadBlock: big.NewInt(0), - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - ByzantiumBlock: big.NewInt(0), - ConstantinopleBlock: big.NewInt(0), - PetersburgBlock: big.NewInt(0), - IstanbulBlock: big.NewInt(0), - MuirGlacierBlock: big.NewInt(0), - BerlinBlock: big.NewInt(0), - LondonBlock: big.NewInt(0), - Ethash: new(params.EthashConfig), - ShanghaiTime: u64(0), - VerkleTime: u64(0), - TerminalTotalDifficulty: common.Big0, - EnableVerkleAtGenesis: true, - BlobScheduleConfig: ¶ms.BlobScheduleConfig{ - Verkle: params.DefaultPragueBlobConfig, - }, - // TODO uncomment when proof generation is merged - // ProofInBlocks: true, - } - testKaustinenLikeChainConfig = ¶ms.ChainConfig{ - ChainID: big.NewInt(69420), - HomesteadBlock: big.NewInt(0), - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - ByzantiumBlock: big.NewInt(0), - ConstantinopleBlock: big.NewInt(0), - PetersburgBlock: big.NewInt(0), - IstanbulBlock: big.NewInt(0), - MuirGlacierBlock: big.NewInt(0), - BerlinBlock: big.NewInt(0), - LondonBlock: big.NewInt(0), - Ethash: new(params.EthashConfig), - ShanghaiTime: u64(0), - VerkleTime: u64(0), - TerminalTotalDifficulty: common.Big0, - EnableVerkleAtGenesis: true, - BlobScheduleConfig: ¶ms.BlobScheduleConfig{ - Verkle: params.DefaultPragueBlobConfig, - }, - } -) - -func TestProcessVerkle(t *testing.T) { - var ( - code = common.FromHex(`6060604052600a8060106000396000f360606040526008565b00`) - intrinsicContractCreationGas, _ = IntrinsicGas(code, nil, nil, true, true, true, true) - // A contract creation that calls EXTCODECOPY in the constructor. Used to ensure that the witness - // will not contain that copied data. - // Source: https://gist.github.com/gballet/a23db1e1cb4ed105616b5920feb75985 - codeWithExtCodeCopy = common.FromHex(`0x60806040526040516100109061017b565b604051809103906000f08015801561002c573d6000803e3d6000fd5b506000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff16021790555034801561007857600080fd5b5060008067ffffffffffffffff8111156100955761009461024a565b5b6040519080825280601f01601f1916602001820160405280156100c75781602001600182028036833780820191505090505b50905060008060009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1690506020600083833c81610101906101e3565b60405161010d90610187565b61011791906101a3565b604051809103906000f080158015610133573d6000803e3d6000fd5b50600160006101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550505061029b565b60d58061046783390190565b6102068061053c83390190565b61019d816101d9565b82525050565b60006020820190506101b86000830184610194565b92915050565b6000819050602082019050919050565b600081519050919050565b6000819050919050565b60006101ee826101ce565b826101f8846101be565b905061020381610279565b925060208210156102435761023e7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8360200360080261028e565b831692505b5050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b600061028582516101d9565b80915050919050565b600082821b905092915050565b6101bd806102aa6000396000f3fe608060405234801561001057600080fd5b506004361061002b5760003560e01c8063f566852414610030575b600080fd5b61003861004e565b6040516100459190610146565b60405180910390f35b6000600160009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff166381ca91d36040518163ffffffff1660e01b815260040160206040518083038186803b1580156100b857600080fd5b505afa1580156100cc573d6000803e3d6000fd5b505050506040513d601f19601f820116820180604052508101906100f0919061010a565b905090565b60008151905061010481610170565b92915050565b6000602082840312156101205761011f61016b565b5b600061012e848285016100f5565b91505092915050565b61014081610161565b82525050565b600060208201905061015b6000830184610137565b92915050565b6000819050919050565b600080fd5b61017981610161565b811461018457600080fd5b5056fea2646970667358221220a6a0e11af79f176f9c421b7b12f441356b25f6489b83d38cc828a701720b41f164736f6c63430008070033608060405234801561001057600080fd5b5060b68061001f6000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c8063ab5ed15014602d575b600080fd5b60336047565b604051603e9190605d565b60405180910390f35b60006001905090565b6057816076565b82525050565b6000602082019050607060008301846050565b92915050565b600081905091905056fea26469706673582212203a14eb0d5cd07c277d3e24912f110ddda3e553245a99afc4eeefb2fbae5327aa64736f6c63430008070033608060405234801561001057600080fd5b5060405161020638038061020683398181016040528101906100329190610063565b60018160001c6100429190610090565b60008190555050610145565b60008151905061005d8161012e565b92915050565b60006020828403121561007957610078610129565b5b60006100878482850161004e565b91505092915050565b600061009b826100f0565b91506100a6836100f0565b9250827fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff038211156100db576100da6100fa565b5b828201905092915050565b6000819050919050565b6000819050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fd5b600080fd5b610137816100e6565b811461014257600080fd5b50565b60b3806101536000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c806381ca91d314602d575b600080fd5b60336047565b604051603e9190605a565b60405180910390f35b60005481565b6054816073565b82525050565b6000602082019050606d6000830184604d565b92915050565b600081905091905056fea26469706673582212209bff7098a2f526de1ad499866f27d6d0d6f17b74a413036d6063ca6a0998ca4264736f6c63430008070033`) - intrinsicCodeWithExtCodeCopyGas, _ = IntrinsicGas(codeWithExtCodeCopy, nil, nil, true, true, true, true) - signer = types.LatestSigner(testVerkleChainConfig) - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - bcdb = rawdb.NewMemoryDatabase() // Database for the blockchain - coinbase = common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7") - gspec = &Genesis{ - Config: testVerkleChainConfig, - Alloc: GenesisAlloc{ - coinbase: { - Balance: big.NewInt(1000000000000000000), // 1 ether - Nonce: 0, - }, - params.BeaconRootsAddress: {Nonce: 1, Code: params.BeaconRootsCode, Balance: common.Big0}, - params.HistoryStorageAddress: {Nonce: 1, Code: params.HistoryStorageCode, Balance: common.Big0}, - params.WithdrawalQueueAddress: {Nonce: 1, Code: params.WithdrawalQueueCode, Balance: common.Big0}, - params.ConsolidationQueueAddress: {Nonce: 1, Code: params.ConsolidationQueueCode, Balance: common.Big0}, - }, - } - ) - // Verkle trees use the snapshot, which must be enabled before the - // data is saved into the tree+database. - // genesis := gspec.MustCommit(bcdb, triedb) - options := DefaultConfig().WithStateScheme(rawdb.PathScheme) - options.SnapshotLimit = 0 - blockchain, _ := NewBlockChain(bcdb, nil, gspec, beacon.New(ethash.NewFaker()), options) - defer blockchain.Stop() - - txCost1 := params.TxGas - txCost2 := params.TxGas - contractCreationCost := intrinsicContractCreationGas + - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + params.WitnessBranchReadCost + params.WitnessBranchWriteCost + /* creation */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* creation with value */ - 739 /* execution costs */ - codeWithExtCodeCopyGas := intrinsicCodeWithExtCodeCopyGas + - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + params.WitnessBranchReadCost + params.WitnessBranchWriteCost + /* creation (tx) */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + params.WitnessBranchReadCost + params.WitnessBranchWriteCost + /* creation (CREATE at pc=0x20) */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* write code hash */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #0 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #1 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #2 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #3 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #4 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #5 */ - params.WitnessChunkReadCost + /* SLOAD in constructor */ - params.WitnessChunkWriteCost + /* SSTORE in constructor */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + params.WitnessBranchReadCost + params.WitnessBranchWriteCost + /* creation (CREATE at PC=0x121) */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* write code hash */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #0 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #1 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #2 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #3 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #4 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #5 */ - params.WitnessChunkReadCost + /* SLOAD in constructor */ - params.WitnessChunkWriteCost + /* SSTORE in constructor */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* write code hash for tx creation */ - 15*(params.WitnessChunkReadCost+params.WitnessChunkWriteCost) + /* code chunks #0..#14 */ - uint64(4844) /* execution costs */ - blockGasUsagesExpected := []uint64{ - txCost1*2 + txCost2, - txCost1*2 + txCost2 + contractCreationCost + codeWithExtCodeCopyGas, - } - _, _, chain, _, proofs, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { - gen.SetPoS() - - // TODO need to check that the tx cost provided is the exact amount used (no remaining left-over) - tx, _ := types.SignTx(types.NewTransaction(uint64(i)*3, common.Address{byte(i), 2, 3}, big.NewInt(999), txCost1, big.NewInt(875000000), nil), signer, testKey) - gen.AddTx(tx) - tx, _ = types.SignTx(types.NewTransaction(uint64(i)*3+1, common.Address{}, big.NewInt(999), txCost1, big.NewInt(875000000), nil), signer, testKey) - gen.AddTx(tx) - tx, _ = types.SignTx(types.NewTransaction(uint64(i)*3+2, common.Address{}, big.NewInt(0), txCost2, big.NewInt(875000000), nil), signer, testKey) - gen.AddTx(tx) - - // Add two contract creations in block #2 - if i == 1 { - tx, _ = types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 6, - Value: big.NewInt(16), - Gas: 3000000, - GasPrice: big.NewInt(875000000), - Data: code, - }) - gen.AddTx(tx) - - tx, _ = types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 7, - Value: big.NewInt(0), - Gas: 3000000, - GasPrice: big.NewInt(875000000), - Data: codeWithExtCodeCopy, - }) - gen.AddTx(tx) - } - }) - - // Check proof for both blocks - err := verkle.Verify(proofs[0], gspec.ToBlock().Root().Bytes(), chain[0].Root().Bytes(), statediffs[0]) - if err != nil { - t.Fatal(err) - } - err = verkle.Verify(proofs[1], chain[0].Root().Bytes(), chain[1].Root().Bytes(), statediffs[1]) - if err != nil { - t.Fatal(err) - } - - t.Log("verified verkle proof, inserting blocks into the chain") - - for i, b := range chain { - fmt.Printf("%d %x\n", i, b.Root()) - } - endnum, err := blockchain.InsertChain(chain) - if err != nil { - t.Fatalf("block %d imported with error: %v", endnum, err) - } - - for i := range 2 { - b := blockchain.GetBlockByNumber(uint64(i) + 1) - if b == nil { - t.Fatalf("expected block %d to be present in chain", i+1) - } - if b.Hash() != chain[i].Hash() { - t.Fatalf("block #%d not found at expected height", b.NumberU64()) - } - if b.GasUsed() != blockGasUsagesExpected[i] { - t.Fatalf("expected block #%d txs to use %d, got %d\n", b.NumberU64(), blockGasUsagesExpected[i], b.GasUsed()) - } - } -} - -func TestProcessParentBlockHash(t *testing.T) { - // This test uses blocks where, - // block 1 parent hash is 0x0100.... - // block 2 parent hash is 0x0200.... - // etc - checkBlockHashes := func(statedb *state.StateDB, isVerkle bool) { - statedb.SetNonce(params.HistoryStorageAddress, 1, tracing.NonceChangeUnspecified) - statedb.SetCode(params.HistoryStorageAddress, params.HistoryStorageCode, tracing.CodeChangeUnspecified) - // Process n blocks, from 1 .. num - var num = 2 - for i := 1; i <= num; i++ { - header := &types.Header{ParentHash: common.Hash{byte(i)}, Number: big.NewInt(int64(i)), Difficulty: new(big.Int)} - chainConfig := params.MergedTestChainConfig - if isVerkle { - chainConfig = testVerkleChainConfig - } - vmContext := NewEVMBlockContext(header, nil, new(common.Address)) - evm := vm.NewEVM(vmContext, statedb, chainConfig, vm.Config{}) - ProcessParentBlockHash(header.ParentHash, evm) - } - // Read block hashes for block 0 .. num-1 - for i := 0; i < num; i++ { - have, want := getContractStoredBlockHash(statedb, uint64(i), isVerkle), common.Hash{byte(i + 1)} - if have != want { - t.Errorf("block %d, verkle=%v, have parent hash %v, want %v", i, isVerkle, have, want) - } - } - } - t.Run("MPT", func(t *testing.T) { - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) - checkBlockHashes(statedb, false) - }) - t.Run("Verkle", func(t *testing.T) { - db := rawdb.NewMemoryDatabase() - cacheConfig := DefaultConfig().WithStateScheme(rawdb.PathScheme) - cacheConfig.SnapshotLimit = 0 - triedb := triedb.NewDatabase(db, cacheConfig.triedbConfig(true)) - statedb, _ := state.New(types.EmptyVerkleHash, state.NewDatabase(triedb, nil)) - checkBlockHashes(statedb, true) - }) -} - -// getContractStoredBlockHash is a utility method which reads the stored parent blockhash for block 'number' -func getContractStoredBlockHash(statedb *state.StateDB, number uint64, isVerkle bool) common.Hash { - ringIndex := number % params.HistoryServeWindow - var key common.Hash - binary.BigEndian.PutUint64(key[24:], ringIndex) - if isVerkle { - return statedb.GetState(params.HistoryStorageAddress, key) - } - return statedb.GetState(params.HistoryStorageAddress, key) -} - -// TestProcessVerkleInvalidContractCreation checks for several modes of contract creation failures -func TestProcessVerkleInvalidContractCreation(t *testing.T) { - var ( - account1 = common.HexToAddress("0x687704DB07e902e9A8B3754031D168D46E3D586e") - account2 = common.HexToAddress("0x6177843db3138ae69679A54b95cf345ED759450d") - gspec = verkleTestGenesis(testKaustinenLikeChainConfig) - ) - // slightly modify it to suit the live txs from the testnet - gspec.Alloc[account2] = types.Account{ - Balance: big.NewInt(1000000000000000000), // 1 ether - Nonce: 1, - } - - // Create two blocks that reproduce what is happening on kaustinen. - // - The first block contains two failing contract creation transactions, that - // write to storage before they revert. - // - // - The second block contains a single failing contract creation transaction, - // that fails right off the bat. - genesisH, _, chain, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { - gen.SetPoS() - - if i == 0 { - for _, rlpData := range []string{ - // SSTORE at slot 41 and reverts - "f8d48084479c2c18830186a08080b8806000602955bda3f9600060ca55600060695523b360006039551983576000601255b0620c2fde2c592ac2600060bc55e0ac6000606455a63e22600060e655eb607e605c5360a2605d5360c7605e53601d605f5360eb606053606b606153608e60625360816063536079606453601e60655360fc60665360b7606753608b60685383021e7ca0cc20c65a97d2e526b8ec0f4266e8b01bdcde43b9aeb59d8bfb44e8eb8119c109a07a8e751813ae1b2ce734960dbc39a4f954917d7822a2c5d1dca18b06c584131f", - // SSTORE at slot 133 and reverts - "02f8db83010f2c01843b9aca0084479c2c18830186a08080b88060006085553fad6000600a55600060565555600060b55506600060cf557f1b8b38183e7bd1bdfaa7123c5a4976e54cce0e42049d841411978fd3595e25c66019527f0538943712953cf08900aae40222a40b2d5a4ac8075ad8cf0870e2be307edbb96039527f9f3174ff85024747041ae7a611acffb987c513c088d90ab288aec080a0cd6ac65ce2cb0a912371f6b5a551ba8caffc22ec55ad4d3cb53de41d05eb77b6a02e0dfe8513dfa6ec7bfd7eda6f5c0dac21b39b982436045e128cec46cfd3f960", - // this one is a simple transfer that succeeds, necessary to get the correct nonce in the other block. - "f8e80184479c2c18830186a094bbbbde4ca27f83fc18aa108170547ff57675936a80b8807ff71f7c15faadb969a76a5f54a81a0117e1e743cb7f24e378eda28442ea4c6eb6604a527fb5409e5718d44e23bfffac926e5ea726067f772772e7e19446acba0c853f62f5606a526020608a536088608b536039608c536004608d5360af608e537f7f7675d9f210e0a61564e6d11e7cd75f5bc9009ac9f6b94a0fc63035441a83021e7ba04a4a172d81ebb02847829b76a387ac09749c8b65668083699abe20c887fb9efca07c5b1a990702ec7b31a5e8e3935cd9a77649f8c25a84131229e24ab61aec6093", - } { - var tx = new(types.Transaction) - if err := tx.UnmarshalBinary(common.Hex2Bytes(rlpData)); err != nil { - t.Fatal(err) - } - gen.AddTx(tx) - } - } else { - var tx = new(types.Transaction) - // immediately reverts - if err := tx.UnmarshalBinary(common.Hex2Bytes("01f8d683010f2c028443ad7d0e830186a08080b880b00e7fa3c849dce891cce5fae8a4c46cbb313d6aec0c0ffe7863e05fb7b22d4807674c6055527ffbfcb0938f3e18f7937aa8fa95d880afebd5c4cec0d85186095832d03c85cf8a60755260ab60955360cf6096536066609753606e60985360fa609953609e609a53608e609b536024609c5360f6609d536072609e5360a4609fc080a08fc6f7101f292ff1fb0de8ac69c2d320fbb23bfe61cf327173786ea5daee6e37a044c42d91838ef06646294bf4f9835588aee66243b16a66a2da37641fae4c045f")); err != nil { - t.Fatal(err) - } - gen.AddTx(tx) - } - }) - - tx1ContractAddress := crypto.CreateAddress(account1, 0) - tx1ContractStem := utils.GetTreeKey(tx1ContractAddress[:], uint256.NewInt(0), 105) - tx1ContractStem = tx1ContractStem[:31] - - tx2ContractAddress := crypto.CreateAddress(account2, 1) - tx2SlotKey := [32]byte{} - tx2SlotKey[31] = 133 - tx2ContractStem := utils.StorageSlotKey(tx2ContractAddress[:], tx2SlotKey[:]) - tx2ContractStem = tx2ContractStem[:31] - - eip2935Stem := utils.GetTreeKey(params.HistoryStorageAddress[:], uint256.NewInt(0), 0) - eip2935Stem = eip2935Stem[:31] - - // Check that the witness contains what we expect: a storage entry for each of the two contract - // creations that failed: one at 133 for the 2nd tx, and one at 105 for the first tx. - for _, stemStateDiff := range statediffs[0] { - // Check that the slot number 133, which is overflowing the account header, - // is present. Note that the offset of the 2nd group (first group after the - // header) is skipping the first 64 values, hence we still have an offset - // of 133, and not 133 - 64. - if bytes.Equal(stemStateDiff.Stem[:], tx2ContractStem[:]) { - for _, suffixDiff := range stemStateDiff.SuffixDiffs { - if suffixDiff.Suffix != 133 { - t.Fatalf("invalid suffix diff found for %x in block #1: %d\n", stemStateDiff.Stem, suffixDiff.Suffix) - } - if suffixDiff.CurrentValue != nil { - t.Fatalf("invalid prestate value found for %x in block #1: %v != nil\n", stemStateDiff.Stem, suffixDiff.CurrentValue) - } - if suffixDiff.NewValue != nil { - t.Fatalf("invalid poststate value found for %x in block #1: %v != nil\n", stemStateDiff.Stem, suffixDiff.NewValue) - } - } - } else if bytes.Equal(stemStateDiff.Stem[:], tx1ContractStem) { - // For this contract creation, check that only the account header and storage slot 41 - // are found in the witness. - for _, suffixDiff := range stemStateDiff.SuffixDiffs { - if suffixDiff.Suffix != 105 && suffixDiff.Suffix != 0 && suffixDiff.Suffix != 1 { - t.Fatalf("invalid suffix diff found for %x in block #1: %d\n", stemStateDiff.Stem, suffixDiff.Suffix) - } - } - } else if bytes.Equal(stemStateDiff.Stem[:], eip2935Stem) { - // Check the eip 2935 group of leaves. - // Check that only one leaf was accessed, and is present in the witness. - if len(stemStateDiff.SuffixDiffs) > 1 { - t.Fatalf("invalid suffix diff count found for BLOCKHASH contract: %d != 1", len(stemStateDiff.SuffixDiffs)) - } - // Check that this leaf is the first storage slot - if stemStateDiff.SuffixDiffs[0].Suffix != 64 { - t.Fatalf("invalid suffix diff value found for BLOCKHASH contract: %d != 64", stemStateDiff.SuffixDiffs[0].Suffix) - } - // check that the prestate value is nil and that the poststate value isn't. - if stemStateDiff.SuffixDiffs[0].CurrentValue != nil { - t.Fatalf("non-nil current value in BLOCKHASH contract insert: %x", stemStateDiff.SuffixDiffs[0].CurrentValue) - } - if stemStateDiff.SuffixDiffs[0].NewValue == nil { - t.Fatalf("nil new value in BLOCKHASH contract insert") - } - if *stemStateDiff.SuffixDiffs[0].NewValue != genesisH { - t.Fatalf("invalid BLOCKHASH value: %x != %x", *stemStateDiff.SuffixDiffs[0].NewValue, genesisH) - } - } else { - // For all other entries present in the witness, check that nothing beyond - // the account header was accessed. - for _, suffixDiff := range stemStateDiff.SuffixDiffs { - if suffixDiff.Suffix > 2 { - t.Fatalf("invalid suffix diff found for %x in block #1: %d\n", stemStateDiff.Stem, suffixDiff.Suffix) - } - } - } - } - - // Check that no account has a value above 4 in the 2nd block as no storage nor - // code should make it to the witness. - for _, stemStateDiff := range statediffs[1] { - for _, suffixDiff := range stemStateDiff.SuffixDiffs { - if bytes.Equal(stemStateDiff.Stem[:], eip2935Stem) { - // BLOCKHASH contract stem - if len(stemStateDiff.SuffixDiffs) > 1 { - t.Fatalf("invalid suffix diff count found for BLOCKHASH contract at block #2: %d != 1", len(stemStateDiff.SuffixDiffs)) - } - if stemStateDiff.SuffixDiffs[0].Suffix != 65 { - t.Fatalf("invalid suffix diff value found for BLOCKHASH contract at block #2: %d != 65", stemStateDiff.SuffixDiffs[0].Suffix) - } - if stemStateDiff.SuffixDiffs[0].NewValue == nil { - t.Fatalf("missing post state value for BLOCKHASH contract at block #2") - } - if *stemStateDiff.SuffixDiffs[0].NewValue != chain[0].Hash() { - t.Fatalf("invalid post state value for BLOCKHASH contract at block #2: %x != %x", chain[0].Hash(), (*stemStateDiff.SuffixDiffs[0].NewValue)[:]) - } - } else if suffixDiff.Suffix > 4 { - t.Fatalf("invalid suffix diff found for %x in block #2: %d\n", stemStateDiff.Stem, suffixDiff.Suffix) - } - } - } -} - -func verkleTestGenesis(config *params.ChainConfig) *Genesis { - var ( - coinbase = common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7") - account1 = common.HexToAddress("0x687704DB07e902e9A8B3754031D168D46E3D586e") - account2 = common.HexToAddress("0x6177843db3138ae69679A54b95cf345ED759450d") - ) - return &Genesis{ - Config: config, - Alloc: GenesisAlloc{ - coinbase: GenesisAccount{ - Balance: big.NewInt(1000000000000000000), // 1 ether - Nonce: 0, - }, - account1: GenesisAccount{ - Balance: big.NewInt(1000000000000000000), // 1 ether - Nonce: 0, - }, - account2: GenesisAccount{ - Balance: big.NewInt(1000000000000000000), // 1 ether - Nonce: 3, - }, - params.BeaconRootsAddress: {Nonce: 1, Code: params.BeaconRootsCode, Balance: common.Big0}, - params.HistoryStorageAddress: {Nonce: 1, Code: params.HistoryStorageCode, Balance: common.Big0}, - params.WithdrawalQueueAddress: {Nonce: 1, Code: params.WithdrawalQueueCode, Balance: common.Big0}, - params.ConsolidationQueueAddress: {Nonce: 1, Code: params.ConsolidationQueueCode, Balance: common.Big0}, - }, - } -} - -// TestProcessVerkleContractWithEmptyCode checks that the witness contains all valid -// entries, if the initcode returns an empty code. -func TestProcessVerkleContractWithEmptyCode(t *testing.T) { - // The test txs were taken from a secondary testnet with chain id 69421 - config := *testKaustinenLikeChainConfig - config.ChainID = new(big.Int).SetUint64(69421) - gspec := verkleTestGenesis(&config) - - genesisH, _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 1, func(i int, gen *BlockGen) { - gen.SetPoS() - var tx types.Transaction - // a transaction that does some PUSH1n but returns a 0-sized contract - txpayload := common.Hex2Bytes("02f8db83010f2d03843b9aca008444cf6a05830186a08080b8807fdfbbb59f2371a76485ce557fd0de00c298d3ede52a3eab56d35af674eb49ec5860335260826053536001605453604c60555360f3605653606060575360446058536096605953600c605a5360df605b5360f3605c5360fb605d53600c605e53609a605f53607f60605360fe606153603d60625360f4606353604b60645360cac001a0486b6dc55b8a311568b7239a2cae1d77e7446dba71df61eaafd53f73820a138fa010bd48a45e56133ac4c5645142c2ea48950d40eb35050e9510b6bad9e15c5865") - if err := tx.UnmarshalBinary(txpayload); err != nil { - t.Fatal(err) - } - gen.AddTx(&tx) - }) - - eip2935Stem := utils.GetTreeKey(params.HistoryStorageAddress[:], uint256.NewInt(0), 0) - eip2935Stem = eip2935Stem[:31] - - for _, stemStateDiff := range statediffs[0] { - // Handle the case of the history contract: make sure only the correct - // slots are added to the witness. - if bytes.Equal(stemStateDiff.Stem[:], eip2935Stem) { - // BLOCKHASH contract stem - if len(stemStateDiff.SuffixDiffs) > 1 { - t.Fatalf("invalid suffix diff count found for BLOCKHASH contract: %d != 1", len(stemStateDiff.SuffixDiffs)) - } - if stemStateDiff.SuffixDiffs[0].Suffix != 64 { - t.Fatalf("invalid suffix diff value found for BLOCKHASH contract: %d != 64", stemStateDiff.SuffixDiffs[0].Suffix) - } - // check that the "current value" is nil and that the new value isn't. - if stemStateDiff.SuffixDiffs[0].CurrentValue != nil { - t.Fatalf("non-nil current value in BLOCKHASH contract insert: %x", stemStateDiff.SuffixDiffs[0].CurrentValue) - } - if stemStateDiff.SuffixDiffs[0].NewValue == nil { - t.Fatalf("nil new value in BLOCKHASH contract insert") - } - if *stemStateDiff.SuffixDiffs[0].NewValue != genesisH { - t.Fatalf("invalid BLOCKHASH value: %x != %x", *stemStateDiff.SuffixDiffs[0].NewValue, genesisH) - } - } else { - for _, suffixDiff := range stemStateDiff.SuffixDiffs { - if suffixDiff.Suffix > 2 { - // if d8898012c484fb48610ecb7963886339207dab004bce968b007b616ffa18e0 shows up, it means that the PUSHn - // in the transaction above added entries into the witness, when they should not have since they are - // part of a contract deployment. - t.Fatalf("invalid suffix diff found for %x in block #1: %d\n", stemStateDiff.Stem, suffixDiff.Suffix) - } - } - } - } -} - -// TestProcessVerkleExtCodeHashOpcode verifies that calling EXTCODEHASH on another -// deployed contract, creates all the right entries in the witness. -func TestProcessVerkleExtCodeHashOpcode(t *testing.T) { - // The test txs were taken from a secondary testnet with chain id 69421 - config := *testKaustinenLikeChainConfig - config.ChainID = new(big.Int).SetUint64(69421) - - var ( - signer = types.LatestSigner(&config) - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - gspec = verkleTestGenesis(&config) - ) - dummyContract := []byte{ - byte(vm.PUSH1), 2, - byte(vm.PUSH1), 12, - byte(vm.PUSH1), 0x00, - byte(vm.CODECOPY), - - byte(vm.PUSH1), 2, - byte(vm.PUSH1), 0x00, - byte(vm.RETURN), - - byte(vm.PUSH1), 42, - } - deployer := crypto.PubkeyToAddress(testKey.PublicKey) - dummyContractAddr := crypto.CreateAddress(deployer, 0) - - // contract that calls EXTCODEHASH on the dummy contract - extCodeHashContract := []byte{ - byte(vm.PUSH1), 22, - byte(vm.PUSH1), 12, - byte(vm.PUSH1), 0x00, - byte(vm.CODECOPY), - - byte(vm.PUSH1), 22, - byte(vm.PUSH1), 0x00, - byte(vm.RETURN), - - byte(vm.PUSH20), - 0x3a, 0x22, 0x0f, 0x35, 0x12, 0x52, 0x08, 0x9d, 0x38, 0x5b, 0x29, 0xbe, 0xca, 0x14, 0xe2, 0x7f, 0x20, 0x4c, 0x29, 0x6a, - byte(vm.EXTCODEHASH), - } - extCodeHashContractAddr := crypto.CreateAddress(deployer, 1) - - _, _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { - gen.SetPoS() - - if i == 0 { - // Create dummy contract. - tx, _ := types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 0, - Value: big.NewInt(0), - Gas: 100_000, - GasPrice: big.NewInt(875000000), - Data: dummyContract, - }) - gen.AddTx(tx) - - // Create contract with EXTCODEHASH opcode. - tx, _ = types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 1, - Value: big.NewInt(0), - Gas: 100_000, - GasPrice: big.NewInt(875000000), - Data: extCodeHashContract}) - gen.AddTx(tx) - } else { - tx, _ := types.SignTx(types.NewTransaction(2, extCodeHashContractAddr, big.NewInt(0), 100_000, big.NewInt(875000000), nil), signer, testKey) - gen.AddTx(tx) - } - }) - - contractKeccakTreeKey := utils.CodeHashKey(dummyContractAddr[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range statediffs[1] { - if bytes.Equal(stemStateDiff.Stem[:], contractKeccakTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatalf("no state diff found for stem") - } - - codeHashStateDiff := statediffs[1][stateDiffIdx].SuffixDiffs[0] - // Check location of code hash was accessed - if codeHashStateDiff.Suffix != utils.CodeHashLeafKey { - t.Fatalf("code hash invalid suffix") - } - // check the code hash wasn't present in the prestate, as - // the contract was deployed in this block. - if codeHashStateDiff.CurrentValue == nil { - t.Fatalf("codeHash.CurrentValue must not be empty") - } - // check the poststate value corresponds to the code hash - // of the deployed contract. - expCodeHash := crypto.Keccak256Hash(dummyContract[12:]) - if *codeHashStateDiff.CurrentValue != expCodeHash { - t.Fatalf("codeHash.CurrentValue unexpected code hash") - } - if codeHashStateDiff.NewValue != nil { - t.Fatalf("codeHash.NewValue must be nil") - } -} - -// TestProcessVerkleBalanceOpcode checks that calling balance -// on another contract will add the correct entries to the witness. -func TestProcessVerkleBalanceOpcode(t *testing.T) { - // The test txs were taken from a secondary testnet with chain id 69421 - config := *testKaustinenLikeChainConfig - config.ChainID = new(big.Int).SetUint64(69421) - - var ( - signer = types.LatestSigner(&config) - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - account2 = common.HexToAddress("0x6177843db3138ae69679A54b95cf345ED759450d") - gspec = verkleTestGenesis(&config) - ) - _, _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 1, func(i int, gen *BlockGen) { - gen.SetPoS() - txData := slices.Concat( - []byte{byte(vm.PUSH20)}, - common.HexToAddress("0x6177843db3138ae69679A54b95cf345ED759450d").Bytes(), - []byte{byte(vm.BALANCE)}) - - tx, _ := types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 0, - Value: big.NewInt(0), - Gas: 100_000, - GasPrice: big.NewInt(875000000), - Data: txData}) - gen.AddTx(tx) - }) - - account2BalanceTreeKey := utils.BasicDataKey(account2[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range statediffs[0] { - if bytes.Equal(stemStateDiff.Stem[:], account2BalanceTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatalf("no state diff found for stem") - } - - var zero [32]byte - balanceStateDiff := statediffs[0][stateDiffIdx].SuffixDiffs[0] - if balanceStateDiff.Suffix != utils.BasicDataLeafKey { - t.Fatalf("invalid suffix diff") - } - // check the prestate balance wasn't 0 or missing - if balanceStateDiff.CurrentValue == nil || *balanceStateDiff.CurrentValue == zero { - t.Fatalf("invalid current value %v", *balanceStateDiff.CurrentValue) - } - // check that the poststate witness value for the balance is nil, - // meaning that it didn't get updated. - if balanceStateDiff.NewValue != nil { - t.Fatalf("invalid new value") - } -} - -// TestProcessVerkleSelfDestructInSeparateTx controls the contents of the witness after -// a non-eip6780-compliant selfdestruct occurs. -func TestProcessVerkleSelfDestructInSeparateTx(t *testing.T) { - // The test txs were taken from a secondary testnet with chain id 69421 - config := *testKaustinenLikeChainConfig - config.ChainID = new(big.Int).SetUint64(69421) - - var ( - signer = types.LatestSigner(&config) - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - account2 = common.HexToAddress("0x6177843db3138ae69679A54b95cf345ED759450d") - gspec = verkleTestGenesis(&config) - ) - - // runtime code: selfdestruct ( 0x6177843db3138ae69679A54b95cf345ED759450d ) - runtimeCode := slices.Concat( - []byte{byte(vm.PUSH20)}, - account2.Bytes(), - []byte{byte(vm.SELFDESTRUCT)}) - - //The goal of this test is to test SELFDESTRUCT that happens in a contract - // execution which is created in a previous transaction. - selfDestructContract := slices.Concat([]byte{ - byte(vm.PUSH1), byte(len(runtimeCode)), - byte(vm.PUSH1), 12, - byte(vm.PUSH1), 0x00, - byte(vm.CODECOPY), // Codecopy( to-offset: 0, code offset: 12, length: 22 ) - - byte(vm.PUSH1), byte(len(runtimeCode)), - byte(vm.PUSH1), 0x00, - byte(vm.RETURN), // Return ( 0 : len(runtimecode) - }, - runtimeCode) - - deployer := crypto.PubkeyToAddress(testKey.PublicKey) - contract := crypto.CreateAddress(deployer, 0) - - _, _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { - gen.SetPoS() - - if i == 0 { - // Create selfdestruct contract, sending 42 wei. - tx, _ := types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 0, - Value: big.NewInt(42), - Gas: 100_000, - GasPrice: big.NewInt(875000000), - Data: selfDestructContract, - }) - gen.AddTx(tx) - } else { - // Call it. - tx, _ := types.SignTx(types.NewTransaction(1, contract, big.NewInt(0), 100_000, big.NewInt(875000000), nil), signer, testKey) - gen.AddTx(tx) - } - }) - - var zero [32]byte - { // Check self-destructed contract in the witness - selfDestructContractTreeKey := utils.CodeHashKey(contract[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range statediffs[1] { - if bytes.Equal(stemStateDiff.Stem[:], selfDestructContractTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatalf("no state diff found for stem") - } - - balanceStateDiff := statediffs[1][stateDiffIdx].SuffixDiffs[0] - if balanceStateDiff.Suffix != utils.BasicDataLeafKey { - t.Fatalf("balance invalid suffix") - } - - // The original balance was 42. - var oldBalance [16]byte - oldBalance[15] = 42 - if !bytes.Equal((*balanceStateDiff.CurrentValue)[utils.BasicDataBalanceOffset:], oldBalance[:]) { - t.Fatalf("the pre-state balance before self-destruct must be %x, got %x", oldBalance, *balanceStateDiff.CurrentValue) - } - - // The new balance must be 0. - if !bytes.Equal((*balanceStateDiff.NewValue)[utils.BasicDataBalanceOffset:], zero[utils.BasicDataBalanceOffset:]) { - t.Fatalf("the post-state balance after self-destruct must be 0") - } - } - { // Check self-destructed target in the witness. - selfDestructTargetTreeKey := utils.CodeHashKey(account2[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range statediffs[1] { - if bytes.Equal(stemStateDiff.Stem[:], selfDestructTargetTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatalf("no state diff found for stem") - } - - balanceStateDiff := statediffs[1][stateDiffIdx].SuffixDiffs[0] - if balanceStateDiff.Suffix != utils.BasicDataLeafKey { - t.Fatalf("balance invalid suffix") - } - if balanceStateDiff.CurrentValue == nil { - t.Fatalf("codeHash.CurrentValue must not be empty") - } - if balanceStateDiff.NewValue == nil { - t.Fatalf("codeHash.NewValue must not be empty") - } - preStateBalance := binary.BigEndian.Uint64(balanceStateDiff.CurrentValue[utils.BasicDataBalanceOffset+8:]) - postStateBalance := binary.BigEndian.Uint64(balanceStateDiff.NewValue[utils.BasicDataBalanceOffset+8:]) - if postStateBalance-preStateBalance != 42 { - t.Fatalf("the post-state balance after self-destruct must be 42, got %d-%d=%d", postStateBalance, preStateBalance, postStateBalance-preStateBalance) - } - } -} - -// TestProcessVerkleSelfDestructInSameTx controls the contents of the witness after -// a eip6780-compliant selfdestruct occurs. -func TestProcessVerkleSelfDestructInSameTx(t *testing.T) { - // The test txs were taken from a secondary testnet with chain id 69421 - config := *testKaustinenLikeChainConfig - config.ChainID = new(big.Int).SetUint64(69421) - - var ( - signer = types.LatestSigner(&config) - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - account2 = common.HexToAddress("0x6177843db3138ae69679A54b95cf345ED759450d") - gspec = verkleTestGenesis(&config) - ) - - // The goal of this test is to test SELFDESTRUCT that happens in a contract - // execution which is created in **the same** transaction sending the remaining - // balance to an external (i.e: not itself) account. - - selfDestructContract := slices.Concat( - []byte{byte(vm.PUSH20)}, - account2.Bytes(), - []byte{byte(vm.SELFDESTRUCT)}) - deployer := crypto.PubkeyToAddress(testKey.PublicKey) - contract := crypto.CreateAddress(deployer, 0) - - _, _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 1, func(i int, gen *BlockGen) { - gen.SetPoS() - tx, _ := types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 0, - Value: big.NewInt(42), - Gas: 100_000, - GasPrice: big.NewInt(875000000), - Data: selfDestructContract, - }) - gen.AddTx(tx) - }) - - { // Check self-destructed contract in the witness - selfDestructContractTreeKey := utils.CodeHashKey(contract[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range statediffs[0] { - if bytes.Equal(stemStateDiff.Stem[:], selfDestructContractTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatalf("no state diff found for stem") - } - - balanceStateDiff := statediffs[0][stateDiffIdx].SuffixDiffs[0] - if balanceStateDiff.Suffix != utils.BasicDataLeafKey { - t.Fatalf("balance invalid suffix") - } - - if balanceStateDiff.CurrentValue != nil { - t.Fatalf("the pre-state balance before must be nil, since the contract didn't exist") - } - - if balanceStateDiff.NewValue != nil { - t.Fatalf("the post-state balance after self-destruct must be nil since the contract shouldn't be created at all") - } - } - { // Check self-destructed target in the witness. - selfDestructTargetTreeKey := utils.CodeHashKey(account2[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range statediffs[0] { - if bytes.Equal(stemStateDiff.Stem[:], selfDestructTargetTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatalf("no state diff found for stem") - } - - balanceStateDiff := statediffs[0][stateDiffIdx].SuffixDiffs[0] - if balanceStateDiff.Suffix != utils.BasicDataLeafKey { - t.Fatalf("balance invalid suffix") - } - if balanceStateDiff.CurrentValue == nil { - t.Fatalf("codeHash.CurrentValue must not be empty") - } - if balanceStateDiff.NewValue == nil { - t.Fatalf("codeHash.NewValue must not be empty") - } - preStateBalance := binary.BigEndian.Uint64(balanceStateDiff.CurrentValue[utils.BasicDataBalanceOffset+8:]) - postStateBalance := binary.BigEndian.Uint64(balanceStateDiff.NewValue[utils.BasicDataBalanceOffset+8:]) - if postStateBalance-preStateBalance != 42 { - t.Fatalf("the post-state balance after self-destruct must be 42. got %d", postStateBalance) - } - } -} - -// TestProcessVerkleSelfDestructInSeparateTxWithSelfBeneficiary checks the content of the witness -// if a selfdestruct occurs in a different tx than the one that created it, but the beneficiary -// is the selfdestructed account. -func TestProcessVerkleSelfDestructInSeparateTxWithSelfBeneficiary(t *testing.T) { - // The test txs were taken from a secondary testnet with chain id 69421 - config := *testKaustinenLikeChainConfig - config.ChainID = new(big.Int).SetUint64(69421) - - var ( - signer = types.LatestSigner(&config) - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - gspec = verkleTestGenesis(&config) - ) - // The goal of this test is to test SELFDESTRUCT that happens in a contract - // execution which is created in a *previous* transaction sending the remaining - // balance to itself. - selfDestructContract := []byte{ - byte(vm.PUSH1), 2, // PUSH1 2 - byte(vm.PUSH1), 10, // PUSH1 12 - byte(vm.PUSH0), // PUSH0 - byte(vm.CODECOPY), // Codecopy ( to offset 0, code@offset: 10, length: 2) - - byte(vm.PUSH1), 22, - byte(vm.PUSH0), - byte(vm.RETURN), // RETURN( memory[0:2] ) - - // Deployed code - byte(vm.ADDRESS), - byte(vm.SELFDESTRUCT), - } - deployer := crypto.PubkeyToAddress(testKey.PublicKey) - contract := crypto.CreateAddress(deployer, 0) - - _, _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { - gen.SetPoS() - if i == 0 { - // Create self-destruct contract, sending 42 wei. - tx, _ := types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 0, - Value: big.NewInt(42), - Gas: 100_000, - GasPrice: big.NewInt(875000000), - Data: selfDestructContract, - }) - gen.AddTx(tx) - } else { - // Call it. - tx, _ := types.SignTx(types.NewTransaction(1, contract, big.NewInt(0), 100_000, big.NewInt(875000000), nil), signer, testKey) - gen.AddTx(tx) - } - }) - - { - // Check self-destructed contract in the witness. - // The way 6780 is implemented today, it always SubBalance from the self-destructed contract, and AddBalance - // to the beneficiary. In this case both addresses are the same, thus this might be optimizable from a gas - // perspective. But until that happens, we need to honor this "balance reading" adding it to the witness. - - selfDestructContractTreeKey := utils.CodeHashKey(contract[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range statediffs[1] { - if bytes.Equal(stemStateDiff.Stem[:], selfDestructContractTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatal("no state diff found for stem") - } - - balanceStateDiff := statediffs[1][stateDiffIdx].SuffixDiffs[0] - if balanceStateDiff.Suffix != utils.BasicDataLeafKey { - t.Fatal("balance invalid suffix") - } - - // The original balance was 42. - var oldBalance [16]byte - oldBalance[15] = 42 - if !bytes.Equal((*balanceStateDiff.CurrentValue)[utils.BasicDataBalanceOffset:], oldBalance[:]) { - t.Fatal("the pre-state balance before self-destruct must be 42") - } - - // Note that the SubBalance+AddBalance net effect is a 0 change, so NewValue - // must be nil. - if balanceStateDiff.NewValue != nil { - t.Fatal("the post-state balance after self-destruct must be empty") - } - } -} - -// TestProcessVerkleSelfDestructInSameTxWithSelfBeneficiary checks the content of the witness -// if a selfdestruct occurs in the same tx as the one that created it, but the beneficiary -// is the selfdestructed account. -func TestProcessVerkleSelfDestructInSameTxWithSelfBeneficiary(t *testing.T) { - // The test txs were taken from a secondary testnet with chain id 69421 - config := *testKaustinenLikeChainConfig - config.ChainID = new(big.Int).SetUint64(69421) - - var ( - signer = types.LatestSigner(&config) - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - gspec = verkleTestGenesis(&config) - deployer = crypto.PubkeyToAddress(testKey.PublicKey) - contract = crypto.CreateAddress(deployer, 0) - ) - - // The goal of this test is to test SELFDESTRUCT that happens while executing - // the init code of a contract creation, that occurs in **the same** transaction. - // The balance is sent to itself. - t.Logf("Contract: %v", contract.String()) - - selfDestructContract := []byte{byte(vm.ADDRESS), byte(vm.SELFDESTRUCT)} - - _, _, _, _, _, stateDiffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 1, func(i int, gen *BlockGen) { - gen.SetPoS() - tx, _ := types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 0, - Value: big.NewInt(42), - Gas: 100_000, - GasPrice: big.NewInt(875000000), - Data: selfDestructContract, - }) - gen.AddTx(tx) - }) - stateDiff := stateDiffs[0] // state difference of block 1 - - { // Check self-destructed contract in the witness - selfDestructContractTreeKey := utils.CodeHashKey(contract[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range stateDiff { - if bytes.Equal(stemStateDiff.Stem[:], selfDestructContractTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatal("no state diff found for stem") - } - balanceStateDiff := stateDiff[stateDiffIdx].SuffixDiffs[0] - if balanceStateDiff.Suffix != utils.BasicDataLeafKey { - t.Fatal("balance invalid suffix") - } - if balanceStateDiff.CurrentValue != nil { - t.Fatal("the pre-state balance before must be nil, since the contract didn't exist") - } - // Ensure that the value is burnt, and therefore that the balance of the self-destructed - // contract isn't modified (it should remain missing from the state) - if balanceStateDiff.NewValue != nil { - t.Fatal("the post-state balance after self-destruct must be nil since the contract shouldn't be created at all") - } - } -} - -// TestProcessVerkleSelfDestructInSameTxWithSelfBeneficiaryAndPrefundedAccount checks the -// content of the witness if a selfdestruct occurs in the same tx as the one that created it, -// it, but the beneficiary is the selfdestructed account. The difference with the test above, -// is that the created account is prefunded and so the final value should be 0. -func TestProcessVerkleSelfDestructInSameTxWithSelfBeneficiaryAndPrefundedAccount(t *testing.T) { - // The test txs were taken from a secondary testnet with chain id 69421 - config := *testKaustinenLikeChainConfig - config.ChainID = new(big.Int).SetUint64(69421) - - var ( - signer = types.LatestSigner(&config) - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - gspec = verkleTestGenesis(&config) - deployer = crypto.PubkeyToAddress(testKey.PublicKey) - contract = crypto.CreateAddress(deployer, 0) - ) - // Prefund the account, at an address that the contract will be deployed at, - // before it selfdestrucs. We can therefore check that the account itseld is - // NOT destroyed, which is what the current version of the spec requires. - // TODO(gballet) revisit after the spec has been modified. - gspec.Alloc[contract] = types.Account{ - Balance: big.NewInt(100), - } - - selfDestructContract := []byte{byte(vm.ADDRESS), byte(vm.SELFDESTRUCT)} - - _, _, _, _, _, stateDiffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 1, func(i int, gen *BlockGen) { - gen.SetPoS() - tx, _ := types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 0, - Value: big.NewInt(42), - Gas: 100_000, - GasPrice: big.NewInt(875000000), - Data: selfDestructContract, - }) - gen.AddTx(tx) - }) - stateDiff := stateDiffs[0] // state difference of block 1 - - { // Check self-destructed contract in the witness - selfDestructContractTreeKey := utils.CodeHashKey(contract[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range stateDiff { - if bytes.Equal(stemStateDiff.Stem[:], selfDestructContractTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatal("no state diff found for stem") - } - balanceStateDiff := stateDiff[stateDiffIdx].SuffixDiffs[0] - if balanceStateDiff.Suffix != utils.BasicDataLeafKey { - t.Fatal("balance invalid suffix") - } - expected, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000064") - if balanceStateDiff.CurrentValue == nil || !bytes.Equal(balanceStateDiff.CurrentValue[:], expected) { - t.Fatalf("incorrect prestate balance: %x != %x", *balanceStateDiff.CurrentValue, expected) - } - // Ensure that the value is burnt, and therefore that the balance of the self-destructed - // contract isn't modified (it should remain missing from the state) - expected = make([]byte, 32) - if balanceStateDiff.NewValue == nil { - t.Fatal("incorrect nil poststate balance") - } - if !bytes.Equal(balanceStateDiff.NewValue[:], expected[:]) { - t.Fatalf("incorrect poststate balance: %x != %x", *balanceStateDiff.NewValue, expected[:]) - } - } -} diff --git a/core/vm/common.go b/core/vm/common.go index 2990f58972..88c193c006 100644 --- a/core/vm/common.go +++ b/core/vm/common.go @@ -17,12 +17,40 @@ package vm import ( + "fmt" "math" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/params" "github.com/holiman/uint256" ) +// CheckMaxInitCodeSize checks the size of contract initcode against the +// per-chain limit set on rules. Arbitrum chains may override the upstream +// Amsterdam/Shanghai default via ArbitrumChainParams.MaxInitCodeSize. +func CheckMaxInitCodeSize(rules *params.Rules, size uint64) error { + if !rules.IsArbitrum && !rules.IsAmsterdam && !rules.IsShanghai { + return nil + } + if size > rules.MaxInitCodeSize { + return fmt.Errorf("%w: code size %v limit %v", ErrMaxInitCodeSizeExceeded, size, rules.MaxInitCodeSize) + } + return nil +} + +// CheckMaxCodeSize checks the size of contract code against the per-chain +// limit set on rules. Arbitrum chains may override the upstream EIP-158/ +// Amsterdam default via ArbitrumChainParams.MaxCodeSize. +func CheckMaxCodeSize(rules *params.Rules, size uint64) error { + if !rules.IsArbitrum && !rules.IsEIP158 { + return nil + } + if size > rules.MaxCodeSize { + return fmt.Errorf("%w: code size %v limit %v", ErrMaxCodeSizeExceeded, size, rules.MaxCodeSize) + } + return nil +} + // calcMemSize64 calculates the required memory size, and returns // the size and whether the result overflowed uint64 func calcMemSize64(off, l *uint256.Int) (uint64, bool) { diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 7163083e96..ad9cd2f3b1 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -294,7 +294,7 @@ type arbosAwarePrecompile interface { // - the returned bytes, // - the _remaining_ gas, // - any error that occurred -func RunPrecompiledContract(p PrecompiledContract, input []byte, suppliedGas uint64, logger *tracing.Hooks, advancedInfo *AdvancedPrecompileCall) (ret []byte, remainingGas uint64, usedMultiGas multigas.MultiGas, err error) { +func RunPrecompiledContract(stateDB StateDB, p PrecompiledContract, address common.Address, input []byte, suppliedGas uint64, logger *tracing.Hooks, advancedInfo *AdvancedPrecompileCall) (ret []byte, remainingGas uint64, usedMultiGas multigas.MultiGas, err error) { advanced, isAdvanced := p.(AdvancedPrecompile) if isAdvanced { return advanced.RunAdvanced(input, suppliedGas, advancedInfo) @@ -311,6 +311,12 @@ func RunPrecompiledContract(p PrecompiledContract, input []byte, suppliedGas uin logger.OnGasChange(suppliedGas, suppliedGas-gasCost, tracing.GasChangeCallPrecompiledContract) } suppliedGas -= gasCost + + // Touch the precompile for block-level accessList recording once Amsterdam + // fork is activated. + if stateDB != nil { + stateDB.Exist(address) + } output, err := p.Run(input) return output, suppliedGas, multigas.ComputationGas(gasCost), err } @@ -339,11 +345,11 @@ func (c *ecrecover) Run(input []byte) ([]byte, error) { } // We must make sure not to modify the 'input', so placing the 'v' along with // the signature needs to be done on a new allocation - sig := make([]byte, 65) - copy(sig, input[64:128]) + var sig [65]byte + copy(sig[:], input[64:128]) sig[64] = v // v needs to be at the end for libsecp256k1 - pubKey, err := crypto.Ecrecover(input[:32], sig) + pubKey, err := crypto.Ecrecover(input[:32], sig[:]) // make sure the public key is a valid one if err != nil { return nil, nil @@ -631,7 +637,6 @@ func (c *bigModExp) RequiredGas(input []byte) uint64 { if expLen > 32 { expHead.SetBytes(getData(input, baseLen, 32)) } else { - // TODO: Check that if expLen < baseLen, then getData will return an empty slice expHead.SetBytes(getData(input, baseLen, expLen)) } } diff --git a/core/vm/contracts_fuzz_test.go b/core/vm/contracts_fuzz_test.go index 7e3851cae5..d3d1c380de 100644 --- a/core/vm/contracts_fuzz_test.go +++ b/core/vm/contracts_fuzz_test.go @@ -36,7 +36,7 @@ func FuzzPrecompiledContracts(f *testing.F) { return } inWant := string(input) - RunPrecompiledContract(p, input, gas, nil, nil) + RunPrecompiledContract(nil, p, a, input, gas, nil, nil) if inHave := string(input); inWant != inHave { t.Errorf("Precompiled %v modified input data", a) } diff --git a/core/vm/contracts_multigas_test.go b/core/vm/contracts_multigas_test.go index 9b7627f78c..e5b981ab2d 100644 --- a/core/vm/contracts_multigas_test.go +++ b/core/vm/contracts_multigas_test.go @@ -20,6 +20,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/arbitrum/multigas" + "github.com/ethereum/go-ethereum/common" ) func TestPrecompiledMultigas(t *testing.T) { @@ -45,7 +46,7 @@ func TestPrecompiledMultigas(t *testing.T) { } for _, test := range cases { t.Run(test.name, func(t *testing.T) { - _, remainingGas, usedMultiGas, err := RunPrecompiledContract(p, []byte(input), test.supplied, nil, nil) + _, remainingGas, usedMultiGas, err := RunPrecompiledContract(nil, p, common.Address{}, []byte(input), test.supplied, nil, nil) if (err == nil) == test.wantErr { t.Fatalf("wrong run precompile result: wantErr=%v, got %v", test.wantErr, err) } diff --git a/core/vm/contracts_test.go b/core/vm/contracts_test.go index d9ee509f1f..2228efd0dc 100644 --- a/core/vm/contracts_test.go +++ b/core/vm/contracts_test.go @@ -101,7 +101,7 @@ func testPrecompiled(addr string, test precompiledTest, t *testing.T) { in := common.Hex2Bytes(test.Input) gas := p.RequiredGas(in) t.Run(fmt.Sprintf("%s-Gas=%d", test.Name, gas), func(t *testing.T) { - if res, _, _, err := RunPrecompiledContract(p, in, gas, nil, nil); err != nil { + if res, _, _, err := RunPrecompiledContract(nil, p, common.HexToAddress(addr), in, gas, nil, nil); err != nil { t.Error(err) } else if common.Bytes2Hex(res) != test.Expected { t.Errorf("Expected %v, got %v", test.Expected, common.Bytes2Hex(res)) @@ -123,7 +123,7 @@ func testPrecompiledOOG(addr string, test precompiledTest, t *testing.T) { gas := test.Gas - 1 t.Run(fmt.Sprintf("%s-Gas=%d", test.Name, gas), func(t *testing.T) { - _, _, _, err := RunPrecompiledContract(p, in, gas, nil, nil) + _, _, _, err := RunPrecompiledContract(nil, p, common.HexToAddress(addr), in, gas, nil, nil) if err.Error() != "out of gas" { t.Errorf("Expected error [out of gas], got [%v]", err) } @@ -140,7 +140,7 @@ func testPrecompiledFailure(addr string, test precompiledFailureTest, t *testing in := common.Hex2Bytes(test.Input) gas := p.RequiredGas(in) t.Run(test.Name, func(t *testing.T) { - _, _, _, err := RunPrecompiledContract(p, in, gas, nil, nil) + _, _, _, err := RunPrecompiledContract(nil, p, common.HexToAddress(addr), in, gas, nil, nil) if err.Error() != test.ExpectedError { t.Errorf("Expected error [%v], got [%v]", test.ExpectedError, err) } @@ -171,7 +171,7 @@ func benchmarkPrecompiled(addr string, test precompiledTest, bench *testing.B) { start := time.Now() for bench.Loop() { copy(data, in) - res, _, _, err = RunPrecompiledContract(p, data, reqGas, nil, nil) + res, _, _, err = RunPrecompiledContract(nil, p, common.HexToAddress(addr), data, reqGas, nil, nil) } elapsed := uint64(time.Since(start)) if elapsed < 1 { diff --git a/core/vm/eips.go b/core/vm/eips.go index 18bdbfdf97..5090225535 100644 --- a/core/vm/eips.go +++ b/core/vm/eips.go @@ -44,6 +44,8 @@ var activators = map[int]func(*JumpTable){ 4762: enable4762, 7702: enable7702, 7939: enable7939, + 8024: enable8024, + 7843: enable7843, } // EnableEIP enables the given EIP on the config. @@ -347,6 +349,28 @@ func enable6780(jt *JumpTable) { } } +// enable8024 applies EIP-8024 (DUPN, SWAPN, EXCHANGE) +func enable8024(jt *JumpTable) { + jt[DUPN] = &operation{ + execute: opDupN, + constantGas: GasFastestStep, + minStack: minStack(1, 0), + maxStack: maxStack(0, 1), + } + jt[SWAPN] = &operation{ + execute: opSwapN, + constantGas: GasFastestStep, + minStack: minStack(2, 0), + maxStack: maxStack(0, 0), + } + jt[EXCHANGE] = &operation{ + execute: opExchange, + constantGas: GasFastestStep, + minStack: minStack(2, 0), + maxStack: maxStack(0, 0), + } +} + func opExtCodeCopyEIP4762(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { var ( stack = scope.Stack @@ -561,3 +585,19 @@ func enable7702(jt *JumpTable) { jt[STATICCALL].dynamicGas = gasStaticCallEIP7702 jt[DELEGATECALL].dynamicGas = gasDelegateCallEIP7702 } + +// opSlotNum enables the SLOTNUM opcode +func opSlotNum(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { + scope.Stack.push(uint256.NewInt(evm.Context.SlotNum)) + return nil, nil +} + +// enable7843 enables the SLOTNUM opcode as specified in EIP-7843. +func enable7843(jt *JumpTable) { + jt[SLOTNUM] = &operation{ + execute: opSlotNum, + constantGas: GasQuickStep, + minStack: minStack(0, 1), + maxStack: maxStack(0, 1), + } +} diff --git a/core/vm/evm.go b/core/vm/evm.go index b49f0ac287..600de7f7ef 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -37,7 +37,7 @@ type ( // CanTransferFunc is the signature of a transfer guard function CanTransferFunc func(StateDB, common.Address, *uint256.Int) bool // TransferFunc is the signature of a transfer function - TransferFunc func(StateDB, common.Address, common.Address, *uint256.Int) + TransferFunc func(StateDB, common.Address, common.Address, *uint256.Int, *params.Rules) // GetHashFunc returns the n'th block hash in the blockchain // and is used by the BLOCKHASH EVM op code. GetHashFunc func(uint64) common.Hash @@ -68,6 +68,7 @@ type BlockContext struct { BaseFee *big.Int // Provides information for BASEFEE (0 if vm runs with NoBaseFee flag and 0 gas price) BlobBaseFee *big.Int // Provides information for BLOBBASEFEE (0 if vm runs with NoBaseFee flag and 0 blob gas price) Random *common.Hash // Provides information for PREVRANDAO + SlotNum uint64 // Provides information for SLOTNUM // Arbitrum information ArbOSVersion uint64 @@ -79,9 +80,8 @@ type BlockContext struct { type TxContext struct { // Message information Origin common.Address // Provides information for ORIGIN - GasPrice *big.Int // Provides information for GASPRICE (and is used to zero the basefee if NoBaseFee is set) + GasPrice *uint256.Int // Provides information for GASPRICE (and is used to zero the basefee if NoBaseFee is set) BlobHashes []common.Hash // Provides information for BLOBHASH - BlobFeeCap *big.Int // Is used to zero the blobbasefee if NoBaseFee is set AccessEvents *state.AccessEvents // Capture all state accesses for this tx } @@ -133,9 +133,6 @@ type EVM struct { // jumpDests stores results of JUMPDEST analysis. jumpDests JumpDestCache - hasher crypto.KeccakState // Keccak256 hasher instance shared across opcodes - hasherBuf common.Hash // Keccak256 hasher result array shared across opcodes - readOnly bool // Whether to throw on stateful modifications returnData []byte // Last CALL's return data for subsequent reuse } @@ -152,12 +149,13 @@ func NewEVM(blockCtx BlockContext, statedb StateDB, chainConfig *params.ChainCon chainConfig: chainConfig, chainRules: chainConfig.Rules(blockCtx.BlockNumber, blockCtx.Random != nil, blockCtx.Time, blockCtx.ArbOSVersion), jumpDests: newMapJumpDests(), - hasher: crypto.NewKeccakState(), } evm.ProcessingHook = DefaultTxProcessor{evm: evm} evm.precompiles = activePrecompiledContracts(evm.chainRules) switch { + case evm.chainRules.IsAmsterdam: + evm.table = &amsterdamInstructionSet case evm.chainRules.IsOsaka: evm.table = &osakaInstructionSet case evm.chainRules.IsVerkle: @@ -207,6 +205,8 @@ func NewEVM(blockCtx BlockContext, statedb StateDB, chainConfig *params.ChainCon return evm } +func (evm *EVM) ChainRules() *params.Rules { return &evm.chainRules } + // SetPrecompiles sets the precompiled contracts for the EVM. // This method is only used through RPC calls. // It is not thread-safe. @@ -223,7 +223,7 @@ func (evm *EVM) SetJumpDestCache(jumpDests JumpDestCache) { // This is not threadsafe and should only be done very cautiously. func (evm *EVM) SetTxContext(txCtx TxContext) { if evm.chainRules.IsEIP4762 { - txCtx.AccessEvents = state.NewAccessEvents(evm.StateDB.PointCache()) + txCtx.AccessEvents = state.NewAccessEvents() } evm.TxContext = txCtx } @@ -259,8 +259,10 @@ func (evm *EVM) Call(caller common.Address, addr common.Address, input []byte, g if evm.depth > int(params.CallCreateDepth) { return nil, gas, multigas.ZeroGas(), ErrDepth } - // Fail if we're trying to transfer more than the available balance - if !value.IsZero() && !evm.Context.CanTransfer(evm.StateDB, caller, value) { + syscall := isSystemCall(caller) + + // Fail if we're trying to transfer more than the available balance. + if !syscall && !value.IsZero() && !evm.Context.CanTransfer(evm.StateDB, caller, value) { return nil, gas, multigas.ZeroGas(), ErrInsufficientBalance } snapshot := evm.StateDB.Snapshot() @@ -290,9 +292,18 @@ func (evm *EVM) Call(caller common.Address, addr common.Address, input []byte, g } evm.StateDB.CreateAccount(addr) } - evm.Context.Transfer(evm.StateDB, caller, addr, value) + // Perform the value transfer only in non-syscall mode. + // Calling this is required even for zero-value transfers, + // to ensure the state clearing mechanism is applied. + if !syscall { + evm.Context.Transfer(evm.StateDB, caller, addr, value, &evm.chainRules) + } if isPrecompile { + var stateDB StateDB + if evm.chainRules.IsAmsterdam { + stateDB = evm.StateDB + } info := &AdvancedPrecompileCall{ PrecompileAddress: addr, ActingAsAddress: addr, @@ -302,7 +313,7 @@ func (evm *EVM) Call(caller common.Address, addr common.Address, input []byte, g Evm: evm, } var precompileMultiGas multigas.MultiGas - ret, gas, precompileMultiGas, err = RunPrecompiledContract(p, input, gas, evm.Config.Tracer, info) + ret, gas, precompileMultiGas, err = RunPrecompiledContract(stateDB, p, addr, input, gas, evm.Config.Tracer, info) usedMultiGas.SaturatingAddInto(precompileMultiGas) } else { // Initialise a new contract and set the code that is to be used by the EVM. @@ -372,6 +383,10 @@ func (evm *EVM) CallCode(caller common.Address, addr common.Address, input []byt // It is allowed to call precompiles, even via delegatecall if p, isPrecompile := evm.precompile(addr); isPrecompile { + var stateDB StateDB + if evm.chainRules.IsAmsterdam { + stateDB = evm.StateDB + } info := &AdvancedPrecompileCall{ PrecompileAddress: addr, ActingAsAddress: caller, @@ -381,7 +396,7 @@ func (evm *EVM) CallCode(caller common.Address, addr common.Address, input []byt Evm: evm, } var precompileMultiGas multigas.MultiGas - ret, gas, precompileMultiGas, err = RunPrecompiledContract(p, input, gas, evm.Config.Tracer, info) + ret, gas, precompileMultiGas, err = RunPrecompiledContract(stateDB, p, addr, input, gas, evm.Config.Tracer, info) usedMultiGas.SaturatingAddInto(precompileMultiGas) } else { // Initialise a new contract and set the code that is to be used by the EVM. @@ -434,6 +449,10 @@ func (evm *EVM) DelegateCall(originCaller common.Address, caller common.Address, // It is allowed to call precompiles, even via delegatecall if p, isPrecompile := evm.precompile(addr); isPrecompile { + var stateDB StateDB + if evm.chainRules.IsAmsterdam { + stateDB = evm.StateDB + } info := &AdvancedPrecompileCall{ PrecompileAddress: addr, ActingAsAddress: caller, @@ -443,7 +462,7 @@ func (evm *EVM) DelegateCall(originCaller common.Address, caller common.Address, Evm: evm, } var precompileMultiGas multigas.MultiGas - ret, gas, precompileMultiGas, err = RunPrecompiledContract(p, input, gas, evm.Config.Tracer, info) + ret, gas, precompileMultiGas, err = RunPrecompiledContract(stateDB, p, addr, input, gas, evm.Config.Tracer, info) usedMultiGas.SaturatingAddInto(precompileMultiGas) } else { // Initialise a new contract and make initialise the delegate values @@ -504,6 +523,10 @@ func (evm *EVM) StaticCall(caller common.Address, addr common.Address, input []b evm.StateDB.AddBalance(addr, new(uint256.Int), tracing.BalanceChangeTouchAccount) if p, isPrecompile := evm.precompile(addr); isPrecompile { + var stateDB StateDB + if evm.chainRules.IsAmsterdam { + stateDB = evm.StateDB + } info := &AdvancedPrecompileCall{ PrecompileAddress: addr, ActingAsAddress: addr, @@ -513,7 +536,7 @@ func (evm *EVM) StaticCall(caller common.Address, addr common.Address, input []b Evm: evm, } var precompileMultiGas multigas.MultiGas - ret, gas, precompileMultiGas, err = RunPrecompiledContract(p, input, gas, evm.Config.Tracer, info) + ret, gas, precompileMultiGas, err = RunPrecompiledContract(stateDB, p, addr, input, gas, evm.Config.Tracer, info) usedMultiGas.SaturatingAddInto(precompileMultiGas) } else { // Initialise a new contract and set the code that is to be used by the EVM. @@ -629,7 +652,7 @@ func (evm *EVM) create(caller common.Address, code []byte, gas uint64, value *ui // Charge consumed init gas as computation usedMultiGas = usedMultiGas.SaturatingIncrement(multigas.ResourceKindComputation, consumed) } - evm.Context.Transfer(evm.StateDB, caller, address, value) + evm.Context.Transfer(evm.StateDB, caller, address, value, &evm.chainRules) // Initialise a new contract and set the code that is to be used by the EVM. // The contract is a scoped environment for this execution context only. @@ -661,8 +684,8 @@ func (evm *EVM) initNewContract(contract *Contract, address common.Address) ([]b } // Check whether the max code size has been exceeded, assign err if the case. - if evm.chainRules.IsEIP158 && len(ret) > int(evm.chainConfig.MaxCodeSize()) { - return ret, ErrMaxCodeSizeExceeded + if err := CheckMaxCodeSize(&evm.chainRules, uint64(len(ret))); err != nil { + return ret, err } // Reject code starting with 0xEF if EIP-3541 is enabled. @@ -703,7 +726,7 @@ func (evm *EVM) Create(caller common.Address, code []byte, gas uint64, value *ui // The different between Create2 with Create is Create2 uses keccak256(0xff ++ msg.sender ++ salt ++ keccak256(init_code))[12:] // instead of the usual sender-and-nonce-hash as the address where the contract is initialized at. func (evm *EVM) Create2(caller common.Address, code []byte, gas uint64, endowment *uint256.Int, salt *uint256.Int) (ret []byte, contractAddr common.Address, leftOverGas uint64, usedMultiGas multigas.MultiGas, err error) { - inithash := crypto.HashData(evm.hasher, code) + inithash := crypto.Keccak256Hash(code) contractAddr = crypto.CreateAddress2(caller, salt.Bytes32(), inithash[:]) return evm.create(caller, code, gas, endowment, contractAddr, CREATE2) } diff --git a/core/vm/evm_arbitrum.go b/core/vm/evm_arbitrum.go index 87c774e6e6..b489831977 100644 --- a/core/vm/evm_arbitrum.go +++ b/core/vm/evm_arbitrum.go @@ -105,7 +105,7 @@ func (p DefaultTxProcessor) L1BlockHash(blockCtx BlockContext, l1BlocKNumber uin } func (p DefaultTxProcessor) GasPriceOp(evm *EVM) *big.Int { - return evm.GasPrice + return evm.GasPrice.ToBig() } func (p DefaultTxProcessor) FillReceiptInfo(*types.Receipt) {} diff --git a/core/vm/evm_arbitrum_test.go b/core/vm/evm_arbitrum_test.go index 380bf28fd9..cba1b81a41 100644 --- a/core/vm/evm_arbitrum_test.go +++ b/core/vm/evm_arbitrum_test.go @@ -64,7 +64,7 @@ func TestCreateStylusComponentPrefixArbosVersion(t *testing.T) { blockCtx := BlockContext{ CanTransfer: func(StateDB, common.Address, *uint256.Int) bool { return true }, - Transfer: func(StateDB, common.Address, common.Address, *uint256.Int) {}, + Transfer: func(StateDB, common.Address, common.Address, *uint256.Int, *params.Rules) {}, BlockNumber: big.NewInt(0), ArbOSVersion: tt.arbosVersion, } diff --git a/core/vm/gas.go b/core/vm/gas.go index f4cebc5bc0..f40be752bd 100644 --- a/core/vm/gas.go +++ b/core/vm/gas.go @@ -51,7 +51,6 @@ func callGas(isEip150 bool, availableGas, base uint64, callCost *uint256.Int) (u if !callCost.IsUint64() { return 0, ErrGasUintOverflow } - return callCost.Uint64(), nil } diff --git a/core/vm/gas_table.go b/core/vm/gas_table.go index 189963634b..e08f3c0fac 100644 --- a/core/vm/gas_table.go +++ b/core/vm/gas_table.go @@ -18,7 +18,6 @@ package vm import ( "errors" - "fmt" "github.com/ethereum/go-ethereum/arbitrum/multigas" "github.com/ethereum/go-ethereum/common" @@ -112,6 +111,9 @@ var ( ) func gasSStore(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (multigas.MultiGas, error) { + if evm.readOnly { + return multigas.ZeroGas(), ErrWriteProtection + } var ( y, x = stack.Back(1), stack.Back(0) current, original = evm.StateDB.GetStateAndCommittedState(contract.Address(), x.Bytes32()) @@ -197,6 +199,9 @@ func gasSStore(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySi // (2.2.2.1.) If original value is 0, add SSTORE_SET_GAS - SLOAD_GAS to refund counter. // (2.2.2.2.) Otherwise, add SSTORE_RESET_GAS - SLOAD_GAS gas to refund counter. func gasSStoreEIP2200(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (multigas.MultiGas, error) { + if evm.readOnly { + return multigas.ZeroGas(), ErrWriteProtection + } // If we fail the minimum gas availability invariant, fail (0) if contract.Gas <= params.SstoreSentryGasEIP2200 { return multigas.ZeroGas(), errors.New("not enough gas for reentrancy sentry") @@ -355,10 +360,10 @@ func gasCreateEip3860(evm *EVM, contract *Contract, stack *Stack, mem *Memory, m if overflow { return multigas.ZeroGas(), ErrGasUintOverflow } - if size > evm.chainConfig.MaxInitCodeSize() { - return multigas.ZeroGas(), fmt.Errorf("%w: size %d", ErrMaxInitCodeSizeExceeded, size) + if err := CheckMaxInitCodeSize(&evm.chainRules, size); err != nil { + return multigas.ZeroGas(), err } - // Since size <= params.MaxInitCodeSize, these multiplication cannot overflow + // Since size <= the protocol-defined maximum initcode size limit, these multiplication cannot overflow moreGas := params.InitCodeWordGas * ((size + 31) / 32) // Init code execution considered as computation. @@ -377,10 +382,10 @@ func gasCreate2Eip3860(evm *EVM, contract *Contract, stack *Stack, mem *Memory, if overflow { return multigas.ZeroGas(), ErrGasUintOverflow } - if size > evm.chainConfig.MaxInitCodeSize() { - return multigas.ZeroGas(), fmt.Errorf("%w: size %d", ErrMaxInitCodeSizeExceeded, size) + if err := CheckMaxInitCodeSize(&evm.chainRules, size); err != nil { + return multigas.ZeroGas(), err } - // Since size <= params.MaxInitCodeSize, these multiplication cannot overflow + // Since size <= the protocol-defined maximum initcode size limit, these multiplication cannot overflow moreGas := (params.InitCodeWordGas + params.Keccak256WordGas) * ((size + 31) / 32) // Init code execution and Keccak hashing both considered as computation. @@ -424,6 +429,10 @@ func gasCall(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize address = common.Address(stack.Back(1).Bytes20()) ) + if evm.readOnly && transfersValue { + return multigas.ZeroGas(), ErrWriteProtection + } + // Storage slot writes (zero → nonzero) considered as storage growth. // See rationale in: https://github.com/OffchainLabs/nitro/blob/master/docs/decisions/0002-multi-dimensional-gas-metering.md if evm.chainRules.IsEIP158 { @@ -536,6 +545,9 @@ func gasStaticCall(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memo } func gasSelfdestruct(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (multigas.MultiGas, error) { + if evm.readOnly { + return multigas.ZeroGas(), ErrWriteProtection + } multiGas := multigas.ZeroGas() // EIP150 homestead gas reprice fork: if evm.chainRules.IsEIP150 { diff --git a/core/vm/gas_table_test.go b/core/vm/gas_table_test.go index 7214ef56ca..2adf75f61b 100644 --- a/core/vm/gas_table_test.go +++ b/core/vm/gas_table_test.go @@ -95,7 +95,7 @@ func TestEIP2200(t *testing.T) { vmctx := BlockContext{ CanTransfer: func(StateDB, common.Address, *uint256.Int) bool { return true }, - Transfer: func(StateDB, common.Address, common.Address, *uint256.Int) {}, + Transfer: func(StateDB, common.Address, common.Address, *uint256.Int, *params.Rules) {}, } evm := NewEVM(vmctx, statedb, params.AllEthashProtocolChanges, Config{ExtraEips: []int{2200}}) @@ -145,15 +145,19 @@ func TestCreateGas(t *testing.T) { statedb.Finalise(true) vmctx := BlockContext{ CanTransfer: func(StateDB, common.Address, *uint256.Int) bool { return true }, - Transfer: func(StateDB, common.Address, common.Address, *uint256.Int) {}, + Transfer: func(StateDB, common.Address, common.Address, *uint256.Int, *params.Rules) {}, BlockNumber: big.NewInt(0), } config := Config{} + chainConfig := params.AllEthashProtocolChanges if tt.eip3860 { config.ExtraEips = []int{3860} + vmctx.Random = new(common.Hash) + + chainConfig = params.MergedTestChainConfig } - evm := NewEVM(vmctx, statedb, params.AllEthashProtocolChanges, config) + evm := NewEVM(vmctx, statedb, chainConfig, config) var startGas = uint64(testGas) ret, gas, _, err := evm.Call(common.Address{}, address, nil, startGas, new(uint256.Int)) if err != nil { diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 86cbf9697c..09ef01daed 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" "github.com/holiman/uint256" ) @@ -237,14 +238,12 @@ func opKeccak256(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { offset, size := scope.Stack.pop(), scope.Stack.peek() data := scope.Memory.GetPtr(offset.Uint64(), size.Uint64()) - evm.hasher.Reset() - evm.hasher.Write(data) - evm.hasher.Read(evm.hasherBuf[:]) + hash := crypto.Keccak256Hash(data) if evm.Config.EnablePreimageRecording { - evm.StateDB.AddPreimage(evm.hasherBuf, data) + evm.StateDB.AddPreimage(hash, data) } - size.SetBytes(evm.hasherBuf[:]) + size.SetBytes(hash[:]) return nil, nil } @@ -923,21 +922,34 @@ func opSelfdestruct(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { if evm.readOnly { return nil, ErrWriteProtection } - beneficiary := scope.Stack.pop() + var ( + this = scope.Contract.Address() + balance = evm.StateDB.GetBalance(this) + top = scope.Stack.pop() + beneficiary = common.Address(top.Bytes20()) + ) + evm.StateDB.TouchAddress(&filter.FilteredAddressRecord{ - Address: beneficiary.Bytes20(), + Address: beneficiary, FilterReason: filter.FilterReason{Reason: filter.ReasonSelfdestructBeneficiary, EventRuleMatch: nil}, }) - balance := evm.StateDB.GetBalance(scope.Contract.Address()) - evm.StateDB.AddBalance(beneficiary.Bytes20(), balance, tracing.BalanceIncreaseSelfdestruct) - evm.StateDB.SelfDestruct(scope.Contract.Address()) - if beneficiary.Bytes20() == scope.Contract.Address() { + + // The funds are burned immediately if the beneficiary is the caller itself, + // in this case, the beneficiary's balance is not increased. + if this != beneficiary { + evm.StateDB.AddBalance(beneficiary, balance, tracing.BalanceIncreaseSelfdestruct) + } + // Clear any leftover funds for the account being destructed. + evm.StateDB.SubBalance(this, balance, tracing.BalanceDecreaseSelfdestruct) + evm.StateDB.SelfDestruct(this) + if this == beneficiary { // Arbitrum: calling selfdestruct(this) burns the balance evm.StateDB.ExpectBalanceBurn(balance.ToBig()) } + if tracer := evm.Config.Tracer; tracer != nil { if tracer.OnEnter != nil { - tracer.OnEnter(evm.depth, byte(SELFDESTRUCT), scope.Contract.Address(), beneficiary.Bytes20(), []byte{}, 0, balance.ToBig()) + tracer.OnEnter(evm.depth, byte(SELFDESTRUCT), this, beneficiary, []byte{}, 0, balance.ToBig()) } if tracer.OnExit != nil { tracer.OnExit(evm.depth, []byte{}, 0, nil, false) @@ -950,33 +962,57 @@ func opSelfdestruct6780(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, erro if evm.readOnly { return nil, ErrWriteProtection } + var ( + this = scope.Contract.Address() + balance = evm.StateDB.GetBalance(this) + top = scope.Stack.pop() + beneficiary = common.Address(top.Bytes20()) + newContract = evm.StateDB.IsNewContract(this) + ) // Arbitrum: revert if acting account is a Stylus program - actingAddress := scope.Contract.Address() - if code := evm.StateDB.GetCode(actingAddress); state.IsStylusComponentPrefix(code, evm.chainRules.ArbOSVersion) { + if code := evm.StateDB.GetCode(this); state.IsStylusComponentPrefix(code, evm.chainRules.ArbOSVersion) { return nil, ErrExecutionReverted } - beneficiary := scope.Stack.pop() evm.StateDB.TouchAddress(&filter.FilteredAddressRecord{ - Address: beneficiary.Bytes20(), + Address: beneficiary, FilterReason: filter.FilterReason{Reason: filter.ReasonSelfdestructBeneficiary, EventRuleMatch: nil}, }) - balance := evm.StateDB.GetBalance(scope.Contract.Address()) - evm.StateDB.SubBalance(scope.Contract.Address(), balance, tracing.BalanceDecreaseSelfdestruct) - evm.StateDB.AddBalance(beneficiary.Bytes20(), balance, tracing.BalanceIncreaseSelfdestruct) - evm.StateDB.SelfDestruct6780(scope.Contract.Address()) - if beneficiary.Bytes20() == scope.Contract.Address() { + + // Contract is new and will actually be deleted. + if newContract { + if this != beneficiary { // Skip no-op transfer when self-destructing to self. + evm.StateDB.AddBalance(beneficiary, balance, tracing.BalanceIncreaseSelfdestruct) + } + evm.StateDB.SubBalance(this, balance, tracing.BalanceDecreaseSelfdestruct) + evm.StateDB.SelfDestruct(this) + } + + // Contract already exists, only do transfer if beneficiary is not self. + if !newContract && this != beneficiary { + evm.StateDB.SubBalance(this, balance, tracing.BalanceDecreaseSelfdestruct) + evm.StateDB.AddBalance(beneficiary, balance, tracing.BalanceIncreaseSelfdestruct) + } + if evm.chainRules.IsAmsterdam && !balance.IsZero() { + if this != beneficiary { + evm.StateDB.AddLog(types.EthTransferLog(this, beneficiary, balance)) + } else if newContract { + evm.StateDB.AddLog(types.EthBurnLog(this, balance)) + } + } + + if this == beneficiary { // SelfDestruct6780 only destructs the contract if selfdestructing in the same transaction as contract creation // So we only account for the balance burn if the contract is actually destructed by checking if the balance is zero. - if evm.StateDB.GetBalance(scope.Contract.Address()).Sign() == 0 { + if evm.StateDB.GetBalance(this).Sign() == 0 { // Arbitrum: calling selfdestruct(this) burns the balance evm.StateDB.ExpectBalanceBurn(balance.ToBig()) } } if tracer := evm.Config.Tracer; tracer != nil { if tracer.OnEnter != nil { - tracer.OnEnter(evm.depth, byte(SELFDESTRUCT), scope.Contract.Address(), beneficiary.Bytes20(), []byte{}, 0, balance.ToBig()) + tracer.OnEnter(evm.depth, byte(SELFDESTRUCT), this, beneficiary, []byte{}, 0, balance.ToBig()) } if tracer.OnExit != nil { tracer.OnExit(evm.depth, []byte{}, 0, nil, false) @@ -985,6 +1021,125 @@ func opSelfdestruct6780(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, erro return nil, errStopToken } +// decodeSingle decodes the immediate operand of a backward-compatible DUPN or SWAPN instruction (EIP-8024) +// https://eips.ethereum.org/EIPS/eip-8024 +func decodeSingle(x byte) int { + // Depths 1-16 are already covered by the legacy opcodes. The forbidden byte range [91, 127] removes + // 37 values from the 256 possible immediates, leaving 219 usable values, so this encoding covers depths + // 17 through 235. The immediate is encoded as (x + 111) % 256, where 111 is chosen so that these values + // avoid the forbidden range. Decoding is simply the modular inverse (i.e. 111+145=256). + return (int(x) + 145) % 256 +} + +// decodePair decodes the immediate operand of a backward-compatible EXCHANGE +// instruction (EIP-8024) into stack indices (n, m) where 1 <= n < m +// and n + m <= 30. The forbidden byte range [82, 127] removes 46 values from +// the 256 possible immediates, leaving exactly 210 usable bytes. +// https://eips.ethereum.org/EIPS/eip-8024 +func decodePair(x byte) (int, int) { + // XOR with 143 remaps the forbidden bytes [82, 127] to an unused corner + // of the 16x16 grid below. + k := int(x ^ 143) + // Split into row q and column r of a 16x16 grid. The 210 valid pairs + // occupy two triangles within this grid. + q, r := k/16, k%16 + // Upper triangle (q < r): pairs where m <= 16, encoded directly as + // (q+1, r+1). + if q < r { + return q + 1, r + 1 + } + // Lower triangle: pairs where m > 16, recovered as (r+1, 29-q). + return r + 1, 29 - q +} + +func opDupN(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { + code := scope.Contract.Code + i := *pc + 1 + + // If the immediate byte is missing, treat as 0x00 (same convention as PUSHn). + var x byte + if i < uint64(len(code)) { + x = code[i] + } + + // This range is excluded to preserve compatibility with existing opcodes. + if x > 90 && x < 128 { + return nil, &ErrInvalidOpCode{opcode: OpCode(x)} + } + n := decodeSingle(x) + + // DUPN duplicates the n'th stack item, so the stack must contain at least n elements. + if scope.Stack.len() < n { + return nil, &ErrStackUnderflow{stackLen: scope.Stack.len(), required: n} + } + + //The n‘th stack item is duplicated at the top of the stack. + scope.Stack.push(scope.Stack.Back(n - 1)) + *pc += 1 + return nil, nil +} + +func opSwapN(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { + code := scope.Contract.Code + i := *pc + 1 + + // If the immediate byte is missing, treat as 0x00 (same convention as PUSHn). + var x byte + if i < uint64(len(code)) { + x = code[i] + } + + // This range is excluded to preserve compatibility with existing opcodes. + if x > 90 && x < 128 { + return nil, &ErrInvalidOpCode{opcode: OpCode(x)} + } + n := decodeSingle(x) + + // SWAPN operates on the top and n+1 stack items, so the stack must contain at least n+1 elements. + if scope.Stack.len() < n+1 { + return nil, &ErrStackUnderflow{stackLen: scope.Stack.len(), required: n + 1} + } + + // The (n+1)‘th stack item is swapped with the top of the stack. + indexTop := scope.Stack.len() - 1 + indexN := scope.Stack.len() - 1 - n + scope.Stack.data[indexTop], scope.Stack.data[indexN] = scope.Stack.data[indexN], scope.Stack.data[indexTop] + *pc += 1 + return nil, nil +} + +func opExchange(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { + code := scope.Contract.Code + i := *pc + 1 + + // If the immediate byte is missing, treat as 0x00 (same convention as PUSHn). + var x byte + if i < uint64(len(code)) { + x = code[i] + } + + // This range is excluded both to preserve compatibility with existing opcodes + // and to keep decode_pair’s 16-aligned arithmetic mapping valid (0–81, 128–255). + if x > 81 && x < 128 { + return nil, &ErrInvalidOpCode{opcode: OpCode(x)} + } + n, m := decodePair(x) + need := max(n, m) + 1 + + // EXCHANGE operates on the (n+1)'th and (m+1)'th stack items, + // so the stack must contain at least max(n, m)+1 elements. + if scope.Stack.len() < need { + return nil, &ErrStackUnderflow{stackLen: scope.Stack.len(), required: need} + } + + // The (n+1)‘th stack item is swapped with the (m+1)‘th stack item. + indexN := scope.Stack.len() - 1 - n + indexM := scope.Stack.len() - 1 - m + scope.Stack.data[indexN], scope.Stack.data[indexM] = scope.Stack.data[indexM], scope.Stack.data[indexN] + *pc += 1 + return nil, nil +} + // following functions are used by the instruction jump table // make log instruction function @@ -1006,9 +1161,6 @@ func makeLog(size int) executionFunc { Address: scope.Contract.Address(), Topics: topics, Data: d, - // This is a non-consensus field, but assigned here because - // core/state doesn't know the current block number. - BlockNumber: evm.Context.BlockNumber.Uint64(), }) return nil, nil diff --git a/core/vm/instructions_multigas_test.go b/core/vm/instructions_multigas_test.go index 9ee87c7d26..65695fbf3c 100644 --- a/core/vm/instructions_multigas_test.go +++ b/core/vm/instructions_multigas_test.go @@ -18,7 +18,7 @@ func TestOpCreatesMultiGas(t *testing.T) { blockCtx := BlockContext{ CanTransfer: func(db StateDB, addr common.Address, amount *uint256.Int) bool { return true }, - Transfer: func(db StateDB, from, to common.Address, amount *uint256.Int) {}, + Transfer: func(db StateDB, from, to common.Address, amount *uint256.Int, rules *params.Rules) {}, GetHash: func(uint64) common.Hash { return common.Hash{} }, } @@ -86,7 +86,7 @@ func TestOpCallsMultiGas(t *testing.T) { CanTransfer: func(db StateDB, addr common.Address, amount *uint256.Int) bool { return true }, - Transfer: func(db StateDB, from, to common.Address, amount *uint256.Int) {}, + Transfer: func(db StateDB, from, to common.Address, amount *uint256.Int, rules *params.Rules) {}, GetHash: func(uint64) common.Hash { return common.Hash{} }, } diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index bf027ce110..b9882081bb 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -19,6 +19,7 @@ package vm import ( "bytes" "encoding/json" + "errors" "fmt" "math/big" "os" @@ -1081,3 +1082,177 @@ func TestOpSelfdestruct6780NonStylusCodeProceeds(t *testing.T) { }) } } + +func TestEIP8024_Execution(t *testing.T) { + evm := NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) + + tests := []struct { + name string + codeHex string + wantErr error + wantOpcode OpCode + wantVals []uint64 + }{ + { + name: "DUPN", + codeHex: "60016000808080808080808080808080808080e680", + wantVals: []uint64{ + 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, + }, + }, + { + name: "SWAPN", + codeHex: "600160008080808080808080808080808080806002e780", + wantVals: []uint64{ + 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2, + }, + }, + { + name: "EXCHANGE_MISSING_IMMEDIATE", + codeHex: "600260008080808080600160008080808080808080e8", + wantVals: []uint64{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2, // 10th from top + 0, 0, 0, 0, 0, 0, + 1, // bottom + }, + }, + { + name: "EXCHANGE", + codeHex: "600060016002e88e", + wantVals: []uint64{2, 0, 1}, + }, + { + name: "EXCHANGE", + codeHex: "600080808080808080808080808080808080808080808080808080808060016002e88f", + wantVals: []uint64{ + 2, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, + }, + }, + { + name: "INVALID_SWAPN_LOW", + codeHex: "e75b", + wantErr: &ErrInvalidOpCode{}, + wantOpcode: SWAPN, + }, + { + name: "JUMP_OVER_INVALID_DUPN", + codeHex: "600456e65b", + wantErr: nil, + }, + { + name: "EXCHANGE", + codeHex: "60008080e88e15", + wantVals: []uint64{1, 0, 0}, + }, + { + name: "INVALID_EXCHANGE", + codeHex: "e852", + wantErr: &ErrInvalidOpCode{}, + wantOpcode: EXCHANGE, + }, + { + name: "UNDERFLOW_DUPN", + codeHex: "6000808080808080808080808080808080e680", + wantErr: &ErrStackUnderflow{}, + wantOpcode: DUPN, + }, + // Additional test cases + { + name: "PC_INCREMENT", + codeHex: "600060006000e88e15", + wantVals: []uint64{1, 0, 0}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + code := common.FromHex(tc.codeHex) + stack := newstack() + pc := uint64(0) + scope := &ScopeContext{Stack: stack, Contract: &Contract{Code: code}} + var err error + var errOp OpCode + for pc < uint64(len(code)) && err == nil { + op := code[pc] + switch OpCode(op) { + case STOP: + return + case PUSH1: + _, err = opPush1(&pc, evm, scope) + case DUP1: + dup1 := makeDup(1) + _, err = dup1(&pc, evm, scope) + case JUMP: + _, err = opJump(&pc, evm, scope) + case JUMPDEST: + _, err = opJumpdest(&pc, evm, scope) + case ISZERO: + _, err = opIszero(&pc, evm, scope) + case PUSH0: + _, err = opPush0(&pc, evm, scope) + case DUPN: + _, err = opDupN(&pc, evm, scope) + case SWAPN: + _, err = opSwapN(&pc, evm, scope) + case EXCHANGE: + _, err = opExchange(&pc, evm, scope) + default: + t.Fatalf("unexpected opcode %s at pc=%d", OpCode(op), pc) + } + if err != nil { + errOp = OpCode(op) + } + pc++ + } + if tc.wantErr != nil { + // Fail because we wanted an error, but didn't get one. + if err == nil { + t.Fatalf("expected error, got nil") + } + // Fail if the wrong opcode threw an error. + if errOp != tc.wantOpcode { + t.Fatalf("expected error from opcode %s, got %s", tc.wantOpcode, errOp) + } + // Fail if we don't get the error we expect. + switch tc.wantErr.(type) { + case *ErrInvalidOpCode: + var want *ErrInvalidOpCode + if !errors.As(err, &want) { + t.Fatalf("expected ErrInvalidOpCode, got %v", err) + } + case *ErrStackUnderflow: + var want *ErrStackUnderflow + if !errors.As(err, &want) { + t.Fatalf("expected ErrStackUnderflow, got %v", err) + } + default: + t.Fatalf("unsupported wantErr type %T", tc.wantErr) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + got := make([]uint64, 0, stack.len()) + for i := stack.len() - 1; i >= 0; i-- { + got = append(got, stack.data[i].Uint64()) + } + if len(got) != len(tc.wantVals) { + t.Fatalf("stack len=%d; want %d", len(got), len(tc.wantVals)) + } + for i := range got { + if got[i] != tc.wantVals[i] { + t.Fatalf("[%s] stack[%d]=%d; want %d\nstack=%v", + tc.name, i, got[i], tc.wantVals[i], got) + } + } + }) + } +} diff --git a/core/vm/interface.go b/core/vm/interface.go index f3a38a4e3c..c6ccc99449 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -27,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie/utils" "github.com/holiman/uint256" ) @@ -95,20 +94,18 @@ type StateDB interface { GetTransientState(addr common.Address, key common.Hash) common.Hash SetTransientState(addr common.Address, key, value common.Hash) - SelfDestruct(common.Address) uint256.Int + SelfDestruct(common.Address) HasSelfDestructed(common.Address) bool GetSelfDestructs() []common.Address - // SelfDestruct6780 is post-EIP6780 selfdestruct, which means that it's a - // send-all-to-beneficiary, unless the contract was created in this same - // transaction, in which case it will be destructed. - // This method returns the prior balance, along with a boolean which is - // true iff the object was indeed destructed. - SelfDestruct6780(common.Address) (uint256.Int, bool) - // Exist reports whether the given account exists in state. // Notably this also returns true for self-destructed accounts within the current transaction. Exist(common.Address) bool + + // IsNewContract reports whether the contract at the given address was deployed + // during the current transaction. + IsNewContract(addr common.Address) bool + // Empty returns whether the given account is empty. Empty // is defined according to EIP161 (balance = nonce = code = 0). Empty(common.Address) bool @@ -124,15 +121,13 @@ type StateDB interface { // GetAccessList returns the access list GetAccessList() (addresses map[common.Address]int, slots []map[common.Hash]struct{}) - // PointCache returns the point cache used in computations - PointCache() *utils.PointCache - Prepare(rules params.Rules, sender, coinbase common.Address, dest *common.Address, precompiles []common.Address, txAccesses types.AccessList) RevertToSnapshot(int) Snapshot() int AddLog(*types.Log) + EmitLogsForBurnAccounts() AddPreimage(common.Hash, []byte) GetCurrentTxLogs() []*types.Log diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index dfeb23fd29..003a724076 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -29,14 +29,12 @@ import ( // Config are the configuration options for the Interpreter type Config struct { - Tracer *tracing.Hooks + Tracer *tracing.Hooks + NoBaseFee bool // Forces the EIP-1559 baseFee to 0 (needed for 0 price calls) EnablePreimageRecording bool // Enables recording of SHA3/keccak preimages ExtraEips []int // Additional EIPS that are to be enabled - StatelessSelfValidation bool // Generate execution witnesses and self-check against them (testing purpose) - EnableWitnessStats bool // Whether trie access statistics collection is enabled - ExposeMultiGas bool // Arbitrum: Expose multi-gas used in transaction receipts } diff --git a/core/vm/interpreter_test.go b/core/vm/interpreter_test.go index ca9a93a27b..235042ad16 100644 --- a/core/vm/interpreter_test.go +++ b/core/vm/interpreter_test.go @@ -40,7 +40,7 @@ var loopInterruptTests = []string{ func TestLoopInterrupt(t *testing.T) { address := common.BytesToAddress([]byte("contract")) vmctx := BlockContext{ - Transfer: func(StateDB, common.Address, common.Address, *uint256.Int) {}, + Transfer: func(StateDB, common.Address, common.Address, *uint256.Int, *params.Rules) {}, } for i, tt := range loopInterruptTests { diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index bce25b7391..aad6cd3330 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -66,6 +66,7 @@ var ( verkleInstructionSet = newVerkleInstructionSet() pragueInstructionSet = newPragueInstructionSet() osakaInstructionSet = newOsakaInstructionSet() + amsterdamInstructionSet = newAmsterdamInstructionSet() ) // JumpTable contains the EVM opcodes supported at a given fork. @@ -95,6 +96,13 @@ func newVerkleInstructionSet() JumpTable { return validate(instructionSet) } +func newAmsterdamInstructionSet() JumpTable { + instructionSet := newOsakaInstructionSet() + enable7843(&instructionSet) // EIP-7843 (SLOTNUM opcode) + enable8024(&instructionSet) // EIP-8024 (Backward compatible SWAPN, DUPN, EXCHANGE) + return validate(instructionSet) +} + func newOsakaInstructionSet() JumpTable { instructionSet := newPragueInstructionSet() enable7939(&instructionSet) // EIP-7939 (CLZ opcode) diff --git a/core/vm/jump_table_export.go b/core/vm/jump_table_export.go index 89a2ebf6f4..fdf814d64c 100644 --- a/core/vm/jump_table_export.go +++ b/core/vm/jump_table_export.go @@ -28,6 +28,8 @@ func LookupInstructionSet(rules params.Rules) (JumpTable, error) { switch { case rules.IsVerkle: return newCancunInstructionSet(), errors.New("verkle-fork not defined yet") + case rules.IsAmsterdam: + return newAmsterdamInstructionSet(), nil case rules.IsOsaka: return newOsakaInstructionSet(), nil case rules.IsPrague: diff --git a/core/vm/memory.go b/core/vm/memory.go index 5e11e83748..54bc2b2849 100644 --- a/core/vm/memory.go +++ b/core/vm/memory.go @@ -44,6 +44,7 @@ func (m *Memory) Free() { // To reduce peak allocation, return only smaller memory instances to the pool. const maxBufferSize = 16 << 10 if cap(m.store) <= maxBufferSize { + clear(m.store) m.store = m.store[:0] m.lastGasCost = 0 memoryPool.Put(m) @@ -76,10 +77,14 @@ func (m *Memory) Set32(offset uint64, val *uint256.Int) { val.PutUint256(m.store[offset:]) } -// Resize resizes the memory to size +// Resize grows the memory to the requested size. func (m *Memory) Resize(size uint64) { - if uint64(m.Len()) < size { - m.store = append(m.store, make([]byte, size-uint64(m.Len()))...) + if uint64(len(m.store)) < size { + if uint64(cap(m.store)) >= size { + m.store = m.store[:size] + } else { + m.store = append(m.store, make([]byte, size-uint64(len(m.store)))...) + } } } diff --git a/core/vm/memory_test.go b/core/vm/memory_test.go index 41389b729a..3890d18cb5 100644 --- a/core/vm/memory_test.go +++ b/core/vm/memory_test.go @@ -83,3 +83,10 @@ func TestMemoryCopy(t *testing.T) { } } } + +func BenchmarkResize(b *testing.B) { + memory := NewMemory() + for i := range b.N { + memory.Resize(uint64(i)) + } +} diff --git a/core/vm/opcodes.go b/core/vm/opcodes.go index 9a32126a80..cb871dca6d 100644 --- a/core/vm/opcodes.go +++ b/core/vm/opcodes.go @@ -105,6 +105,7 @@ const ( BASEFEE OpCode = 0x48 BLOBHASH OpCode = 0x49 BLOBBASEFEE OpCode = 0x4a + SLOTNUM OpCode = 0x4b ) // 0x50 range - 'storage' and execution. @@ -320,6 +321,7 @@ var opCodeToString = [256]string{ BASEFEE: "BASEFEE", BLOBHASH: "BLOBHASH", BLOBBASEFEE: "BLOBBASEFEE", + SLOTNUM: "SLOTNUM", // 0x50 range - 'storage' and execution. POP: "POP", @@ -502,6 +504,7 @@ var stringToOp = map[string]OpCode{ "BASEFEE": BASEFEE, "BLOBHASH": BLOBHASH, "BLOBBASEFEE": BLOBBASEFEE, + "SLOTNUM": SLOTNUM, "DELEGATECALL": DELEGATECALL, "STATICCALL": STATICCALL, "CODESIZE": CODESIZE, diff --git a/core/vm/operations_acl.go b/core/vm/operations_acl.go index ca42d8148c..ac17b89249 100644 --- a/core/vm/operations_acl.go +++ b/core/vm/operations_acl.go @@ -28,6 +28,9 @@ import ( func makeGasSStoreFunc(clearingRefund uint64) gasFunc { return func(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (multigas.MultiGas, error) { + if evm.readOnly { + return multigas.ZeroGas(), ErrWriteProtection + } // If we fail the minimum gas availability invariant, fail (0) if contract.Gas <= params.SstoreSentryGasEIP2200 { return multigas.ZeroGas(), errors.New("not enough gas for reentrancy sentry") @@ -257,12 +260,21 @@ func makeSelfdestructGasFn(refundsEnabled bool) gasFunc { multiGas = multigas.ZeroGas() address = common.Address(stack.peek().Bytes20()) ) + if evm.readOnly { + return multigas.ZeroGas(), ErrWriteProtection + } if !evm.StateDB.AddressInAccessList(address) { // If the caller cannot afford the cost, this change will be rolled back evm.StateDB.AddAddressToAccessList(address) // Cold account access considered as storage access read. // See rationale in: https://github.com/OffchainLabs/nitro/blob/master/docs/decisions/0002-multi-dimensional-gas-metering.md multiGas = multiGas.SaturatingIncrement(multigas.ResourceKindStorageAccessRead, params.ColdAccountAccessCostEIP2929) + + // Terminate the gas measurement if the leftover gas is not sufficient, + // it can effectively prevent accessing the states in the following steps + if contract.Gas < multiGas.SingleGas() { + return multigas.ZeroGas(), ErrOutOfGas + } } // if empty and transfers value if evm.StateDB.Empty(address) && evm.StateDB.GetBalance(contract.Address()).Sign() != 0 { @@ -279,12 +291,24 @@ func makeSelfdestructGasFn(refundsEnabled bool) gasFunc { } var ( - gasCallEIP7702 = makeCallVariantGasCallEIP7702(gasCall) + innerGasCallEIP7702 = makeCallVariantGasCallEIP7702(gasCall) gasDelegateCallEIP7702 = makeCallVariantGasCallEIP7702(gasDelegateCall) gasStaticCallEIP7702 = makeCallVariantGasCallEIP7702(gasStaticCall) gasCallCodeEIP7702 = makeCallVariantGasCallEIP7702(gasCallCode) ) +func gasCallEIP7702(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (multigas.MultiGas, error) { + // Return early if this call attempts to transfer value in a static context. + // Although it's checked in `gasCall`, EIP-7702 loads the target's code before + // to determine if it is resolving a delegation. This could incorrectly record + // the target in the block access list (BAL) if the call later fails. + transfersValue := !stack.Back(2).IsZero() + if evm.readOnly && transfersValue { + return multigas.ZeroGas(), ErrWriteProtection + } + return innerGasCallEIP7702(evm, contract, stack, mem, memorySize) +} + func makeCallVariantGasCallEIP7702(oldCalculator gasFunc) gasFunc { return func(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (multigas.MultiGas, error) { var ( @@ -335,7 +359,6 @@ func makeCallVariantGasCallEIP7702(oldCalculator gasFunc) gasFunc { if err != nil { return multiOld, err } - // Temporarily add the gas charge back to the contract and return value. By // adding it to the return, it will be charged outside of this function, as // part of the dynamic gas. This will ensure it is correctly reported to diff --git a/core/vm/operations_gas_test.go b/core/vm/operations_gas_test.go index bea744b222..f21c5c5430 100644 --- a/core/vm/operations_gas_test.go +++ b/core/vm/operations_gas_test.go @@ -420,7 +420,7 @@ func TestGasSStore4762(t *testing.T) { mem := NewMemory() // Setup access list and stack - accessList := state.NewAccessEvents(evm.StateDB.PointCache()) + accessList := state.NewAccessEvents() accessList.AddAccount(caller, false, 0) evm.AccessEvents = accessList @@ -488,7 +488,7 @@ func testGasSLoad(t *testing.T, gasFunc gasFunc, slotKeyProviders ...func(contra mem := NewMemory() // Setup access list and stack - accessList := state.NewAccessEvents(evm.StateDB.PointCache()) + accessList := state.NewAccessEvents() accessList.AddAccount(caller, false, 0) evm.AccessEvents = accessList @@ -589,7 +589,7 @@ func testGasCallFuncFuncWithCases(t *testing.T, config *params.ChainConfig, gasC // Mock AccessEvents for EIP4762 if tc.isEIP4762 { - accessEvents := state.NewAccessEvents(stateDb.PointCache()) + accessEvents := state.NewAccessEvents() evm.AccessEvents = accessEvents } @@ -987,6 +987,13 @@ func testGasCreateFunc(t *testing.T, gasImplFunc gasFunc, includeHashCost bool, } evm := NewEVM(BlockContext{}, statedb, chainConfig, Config{}) + if enableEip3860 { + // BlockContext{} has nil BlockNumber, so IsLondon (and thus IsShanghai) + // is false in the auto-derived rules. The new vm.CheckMaxInitCodeSize + // helper gates the init-code-size limit on rules.IsShanghai/IsAmsterdam, + // so set it explicitly here to exercise the EIP-3860 limit. + evm.chainRules.IsShanghai = true + } caller := common.Address{} contractAddr := common.Address{1} @@ -1263,8 +1270,8 @@ func TestGasSelfdestructEIP4762(t *testing.T) { mem := NewMemory() // Setup access list - accessList := state.NewAccessEvents(evm.StateDB.PointCache()) - accessListForExpected := state.NewAccessEvents(evm.StateDB.PointCache()) + accessList := state.NewAccessEvents() + accessListForExpected := state.NewAccessEvents() evm.AccessEvents = accessList stateDb.CreateAccount(beneficiaryAddr) @@ -1437,7 +1444,7 @@ func TestGasCodeCopyEIP4762(t *testing.T) { // EVM + access events statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) evm := NewEVM(BlockContext{BlockNumber: big.NewInt(0)}, statedb, params.TestChainConfig, Config{}) - evm.AccessEvents = state.NewAccessEvents(evm.StateDB.PointCache()) + evm.AccessEvents = state.NewAccessEvents() contract := NewContract(common.Address{}, common.HexToAddress("0x1234"), new(uint256.Int), 100000, nil) contract.Code = make([]byte, c.codeLen) @@ -1464,7 +1471,7 @@ func TestGasCodeCopyEIP4762(t *testing.T) { if c.expectChunks { _, copyOff, nonPadded := getDataAndAdjustedBounds(contract.Code, c.codeOffset, c.length) single := expectedMultiGas.SingleGas() - aeExp := state.NewAccessEvents(evm.StateDB.PointCache()) + aeExp := state.NewAccessEvents() _, wanted := aeExp.CodeChunksRangeGas(contract.Address(), copyOff, nonPadded, uint64(len(contract.Code)), false, contract.Gas-single) expectedMultiGas = expectedMultiGas.SaturatingIncrement(multigas.ResourceKindStorageAccessRead, wanted) } @@ -1519,7 +1526,7 @@ func TestGasExtCodeCopyEIP4762(t *testing.T) { // EVM + access events stateDb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) evm := NewEVM(BlockContext{BlockNumber: big.NewInt(0)}, stateDb, params.TestChainConfig, Config{}) - evm.AccessEvents = state.NewAccessEvents(evm.StateDB.PointCache()) + evm.AccessEvents = state.NewAccessEvents() contract := NewContract(common.Address{}, common.HexToAddress("0xBEEF"), new(uint256.Int), 100000, nil) @@ -1545,7 +1552,7 @@ func TestGasExtCodeCopyEIP4762(t *testing.T) { expectedMultiGas = expectedMultiGas.SaturatingIncrement(multigas.ResourceKindComputation, params.WarmStorageReadCostEIP2929) } else { singleGas := expectedMultiGas.SingleGas() - accessEventsForExpected := state.NewAccessEvents(evm.StateDB.PointCache()) + accessEventsForExpected := state.NewAccessEvents() wgas := accessEventsForExpected.BasicDataGas(c.addr, false, contract.Gas-singleGas, true) expectedMultiGas = expectedMultiGas.SaturatingIncrement(multigas.ResourceKindStorageAccessRead, wgas) } @@ -1724,7 +1731,7 @@ func TestGasBalanceExtCodeSizeExtCodeHash4762(t *testing.T) { // EVM + AccessEvents stateDb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) evm := NewEVM(BlockContext{BlockNumber: big.NewInt(0)}, stateDb, params.TestChainConfig, Config{}) - evm.AccessEvents = state.NewAccessEvents(evm.StateDB.PointCache()) + evm.AccessEvents = state.NewAccessEvents() contract := NewContract(common.Address{}, common.HexToAddress("0xBEEF"), new(uint256.Int), 100000, nil) @@ -1744,7 +1751,7 @@ func TestGasBalanceExtCodeSizeExtCodeHash4762(t *testing.T) { } // Reset AccessEvents so the call sees the same state - evm.AccessEvents = state.NewAccessEvents(evm.StateDB.PointCache()) + evm.AccessEvents = state.NewAccessEvents() multiGas, err := tt.gasFn(evm, contract, stack, mem, 0) if err != nil { diff --git a/core/vm/runtime/env.go b/core/vm/runtime/env.go index 6865f72a3f..2632cce6d7 100644 --- a/core/vm/runtime/env.go +++ b/core/vm/runtime/env.go @@ -19,6 +19,7 @@ package runtime import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/vm" + "github.com/holiman/uint256" ) func NewEnv(cfg *Config) *vm.EVM { @@ -27,9 +28,8 @@ func NewEnv(cfg *Config) *vm.EVM { } txContext := vm.TxContext{ Origin: cfg.Origin, - GasPrice: cfg.GasPrice, + GasPrice: uint256.MustFromBig(cfg.GasPrice), BlobHashes: cfg.BlobHashes, - BlobFeeCap: cfg.BlobFeeCap, } blockContext := vm.BlockContext{ CanTransfer: core.CanTransfer, diff --git a/crypto/ecies/ecies.go b/crypto/ecies/ecies.go index 9a892781f4..378d764a19 100644 --- a/crypto/ecies/ecies.go +++ b/crypto/ecies/ecies.go @@ -124,6 +124,9 @@ func (prv *PrivateKey) GenerateShared(pub *PublicKey, skLen, macLen int) (sk []b if prv.PublicKey.Curve != pub.Curve { return nil, ErrInvalidCurve } + if pub.X == nil || pub.Y == nil || !pub.Curve.IsOnCurve(pub.X, pub.Y) { + return nil, ErrInvalidPublicKey + } if skLen+macLen > MaxSharedKeyLength(pub) { return nil, ErrSharedKeyTooBig } diff --git a/crypto/keccak/LICENSE b/crypto/keccak/LICENSE new file mode 100644 index 0000000000..2a7cf70da6 --- /dev/null +++ b/crypto/keccak/LICENSE @@ -0,0 +1,27 @@ +Copyright 2009 The Go Authors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google LLC nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/crypto/keccak/README.md b/crypto/keccak/README.md new file mode 100644 index 0000000000..295a5b958c --- /dev/null +++ b/crypto/keccak/README.md @@ -0,0 +1,6 @@ +This is a vendored and modified copy of golang.org/x/crypto/sha3, with an assembly +implementation of keccak256. We wish to retain the assembly implementation, +which was removed in v0.44.0. + +Ethereum uses a 'legacy' variant of Keccak, which was defined before it became SHA3. As +such, we cannot use the standard library crypto/sha3 package. diff --git a/crypto/keccak/hashes.go b/crypto/keccak/hashes.go new file mode 100644 index 0000000000..c78c5fe992 --- /dev/null +++ b/crypto/keccak/hashes.go @@ -0,0 +1,44 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package keccak + +// This file provides functions for creating instances of the SHA-3 +// and SHAKE hash functions, as well as utility functions for hashing +// bytes. + +import ( + "hash" +) + +const ( + dsbyteSHA3 = 0b00000110 + dsbyteKeccak = 0b00000001 + dsbyteShake = 0b00011111 + dsbyteCShake = 0b00000100 + + // rateK[c] is the rate in bytes for Keccak[c] where c is the capacity in + // bits. Given the sponge size is 1600 bits, the rate is 1600 - c bits. + rateK256 = (1600 - 256) / 8 + rateK448 = (1600 - 448) / 8 + rateK512 = (1600 - 512) / 8 + rateK768 = (1600 - 768) / 8 + rateK1024 = (1600 - 1024) / 8 +) + +// NewLegacyKeccak256 creates a new Keccak-256 hash. +// +// Only use this function if you require compatibility with an existing cryptosystem +// that uses non-standard padding. All other users should use New256 instead. +func NewLegacyKeccak256() hash.Hash { + return &state{rate: rateK512, outputLen: 32, dsbyte: dsbyteKeccak} +} + +// NewLegacyKeccak512 creates a new Keccak-512 hash. +// +// Only use this function if you require compatibility with an existing cryptosystem +// that uses non-standard padding. All other users should use New512 instead. +func NewLegacyKeccak512() hash.Hash { + return &state{rate: rateK1024, outputLen: 64, dsbyte: dsbyteKeccak} +} diff --git a/crypto/keccak/keccakf.go b/crypto/keccak/keccakf.go new file mode 100644 index 0000000000..82694fa4a3 --- /dev/null +++ b/crypto/keccak/keccakf.go @@ -0,0 +1,414 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !amd64 || purego || !gc + +package keccak + +import "math/bits" + +// rc stores the round constants for use in the ι step. +var rc = [24]uint64{ + 0x0000000000000001, + 0x0000000000008082, + 0x800000000000808A, + 0x8000000080008000, + 0x000000000000808B, + 0x0000000080000001, + 0x8000000080008081, + 0x8000000000008009, + 0x000000000000008A, + 0x0000000000000088, + 0x0000000080008009, + 0x000000008000000A, + 0x000000008000808B, + 0x800000000000008B, + 0x8000000000008089, + 0x8000000000008003, + 0x8000000000008002, + 0x8000000000000080, + 0x000000000000800A, + 0x800000008000000A, + 0x8000000080008081, + 0x8000000000008080, + 0x0000000080000001, + 0x8000000080008008, +} + +// keccakF1600 applies the Keccak permutation to a 1600b-wide +// state represented as a slice of 25 uint64s. +func keccakF1600(a *[25]uint64) { + // Implementation translated from Keccak-inplace.c + // in the keccak reference code. + var t, bc0, bc1, bc2, bc3, bc4, d0, d1, d2, d3, d4 uint64 + + for i := 0; i < 24; i += 4 { + // Combines the 5 steps in each round into 2 steps. + // Unrolls 4 rounds per loop and spreads some steps across rounds. + + // Round 1 + bc0 = a[0] ^ a[5] ^ a[10] ^ a[15] ^ a[20] + bc1 = a[1] ^ a[6] ^ a[11] ^ a[16] ^ a[21] + bc2 = a[2] ^ a[7] ^ a[12] ^ a[17] ^ a[22] + bc3 = a[3] ^ a[8] ^ a[13] ^ a[18] ^ a[23] + bc4 = a[4] ^ a[9] ^ a[14] ^ a[19] ^ a[24] + d0 = bc4 ^ (bc1<<1 | bc1>>63) + d1 = bc0 ^ (bc2<<1 | bc2>>63) + d2 = bc1 ^ (bc3<<1 | bc3>>63) + d3 = bc2 ^ (bc4<<1 | bc4>>63) + d4 = bc3 ^ (bc0<<1 | bc0>>63) + + bc0 = a[0] ^ d0 + t = a[6] ^ d1 + bc1 = bits.RotateLeft64(t, 44) + t = a[12] ^ d2 + bc2 = bits.RotateLeft64(t, 43) + t = a[18] ^ d3 + bc3 = bits.RotateLeft64(t, 21) + t = a[24] ^ d4 + bc4 = bits.RotateLeft64(t, 14) + a[0] = bc0 ^ (bc2 &^ bc1) ^ rc[i] + a[6] = bc1 ^ (bc3 &^ bc2) + a[12] = bc2 ^ (bc4 &^ bc3) + a[18] = bc3 ^ (bc0 &^ bc4) + a[24] = bc4 ^ (bc1 &^ bc0) + + t = a[10] ^ d0 + bc2 = bits.RotateLeft64(t, 3) + t = a[16] ^ d1 + bc3 = bits.RotateLeft64(t, 45) + t = a[22] ^ d2 + bc4 = bits.RotateLeft64(t, 61) + t = a[3] ^ d3 + bc0 = bits.RotateLeft64(t, 28) + t = a[9] ^ d4 + bc1 = bits.RotateLeft64(t, 20) + a[10] = bc0 ^ (bc2 &^ bc1) + a[16] = bc1 ^ (bc3 &^ bc2) + a[22] = bc2 ^ (bc4 &^ bc3) + a[3] = bc3 ^ (bc0 &^ bc4) + a[9] = bc4 ^ (bc1 &^ bc0) + + t = a[20] ^ d0 + bc4 = bits.RotateLeft64(t, 18) + t = a[1] ^ d1 + bc0 = bits.RotateLeft64(t, 1) + t = a[7] ^ d2 + bc1 = bits.RotateLeft64(t, 6) + t = a[13] ^ d3 + bc2 = bits.RotateLeft64(t, 25) + t = a[19] ^ d4 + bc3 = bits.RotateLeft64(t, 8) + a[20] = bc0 ^ (bc2 &^ bc1) + a[1] = bc1 ^ (bc3 &^ bc2) + a[7] = bc2 ^ (bc4 &^ bc3) + a[13] = bc3 ^ (bc0 &^ bc4) + a[19] = bc4 ^ (bc1 &^ bc0) + + t = a[5] ^ d0 + bc1 = bits.RotateLeft64(t, 36) + t = a[11] ^ d1 + bc2 = bits.RotateLeft64(t, 10) + t = a[17] ^ d2 + bc3 = bits.RotateLeft64(t, 15) + t = a[23] ^ d3 + bc4 = bits.RotateLeft64(t, 56) + t = a[4] ^ d4 + bc0 = bits.RotateLeft64(t, 27) + a[5] = bc0 ^ (bc2 &^ bc1) + a[11] = bc1 ^ (bc3 &^ bc2) + a[17] = bc2 ^ (bc4 &^ bc3) + a[23] = bc3 ^ (bc0 &^ bc4) + a[4] = bc4 ^ (bc1 &^ bc0) + + t = a[15] ^ d0 + bc3 = bits.RotateLeft64(t, 41) + t = a[21] ^ d1 + bc4 = bits.RotateLeft64(t, 2) + t = a[2] ^ d2 + bc0 = bits.RotateLeft64(t, 62) + t = a[8] ^ d3 + bc1 = bits.RotateLeft64(t, 55) + t = a[14] ^ d4 + bc2 = bits.RotateLeft64(t, 39) + a[15] = bc0 ^ (bc2 &^ bc1) + a[21] = bc1 ^ (bc3 &^ bc2) + a[2] = bc2 ^ (bc4 &^ bc3) + a[8] = bc3 ^ (bc0 &^ bc4) + a[14] = bc4 ^ (bc1 &^ bc0) + + // Round 2 + bc0 = a[0] ^ a[5] ^ a[10] ^ a[15] ^ a[20] + bc1 = a[1] ^ a[6] ^ a[11] ^ a[16] ^ a[21] + bc2 = a[2] ^ a[7] ^ a[12] ^ a[17] ^ a[22] + bc3 = a[3] ^ a[8] ^ a[13] ^ a[18] ^ a[23] + bc4 = a[4] ^ a[9] ^ a[14] ^ a[19] ^ a[24] + d0 = bc4 ^ (bc1<<1 | bc1>>63) + d1 = bc0 ^ (bc2<<1 | bc2>>63) + d2 = bc1 ^ (bc3<<1 | bc3>>63) + d3 = bc2 ^ (bc4<<1 | bc4>>63) + d4 = bc3 ^ (bc0<<1 | bc0>>63) + + bc0 = a[0] ^ d0 + t = a[16] ^ d1 + bc1 = bits.RotateLeft64(t, 44) + t = a[7] ^ d2 + bc2 = bits.RotateLeft64(t, 43) + t = a[23] ^ d3 + bc3 = bits.RotateLeft64(t, 21) + t = a[14] ^ d4 + bc4 = bits.RotateLeft64(t, 14) + a[0] = bc0 ^ (bc2 &^ bc1) ^ rc[i+1] + a[16] = bc1 ^ (bc3 &^ bc2) + a[7] = bc2 ^ (bc4 &^ bc3) + a[23] = bc3 ^ (bc0 &^ bc4) + a[14] = bc4 ^ (bc1 &^ bc0) + + t = a[20] ^ d0 + bc2 = bits.RotateLeft64(t, 3) + t = a[11] ^ d1 + bc3 = bits.RotateLeft64(t, 45) + t = a[2] ^ d2 + bc4 = bits.RotateLeft64(t, 61) + t = a[18] ^ d3 + bc0 = bits.RotateLeft64(t, 28) + t = a[9] ^ d4 + bc1 = bits.RotateLeft64(t, 20) + a[20] = bc0 ^ (bc2 &^ bc1) + a[11] = bc1 ^ (bc3 &^ bc2) + a[2] = bc2 ^ (bc4 &^ bc3) + a[18] = bc3 ^ (bc0 &^ bc4) + a[9] = bc4 ^ (bc1 &^ bc0) + + t = a[15] ^ d0 + bc4 = bits.RotateLeft64(t, 18) + t = a[6] ^ d1 + bc0 = bits.RotateLeft64(t, 1) + t = a[22] ^ d2 + bc1 = bits.RotateLeft64(t, 6) + t = a[13] ^ d3 + bc2 = bits.RotateLeft64(t, 25) + t = a[4] ^ d4 + bc3 = bits.RotateLeft64(t, 8) + a[15] = bc0 ^ (bc2 &^ bc1) + a[6] = bc1 ^ (bc3 &^ bc2) + a[22] = bc2 ^ (bc4 &^ bc3) + a[13] = bc3 ^ (bc0 &^ bc4) + a[4] = bc4 ^ (bc1 &^ bc0) + + t = a[10] ^ d0 + bc1 = bits.RotateLeft64(t, 36) + t = a[1] ^ d1 + bc2 = bits.RotateLeft64(t, 10) + t = a[17] ^ d2 + bc3 = bits.RotateLeft64(t, 15) + t = a[8] ^ d3 + bc4 = bits.RotateLeft64(t, 56) + t = a[24] ^ d4 + bc0 = bits.RotateLeft64(t, 27) + a[10] = bc0 ^ (bc2 &^ bc1) + a[1] = bc1 ^ (bc3 &^ bc2) + a[17] = bc2 ^ (bc4 &^ bc3) + a[8] = bc3 ^ (bc0 &^ bc4) + a[24] = bc4 ^ (bc1 &^ bc0) + + t = a[5] ^ d0 + bc3 = bits.RotateLeft64(t, 41) + t = a[21] ^ d1 + bc4 = bits.RotateLeft64(t, 2) + t = a[12] ^ d2 + bc0 = bits.RotateLeft64(t, 62) + t = a[3] ^ d3 + bc1 = bits.RotateLeft64(t, 55) + t = a[19] ^ d4 + bc2 = bits.RotateLeft64(t, 39) + a[5] = bc0 ^ (bc2 &^ bc1) + a[21] = bc1 ^ (bc3 &^ bc2) + a[12] = bc2 ^ (bc4 &^ bc3) + a[3] = bc3 ^ (bc0 &^ bc4) + a[19] = bc4 ^ (bc1 &^ bc0) + + // Round 3 + bc0 = a[0] ^ a[5] ^ a[10] ^ a[15] ^ a[20] + bc1 = a[1] ^ a[6] ^ a[11] ^ a[16] ^ a[21] + bc2 = a[2] ^ a[7] ^ a[12] ^ a[17] ^ a[22] + bc3 = a[3] ^ a[8] ^ a[13] ^ a[18] ^ a[23] + bc4 = a[4] ^ a[9] ^ a[14] ^ a[19] ^ a[24] + d0 = bc4 ^ (bc1<<1 | bc1>>63) + d1 = bc0 ^ (bc2<<1 | bc2>>63) + d2 = bc1 ^ (bc3<<1 | bc3>>63) + d3 = bc2 ^ (bc4<<1 | bc4>>63) + d4 = bc3 ^ (bc0<<1 | bc0>>63) + + bc0 = a[0] ^ d0 + t = a[11] ^ d1 + bc1 = bits.RotateLeft64(t, 44) + t = a[22] ^ d2 + bc2 = bits.RotateLeft64(t, 43) + t = a[8] ^ d3 + bc3 = bits.RotateLeft64(t, 21) + t = a[19] ^ d4 + bc4 = bits.RotateLeft64(t, 14) + a[0] = bc0 ^ (bc2 &^ bc1) ^ rc[i+2] + a[11] = bc1 ^ (bc3 &^ bc2) + a[22] = bc2 ^ (bc4 &^ bc3) + a[8] = bc3 ^ (bc0 &^ bc4) + a[19] = bc4 ^ (bc1 &^ bc0) + + t = a[15] ^ d0 + bc2 = bits.RotateLeft64(t, 3) + t = a[1] ^ d1 + bc3 = bits.RotateLeft64(t, 45) + t = a[12] ^ d2 + bc4 = bits.RotateLeft64(t, 61) + t = a[23] ^ d3 + bc0 = bits.RotateLeft64(t, 28) + t = a[9] ^ d4 + bc1 = bits.RotateLeft64(t, 20) + a[15] = bc0 ^ (bc2 &^ bc1) + a[1] = bc1 ^ (bc3 &^ bc2) + a[12] = bc2 ^ (bc4 &^ bc3) + a[23] = bc3 ^ (bc0 &^ bc4) + a[9] = bc4 ^ (bc1 &^ bc0) + + t = a[5] ^ d0 + bc4 = bits.RotateLeft64(t, 18) + t = a[16] ^ d1 + bc0 = bits.RotateLeft64(t, 1) + t = a[2] ^ d2 + bc1 = bits.RotateLeft64(t, 6) + t = a[13] ^ d3 + bc2 = bits.RotateLeft64(t, 25) + t = a[24] ^ d4 + bc3 = bits.RotateLeft64(t, 8) + a[5] = bc0 ^ (bc2 &^ bc1) + a[16] = bc1 ^ (bc3 &^ bc2) + a[2] = bc2 ^ (bc4 &^ bc3) + a[13] = bc3 ^ (bc0 &^ bc4) + a[24] = bc4 ^ (bc1 &^ bc0) + + t = a[20] ^ d0 + bc1 = bits.RotateLeft64(t, 36) + t = a[6] ^ d1 + bc2 = bits.RotateLeft64(t, 10) + t = a[17] ^ d2 + bc3 = bits.RotateLeft64(t, 15) + t = a[3] ^ d3 + bc4 = bits.RotateLeft64(t, 56) + t = a[14] ^ d4 + bc0 = bits.RotateLeft64(t, 27) + a[20] = bc0 ^ (bc2 &^ bc1) + a[6] = bc1 ^ (bc3 &^ bc2) + a[17] = bc2 ^ (bc4 &^ bc3) + a[3] = bc3 ^ (bc0 &^ bc4) + a[14] = bc4 ^ (bc1 &^ bc0) + + t = a[10] ^ d0 + bc3 = bits.RotateLeft64(t, 41) + t = a[21] ^ d1 + bc4 = bits.RotateLeft64(t, 2) + t = a[7] ^ d2 + bc0 = bits.RotateLeft64(t, 62) + t = a[18] ^ d3 + bc1 = bits.RotateLeft64(t, 55) + t = a[4] ^ d4 + bc2 = bits.RotateLeft64(t, 39) + a[10] = bc0 ^ (bc2 &^ bc1) + a[21] = bc1 ^ (bc3 &^ bc2) + a[7] = bc2 ^ (bc4 &^ bc3) + a[18] = bc3 ^ (bc0 &^ bc4) + a[4] = bc4 ^ (bc1 &^ bc0) + + // Round 4 + bc0 = a[0] ^ a[5] ^ a[10] ^ a[15] ^ a[20] + bc1 = a[1] ^ a[6] ^ a[11] ^ a[16] ^ a[21] + bc2 = a[2] ^ a[7] ^ a[12] ^ a[17] ^ a[22] + bc3 = a[3] ^ a[8] ^ a[13] ^ a[18] ^ a[23] + bc4 = a[4] ^ a[9] ^ a[14] ^ a[19] ^ a[24] + d0 = bc4 ^ (bc1<<1 | bc1>>63) + d1 = bc0 ^ (bc2<<1 | bc2>>63) + d2 = bc1 ^ (bc3<<1 | bc3>>63) + d3 = bc2 ^ (bc4<<1 | bc4>>63) + d4 = bc3 ^ (bc0<<1 | bc0>>63) + + bc0 = a[0] ^ d0 + t = a[1] ^ d1 + bc1 = bits.RotateLeft64(t, 44) + t = a[2] ^ d2 + bc2 = bits.RotateLeft64(t, 43) + t = a[3] ^ d3 + bc3 = bits.RotateLeft64(t, 21) + t = a[4] ^ d4 + bc4 = bits.RotateLeft64(t, 14) + a[0] = bc0 ^ (bc2 &^ bc1) ^ rc[i+3] + a[1] = bc1 ^ (bc3 &^ bc2) + a[2] = bc2 ^ (bc4 &^ bc3) + a[3] = bc3 ^ (bc0 &^ bc4) + a[4] = bc4 ^ (bc1 &^ bc0) + + t = a[5] ^ d0 + bc2 = bits.RotateLeft64(t, 3) + t = a[6] ^ d1 + bc3 = bits.RotateLeft64(t, 45) + t = a[7] ^ d2 + bc4 = bits.RotateLeft64(t, 61) + t = a[8] ^ d3 + bc0 = bits.RotateLeft64(t, 28) + t = a[9] ^ d4 + bc1 = bits.RotateLeft64(t, 20) + a[5] = bc0 ^ (bc2 &^ bc1) + a[6] = bc1 ^ (bc3 &^ bc2) + a[7] = bc2 ^ (bc4 &^ bc3) + a[8] = bc3 ^ (bc0 &^ bc4) + a[9] = bc4 ^ (bc1 &^ bc0) + + t = a[10] ^ d0 + bc4 = bits.RotateLeft64(t, 18) + t = a[11] ^ d1 + bc0 = bits.RotateLeft64(t, 1) + t = a[12] ^ d2 + bc1 = bits.RotateLeft64(t, 6) + t = a[13] ^ d3 + bc2 = bits.RotateLeft64(t, 25) + t = a[14] ^ d4 + bc3 = bits.RotateLeft64(t, 8) + a[10] = bc0 ^ (bc2 &^ bc1) + a[11] = bc1 ^ (bc3 &^ bc2) + a[12] = bc2 ^ (bc4 &^ bc3) + a[13] = bc3 ^ (bc0 &^ bc4) + a[14] = bc4 ^ (bc1 &^ bc0) + + t = a[15] ^ d0 + bc1 = bits.RotateLeft64(t, 36) + t = a[16] ^ d1 + bc2 = bits.RotateLeft64(t, 10) + t = a[17] ^ d2 + bc3 = bits.RotateLeft64(t, 15) + t = a[18] ^ d3 + bc4 = bits.RotateLeft64(t, 56) + t = a[19] ^ d4 + bc0 = bits.RotateLeft64(t, 27) + a[15] = bc0 ^ (bc2 &^ bc1) + a[16] = bc1 ^ (bc3 &^ bc2) + a[17] = bc2 ^ (bc4 &^ bc3) + a[18] = bc3 ^ (bc0 &^ bc4) + a[19] = bc4 ^ (bc1 &^ bc0) + + t = a[20] ^ d0 + bc3 = bits.RotateLeft64(t, 41) + t = a[21] ^ d1 + bc4 = bits.RotateLeft64(t, 2) + t = a[22] ^ d2 + bc0 = bits.RotateLeft64(t, 62) + t = a[23] ^ d3 + bc1 = bits.RotateLeft64(t, 55) + t = a[24] ^ d4 + bc2 = bits.RotateLeft64(t, 39) + a[20] = bc0 ^ (bc2 &^ bc1) + a[21] = bc1 ^ (bc3 &^ bc2) + a[22] = bc2 ^ (bc4 &^ bc3) + a[23] = bc3 ^ (bc0 &^ bc4) + a[24] = bc4 ^ (bc1 &^ bc0) + } +} diff --git a/crypto/keccak/keccakf_amd64.go b/crypto/keccak/keccakf_amd64.go new file mode 100644 index 0000000000..cb6eca44c3 --- /dev/null +++ b/crypto/keccak/keccakf_amd64.go @@ -0,0 +1,13 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build amd64 && !purego && gc + +package keccak + +// This function is implemented in keccakf_amd64.s. + +//go:noescape + +func keccakF1600(a *[25]uint64) diff --git a/crypto/keccak/keccakf_amd64.s b/crypto/keccak/keccakf_amd64.s new file mode 100644 index 0000000000..99e2f16e97 --- /dev/null +++ b/crypto/keccak/keccakf_amd64.s @@ -0,0 +1,5419 @@ +// Code generated by command: go run keccakf_amd64_asm.go -out ../keccakf_amd64.s -pkg sha3. DO NOT EDIT. + +//go:build amd64 && !purego && gc + +// func keccakF1600(a *[25]uint64) +TEXT ·keccakF1600(SB), $200-8 + MOVQ a+0(FP), DI + + // Convert the user state into an internal state + NOTQ 8(DI) + NOTQ 16(DI) + NOTQ 64(DI) + NOTQ 96(DI) + NOTQ 136(DI) + NOTQ 160(DI) + + // Execute the KeccakF permutation + MOVQ (DI), SI + MOVQ 8(DI), BP + MOVQ 32(DI), R15 + XORQ 40(DI), SI + XORQ 48(DI), BP + XORQ 72(DI), R15 + XORQ 80(DI), SI + XORQ 88(DI), BP + XORQ 112(DI), R15 + XORQ 120(DI), SI + XORQ 128(DI), BP + XORQ 152(DI), R15 + XORQ 160(DI), SI + XORQ 168(DI), BP + MOVQ 176(DI), DX + MOVQ 184(DI), R8 + XORQ 192(DI), R15 + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x0000000000000001, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x0000000000008082, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x800000000000808a, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000080008000, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x000000000000808b, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x0000000080000001, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000080008081, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000000008009, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x000000000000008a, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x0000000000000088, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x0000000080008009, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x000000008000000a, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x000000008000808b, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x800000000000008b, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000000008089, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000000008003, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000000008002, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000000000080, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x000000000000800a, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x800000008000000a, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000080008081, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000000008080, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x0000000080000001, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000080008008, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + NOP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + NOP + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + NOP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + NOP + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + NOP + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + NOP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + NOP + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + NOP + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + NOP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + NOP + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + NOP + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + NOP + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + NOP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Revert the internal state to the user state + NOTQ 8(DI) + NOTQ 16(DI) + NOTQ 64(DI) + NOTQ 96(DI) + NOTQ 136(DI) + NOTQ 160(DI) + RET diff --git a/crypto/keccak/sha3.go b/crypto/keccak/sha3.go new file mode 100644 index 0000000000..a554323244 --- /dev/null +++ b/crypto/keccak/sha3.go @@ -0,0 +1,244 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package keccak + +import ( + "crypto/subtle" + "encoding/binary" + "errors" + "unsafe" + + "golang.org/x/sys/cpu" +) + +// spongeDirection indicates the direction bytes are flowing through the sponge. +type spongeDirection int + +const ( + // spongeAbsorbing indicates that the sponge is absorbing input. + spongeAbsorbing spongeDirection = iota + // spongeSqueezing indicates that the sponge is being squeezed. + spongeSqueezing +) + +type state struct { + a [1600 / 8]byte // main state of the hash + + // a[n:rate] is the buffer. If absorbing, it's the remaining space to XOR + // into before running the permutation. If squeezing, it's the remaining + // output to produce before running the permutation. + n, rate int + + // dsbyte contains the "domain separation" bits and the first bit of + // the padding. Sections 6.1 and 6.2 of [1] separate the outputs of the + // SHA-3 and SHAKE functions by appending bitstrings to the message. + // Using a little-endian bit-ordering convention, these are "01" for SHA-3 + // and "1111" for SHAKE, or 00000010b and 00001111b, respectively. Then the + // padding rule from section 5.1 is applied to pad the message to a multiple + // of the rate, which involves adding a "1" bit, zero or more "0" bits, and + // a final "1" bit. We merge the first "1" bit from the padding into dsbyte, + // giving 00000110b (0x06) and 00011111b (0x1f). + // [1] http://csrc.nist.gov/publications/drafts/fips-202/fips_202_draft.pdf + // "Draft FIPS 202: SHA-3 Standard: Permutation-Based Hash and + // Extendable-Output Functions (May 2014)" + dsbyte byte + + outputLen int // the default output size in bytes + state spongeDirection // whether the sponge is absorbing or squeezing +} + +// BlockSize returns the rate of sponge underlying this hash function. +func (d *state) BlockSize() int { return d.rate } + +// Size returns the output size of the hash function in bytes. +func (d *state) Size() int { return d.outputLen } + +// Reset clears the internal state by zeroing the sponge state and +// the buffer indexes, and setting Sponge.state to absorbing. +func (d *state) Reset() { + // Zero the permutation's state. + for i := range d.a { + d.a[i] = 0 + } + d.state = spongeAbsorbing + d.n = 0 +} + +func (d *state) clone() *state { + ret := *d + return &ret +} + +// permute applies the KeccakF-1600 permutation. +func (d *state) permute() { + var a *[25]uint64 + if cpu.IsBigEndian { + a = new([25]uint64) + for i := range a { + a[i] = binary.LittleEndian.Uint64(d.a[i*8:]) + } + } else { + a = (*[25]uint64)(unsafe.Pointer(&d.a)) + } + + keccakF1600(a) + d.n = 0 + + if cpu.IsBigEndian { + for i := range a { + binary.LittleEndian.PutUint64(d.a[i*8:], a[i]) + } + } +} + +// pads appends the domain separation bits in dsbyte, applies +// the multi-bitrate 10..1 padding rule, and permutes the state. +func (d *state) padAndPermute() { + // Pad with this instance's domain-separator bits. We know that there's + // at least one byte of space in the sponge because, if it were full, + // permute would have been called to empty it. dsbyte also contains the + // first one bit for the padding. See the comment in the state struct. + d.a[d.n] ^= d.dsbyte + // This adds the final one bit for the padding. Because of the way that + // bits are numbered from the LSB upwards, the final bit is the MSB of + // the last byte. + d.a[d.rate-1] ^= 0x80 + // Apply the permutation + d.permute() + d.state = spongeSqueezing +} + +// Write absorbs more data into the hash's state. It panics if any +// output has already been read. +func (d *state) Write(p []byte) (n int, err error) { + if d.state != spongeAbsorbing { + panic("sha3: Write after Read") + } + + n = len(p) + + for len(p) > 0 { + x := subtle.XORBytes(d.a[d.n:d.rate], d.a[d.n:d.rate], p) + d.n += x + p = p[x:] + + // If the sponge is full, apply the permutation. + if d.n == d.rate { + d.permute() + } + } + + return +} + +// Read squeezes an arbitrary number of bytes from the sponge. +func (d *state) Read(out []byte) (n int, err error) { + // If we're still absorbing, pad and apply the permutation. + if d.state == spongeAbsorbing { + d.padAndPermute() + } + + n = len(out) + + // Now, do the squeezing. + for len(out) > 0 { + // Apply the permutation if we've squeezed the sponge dry. + if d.n == d.rate { + d.permute() + } + + x := copy(out, d.a[d.n:d.rate]) + d.n += x + out = out[x:] + } + + return +} + +// Sum applies padding to the hash state and then squeezes out the desired +// number of output bytes. It panics if any output has already been read. +func (d *state) Sum(in []byte) []byte { + if d.state != spongeAbsorbing { + panic("sha3: Sum after Read") + } + + // Make a copy of the original hash so that caller can keep writing + // and summing. + dup := d.clone() + hash := make([]byte, dup.outputLen, 64) // explicit cap to allow stack allocation + dup.Read(hash) + return append(in, hash...) +} + +const ( + magicSHA3 = "sha\x08" + magicShake = "sha\x09" + magicCShake = "sha\x0a" + magicKeccak = "sha\x0b" + // magic || rate || main state || n || sponge direction + marshaledSize = len(magicSHA3) + 1 + 200 + 1 + 1 +) + +func (d *state) MarshalBinary() ([]byte, error) { + return d.AppendBinary(make([]byte, 0, marshaledSize)) +} + +func (d *state) AppendBinary(b []byte) ([]byte, error) { + switch d.dsbyte { + case dsbyteSHA3: + b = append(b, magicSHA3...) + case dsbyteShake: + b = append(b, magicShake...) + case dsbyteCShake: + b = append(b, magicCShake...) + case dsbyteKeccak: + b = append(b, magicKeccak...) + default: + panic("unknown dsbyte") + } + // rate is at most 168, and n is at most rate. + b = append(b, byte(d.rate)) + b = append(b, d.a[:]...) + b = append(b, byte(d.n), byte(d.state)) + return b, nil +} + +func (d *state) UnmarshalBinary(b []byte) error { + if len(b) != marshaledSize { + return errors.New("sha3: invalid hash state") + } + + magic := string(b[:len(magicSHA3)]) + b = b[len(magicSHA3):] + switch { + case magic == magicSHA3 && d.dsbyte == dsbyteSHA3: + case magic == magicShake && d.dsbyte == dsbyteShake: + case magic == magicCShake && d.dsbyte == dsbyteCShake: + case magic == magicKeccak && d.dsbyte == dsbyteKeccak: + default: + return errors.New("sha3: invalid hash state identifier") + } + + rate := int(b[0]) + b = b[1:] + if rate != d.rate { + return errors.New("sha3: invalid hash state function") + } + + copy(d.a[:], b) + b = b[len(d.a):] + + n, state := int(b[0]), spongeDirection(b[1]) + if n > d.rate { + return errors.New("sha3: invalid hash state") + } + d.n = n + if state != spongeAbsorbing && state != spongeSqueezing { + return errors.New("sha3: invalid hash state") + } + d.state = state + + return nil +} diff --git a/crypto/keccak/sha3_test.go b/crypto/keccak/sha3_test.go new file mode 100644 index 0000000000..28a20ec72d --- /dev/null +++ b/crypto/keccak/sha3_test.go @@ -0,0 +1,210 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package keccak + +// Tests include all the ShortMsgKATs provided by the Keccak team at +// https://github.com/gvanas/KeccakCodePackage +// +// They only include the zero-bit case of the bitwise testvectors +// published by NIST in the draft of FIPS-202. + +import ( + "bytes" + "compress/flate" + "encoding" + "encoding/hex" + "encoding/json" + "hash" + "math/rand" + "os" + "strings" + "testing" +) + +const ( + testString = "brekeccakkeccak koax koax" + katFilename = "testdata/keccakKats.json.deflate" +) + +// testDigests contains functions returning hash.Hash instances +// with output-length equal to the KAT length for SHA-3, Keccak +// and SHAKE instances. +var testDigests = map[string]func() hash.Hash{ + "Keccak-256": NewLegacyKeccak256, + "Keccak-512": NewLegacyKeccak512, +} + +// decodeHex converts a hex-encoded string into a raw byte string. +func decodeHex(s string) []byte { + b, err := hex.DecodeString(s) + if err != nil { + panic(err) + } + return b +} + +// structs used to marshal JSON test-cases. +type KeccakKats struct { + Kats map[string][]struct { + Digest string `json:"digest"` + Length int64 `json:"length"` + Message string `json:"message"` + + // Defined only for cSHAKE + N string `json:"N"` + S string `json:"S"` + } +} + +// TestKeccakKats tests the SHA-3 and Shake implementations against all the +// ShortMsgKATs from https://github.com/gvanas/KeccakCodePackage +// (The testvectors are stored in keccakKats.json.deflate due to their length.) +func TestKeccakKats(t *testing.T) { + // Read the KATs. + deflated, err := os.Open(katFilename) + if err != nil { + t.Errorf("error opening %s: %s", katFilename, err) + } + file := flate.NewReader(deflated) + dec := json.NewDecoder(file) + var katSet KeccakKats + err = dec.Decode(&katSet) + if err != nil { + t.Errorf("error decoding KATs: %s", err) + } + + for algo, function := range testDigests { + d := function() + for _, kat := range katSet.Kats[algo] { + d.Reset() + in, err := hex.DecodeString(kat.Message) + if err != nil { + t.Errorf("error decoding KAT: %s", err) + } + d.Write(in[:kat.Length/8]) + got := strings.ToUpper(hex.EncodeToString(d.Sum(nil))) + if got != kat.Digest { + t.Errorf("function=%s, length=%d\nmessage:\n %s\ngot:\n %s\nwanted:\n %s", + algo, kat.Length, kat.Message, got, kat.Digest) + t.Logf("wanted %+v", kat) + t.FailNow() + } + continue + } + } +} + +// TestKeccak does a basic test of the non-standardized Keccak hash functions. +func TestKeccak(t *testing.T) { + tests := []struct { + fn func() hash.Hash + data []byte + want string + }{ + { + NewLegacyKeccak256, + []byte("abc"), + "4e03657aea45a94fc7d47ba826c8d667c0d1e6e33a64a036ec44f58fa12d6c45", + }, + { + NewLegacyKeccak512, + []byte("abc"), + "18587dc2ea106b9a1563e32b3312421ca164c7f1f07bc922a9c83d77cea3a1e5d0c69910739025372dc14ac9642629379540c17e2a65b19d77aa511a9d00bb96", + }, + } + + for _, u := range tests { + h := u.fn() + h.Write(u.data) + got := h.Sum(nil) + want := decodeHex(u.want) + if !bytes.Equal(got, want) { + t.Errorf("unexpected hash for size %d: got '%x' want '%s'", h.Size()*8, got, u.want) + } + } +} + +// TestUnalignedWrite tests that writing data in an arbitrary pattern with +// small input buffers. +func TestUnalignedWrite(t *testing.T) { + buf := sequentialBytes(0x10000) + for alg, df := range testDigests { + d := df() + d.Reset() + d.Write(buf) + want := d.Sum(nil) + d.Reset() + for i := 0; i < len(buf); { + // Cycle through offsets which make a 137 byte sequence. + // Because 137 is prime this sequence should exercise all corner cases. + offsets := [17]int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1} + for _, j := range offsets { + if v := len(buf) - i; v < j { + j = v + } + d.Write(buf[i : i+j]) + i += j + } + } + got := d.Sum(nil) + if !bytes.Equal(got, want) { + t.Errorf("Unaligned writes, alg=%s\ngot %q, want %q", alg, got, want) + } + } +} + +// sequentialBytes produces a buffer of size consecutive bytes 0x00, 0x01, ..., used for testing. +// +// The alignment of each slice is intentionally randomized to detect alignment +// issues in the implementation. See https://golang.org/issue/37644. +// Ideally, the compiler should fuzz the alignment itself. +// (See https://golang.org/issue/35128.) +func sequentialBytes(size int) []byte { + alignmentOffset := rand.Intn(8) + result := make([]byte, size+alignmentOffset)[alignmentOffset:] + for i := range result { + result[i] = byte(i) + } + return result +} + +func TestMarshalUnmarshal(t *testing.T) { + t.Run("Keccak-256", func(t *testing.T) { testMarshalUnmarshal(t, NewLegacyKeccak256()) }) + t.Run("Keccak-512", func(t *testing.T) { testMarshalUnmarshal(t, NewLegacyKeccak512()) }) +} + +// TODO(filippo): move this to crypto/internal/cryptotest. +func testMarshalUnmarshal(t *testing.T, h hash.Hash) { + buf := make([]byte, 200) + rand.Read(buf) + n := rand.Intn(200) + h.Write(buf) + want := h.Sum(nil) + h.Reset() + h.Write(buf[:n]) + b, err := h.(encoding.BinaryMarshaler).MarshalBinary() + if err != nil { + t.Errorf("MarshalBinary: %v", err) + } + h.Write(bytes.Repeat([]byte{0}, 200)) + if err := h.(encoding.BinaryUnmarshaler).UnmarshalBinary(b); err != nil { + t.Errorf("UnmarshalBinary: %v", err) + } + h.Write(buf[n:]) + got := h.Sum(nil) + if !bytes.Equal(got, want) { + t.Errorf("got %x, want %x", got, want) + } +} + +// BenchmarkPermutationFunction measures the speed of the permutation function +// with no input data. +func BenchmarkPermutationFunction(b *testing.B) { + b.SetBytes(int64(200)) + var lanes [25]uint64 + for i := 0; i < b.N; i++ { + keccakF1600(&lanes) + } +} diff --git a/crypto/keccak/testdata/keccakKats.json.deflate b/crypto/keccak/testdata/keccakKats.json.deflate new file mode 100644 index 0000000000..7a94c2f8bc Binary files /dev/null and b/crypto/keccak/testdata/keccakKats.json.deflate differ diff --git a/crypto/kzg4844/kzg4844_ckzg_cgo.go b/crypto/kzg4844/kzg4844_ckzg_cgo.go index 46509674b6..93d5f4ff94 100644 --- a/crypto/kzg4844/kzg4844_ckzg_cgo.go +++ b/crypto/kzg4844/kzg4844_ckzg_cgo.go @@ -143,9 +143,9 @@ func ckzgComputeCellProofs(blob *Blob) ([]Proof, error) { if err != nil { return []Proof{}, err } - var p []Proof - for _, proof := range proofs { - p = append(p, (Proof)(proof)) + p := make([]Proof, len(proofs)) + for i, proof := range proofs { + p[i] = (Proof)(proof) } return p, nil } diff --git a/crypto/kzg4844/kzg4844_gokzg.go b/crypto/kzg4844/kzg4844_gokzg.go index e9676ff1b8..03627ebafb 100644 --- a/crypto/kzg4844/kzg4844_gokzg.go +++ b/crypto/kzg4844/kzg4844_gokzg.go @@ -108,9 +108,9 @@ func gokzgComputeCellProofs(blob *Blob) ([]Proof, error) { if err != nil { return []Proof{}, err } - var p []Proof - for _, proof := range proofs { - p = append(p, (Proof)(proof)) + p := make([]Proof, len(proofs)) + for i, proof := range proofs { + p[i] = (Proof)(proof) } return p, nil } diff --git a/crypto/secp256k1/curve.go b/crypto/secp256k1/curve.go index b82b147e3c..504602f5be 100644 --- a/crypto/secp256k1/curve.go +++ b/crypto/secp256k1/curve.go @@ -73,6 +73,10 @@ func (bitCurve *BitCurve) Params() *elliptic.CurveParams { // IsOnCurve returns true if the given (x,y) lies on the BitCurve. func (bitCurve *BitCurve) IsOnCurve(x, y *big.Int) bool { + if x.Cmp(bitCurve.P) >= 0 || y.Cmp(bitCurve.P) >= 0 { + return false + } + // y² = x³ + b y2 := new(big.Int).Mul(y, y) //y² y2.Mod(y2, bitCurve.P) //y²%P diff --git a/crypto/secp256k1/ext.h b/crypto/secp256k1/ext.h index 1c485e2603..baafb4404b 100644 --- a/crypto/secp256k1/ext.h +++ b/crypto/secp256k1/ext.h @@ -109,8 +109,10 @@ int secp256k1_ext_scalar_mul(const secp256k1_context* ctx, unsigned char *point, ARG_CHECK(scalar != NULL); (void)ctx; - secp256k1_fe_set_b32_limit(&feX, point); - secp256k1_fe_set_b32_limit(&feY, point+32); + if (!secp256k1_fe_set_b32_limit(&feX, point) || + !secp256k1_fe_set_b32_limit(&feY, point+32)) { + return 0; + } secp256k1_ge_set_xy(&ge, &feX, &feY); secp256k1_scalar_set_b32(&s, scalar, &overflow); if (overflow || secp256k1_scalar_is_zero(&s)) { diff --git a/crypto/signature_nocgo.go b/crypto/signature_nocgo.go index 014e166b7c..0dc360ee50 100644 --- a/crypto/signature_nocgo.go +++ b/crypto/signature_nocgo.go @@ -65,8 +65,8 @@ func SigToPub(hash, sig []byte) (*ecdsa.PublicKey, error) { // // The produced signature is in the [R || S || V] format where V is 0 or 1. func Sign(hash []byte, prv *ecdsa.PrivateKey) ([]byte, error) { - if len(hash) != 32 { - return nil, fmt.Errorf("hash is required to be exactly 32 bytes (%d)", len(hash)) + if len(hash) != DigestLength { + return nil, fmt.Errorf("hash is required to be exactly %d bytes (%d)", DigestLength, len(hash)) } if prv.Curve != S256() { return nil, errors.New("private key curve is not secp256k1") @@ -153,6 +153,13 @@ type btCurve struct { *secp256k1.KoblitzCurve } +func (curve btCurve) IsOnCurve(x, y *big.Int) bool { + if x.Cmp(secp256k1.Params().P) >= 0 || y.Cmp(secp256k1.Params().P) >= 0 { + return false + } + return curve.KoblitzCurve.IsOnCurve(x, y) +} + // Marshal converts a point given as (x, y) into a byte slice. func (curve btCurve) Marshal(x, y *big.Int) []byte { byteLen := (curve.Params().BitSize + 7) / 8 diff --git a/crypto/signify/signify_fuzz.go b/crypto/signify/signify_fuzz.go index 239a2134df..d11125c697 100644 --- a/crypto/signify/signify_fuzz.go +++ b/crypto/signify/signify_fuzz.go @@ -56,7 +56,7 @@ func Fuzz(data []byte) int { fmt.Printf("untrusted: %v\n", untrustedComment) fmt.Printf("trusted: %v\n", trustedComment) - err = SignifySignFile(tmpFile.Name(), tmpFile.Name()+".sig", testSecKey, untrustedComment, trustedComment) + err = SignFile(tmpFile.Name(), tmpFile.Name()+".sig", testSecKey, untrustedComment, trustedComment) if err != nil { panic(err) } @@ -68,7 +68,7 @@ func Fuzz(data []byte) int { signify = path } - _, err := exec.LookPath(signify) + _, err = exec.LookPath(signify) if err != nil { panic(err) } diff --git a/eth/api_backend.go b/eth/api_backend.go index e57d3c95d4..75b772c189 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -319,6 +319,11 @@ func (b *EthAPIBackend) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) e return b.eth.BlockChain().SubscribeChainHeadEvent(ch) } +// SubscribeNewPayloadEvent registers a subscription for NewPayloadEvent. +func (b *EthAPIBackend) SubscribeNewPayloadEvent(ch chan<- core.NewPayloadEvent) event.Subscription { + return b.eth.BlockChain().SubscribeNewPayloadEvent(ch) +} + func (b *EthAPIBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { return b.eth.BlockChain().SubscribeLogsEvent(ch) } @@ -346,7 +351,7 @@ func (b *EthAPIBackend) SendTx(ctx context.Context, signedTx *types.Transaction) } func (b *EthAPIBackend) GetPoolTransactions() (types.Transactions, error) { - pending := b.eth.txPool.Pending(txpool.PendingFilter{}) + pending, _ := b.eth.txPool.Pending(txpool.PendingFilter{}) var txs types.Transactions for _, batch := range pending { for _, lazy := range batch { diff --git a/eth/api_debug.go b/eth/api_debug.go index c5bf1c359e..2dddba82f0 100644 --- a/eth/api_debug.go +++ b/eth/api_debug.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/stateless" @@ -503,71 +504,60 @@ func (api *DebugAPI) StateSize(blockHashOrNumber *rpc.BlockNumberOrHash) (interf }, nil } -func (api *DebugAPI) ExecutionWitness(bn rpc.BlockNumber) (*stateless.ExtWitness, error) { +func (api *DebugAPI) ExecutionWitness(bn rpc.BlockNumberOrHash) (*stateless.ExtWitness, error) { bc := api.eth.blockchain var block *types.Block - switch bn { - case rpc.PendingBlockNumber: - return &stateless.ExtWitness{}, errors.New("pending block is not supported for execution witness") - case rpc.EarliestBlockNumber: - return &stateless.ExtWitness{}, errors.New("earliest block is not supported for execution witness") - case rpc.LatestBlockNumber: - latestBlock := bc.CurrentBlock() - if latestBlock == nil { - return &stateless.ExtWitness{}, errors.New("latest block not found") - } - block = bc.GetBlock(latestBlock.Hash(), latestBlock.Number.Uint64()) - case rpc.FinalizedBlockNumber: - finalizedBlock := bc.CurrentFinalBlock() - if finalizedBlock == nil { - return &stateless.ExtWitness{}, errors.New("finalized block not found") - } - block = bc.GetBlock(finalizedBlock.Hash(), finalizedBlock.Number.Uint64()) - case rpc.SafeBlockNumber: - safeBlock := bc.CurrentSafeBlock() - if safeBlock == nil { - return &stateless.ExtWitness{}, errors.New("safe block not found") - } - block = bc.GetBlock(safeBlock.Hash(), safeBlock.Number.Uint64()) - default: - if bn < 0 { - return &stateless.ExtWitness{}, fmt.Errorf("negative block numbers are not unsupported: %v", bn) + blockNumber, isNumber := bn.Number() + if isNumber { + switch blockNumber { + case rpc.PendingBlockNumber: + return &stateless.ExtWitness{}, errors.New("pending block is not supported for execution witness") + case rpc.EarliestBlockNumber: + return &stateless.ExtWitness{}, errors.New("earliest block is not supported for execution witness") + case rpc.LatestBlockNumber: + latestBlock := bc.CurrentBlock() + if latestBlock == nil { + return &stateless.ExtWitness{}, errors.New("latest block not found") + } + block = bc.GetBlock(latestBlock.Hash(), latestBlock.Number.Uint64()) + case rpc.FinalizedBlockNumber: + finalizedBlock := bc.CurrentFinalBlock() + if finalizedBlock == nil { + return &stateless.ExtWitness{}, errors.New("finalized block not found") + } + block = bc.GetBlock(finalizedBlock.Hash(), finalizedBlock.Number.Uint64()) + case rpc.SafeBlockNumber: + safeBlock := bc.CurrentSafeBlock() + if safeBlock == nil { + return &stateless.ExtWitness{}, errors.New("safe block not found") + } + block = bc.GetBlock(safeBlock.Hash(), safeBlock.Number.Uint64()) + default: + if blockNumber < 0 { + return &stateless.ExtWitness{}, fmt.Errorf("negative block numbers are not unsupported: %v", blockNumber) + } + block = bc.GetBlockByNumber(uint64(blockNumber.Int64())) } - block = bc.GetBlockByNumber(uint64(bn.Int64())) + } + blockHash, isHash := bn.Hash() + if isHash { + block = bc.GetBlockByHash(blockHash) } if block == nil { return &stateless.ExtWitness{}, fmt.Errorf("block number %v not found", bn) } - parent := bc.GetHeader(block.ParentHash(), block.NumberU64()-1) if parent == nil { - return &stateless.ExtWitness{}, fmt.Errorf("block number %v found, but parent missing", bn) + return &stateless.ExtWitness{}, fmt.Errorf("block %v found, but parent missing", bn) } - - result, err := bc.ProcessBlock(parent.Root, block, false, true) - if err != nil { - return &stateless.ExtWitness{}, err - } - - return result.Witness().ToExtWitness(), nil -} - -func (api *DebugAPI) ExecutionWitnessByHash(hash common.Hash) (*stateless.ExtWitness, error) { - bc := api.eth.blockchain - block := bc.GetBlockByHash(hash) - if block == nil { - return &stateless.ExtWitness{}, fmt.Errorf("block hash %x not found", hash) + config := core.ExecuteConfig{ + WriteState: false, + EnableTracer: false, + MakeWitness: true, } - - parent := bc.GetHeader(block.ParentHash(), block.NumberU64()-1) - if parent == nil { - return &stateless.ExtWitness{}, fmt.Errorf("block number %x found, but parent missing", hash) - } - - result, err := bc.ProcessBlock(parent.Root, block, false, true) + result, err := bc.ProcessBlock(context.Background(), parent.Root, block, config) if err != nil { return &stateless.ExtWitness{}, err } - return result.Witness().ToExtWitness(), nil } diff --git a/eth/backend.go b/eth/backend.go index 6fded60a78..62f297461e 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/filtermaps" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/pruner" "github.com/ethereum/go-ethereum/core/txpool" @@ -175,7 +176,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { // Here we determine genesis hash and active ChainConfig. // We need these to figure out the consensus parameters and to set up history pruning. - chainConfig, _, err := core.LoadChainConfig(chainDb, config.Genesis) + chainConfig, genesisHash, err := core.LoadChainConfig(chainDb, config.Genesis) if err != nil { return nil, err } @@ -220,6 +221,10 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { rawdb.WriteDatabaseVersion(chainDb, core.BlockChainVersion) } } + histPolicy, err := history.NewPolicy(config.HistoryMode, genesisHash) + if err != nil { + return nil, err + } var ( options = &core.BlockChainConfig{ @@ -227,24 +232,24 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { TriesInMemory: 128, TrieRetention: 30 * time.Minute, - TrieCleanLimit: config.TrieCleanCache, - NoPrefetch: config.NoPrefetch, - TrieDirtyLimit: config.TrieDirtyCache, - ArchiveMode: config.NoPruning, - TrieTimeLimit: config.TrieTimeout, - SnapshotLimit: config.SnapshotCache, - Preimages: config.Preimages, - StateHistory: config.StateHistory, - MaxDiffLayers: config.MaxDiffLayers, - StateScheme: scheme, - ChainHistoryMode: config.HistoryMode, + TrieCleanLimit: config.TrieCleanCache, + NoPrefetch: config.NoPrefetch, + TrieDirtyLimit: config.TrieDirtyCache, + ArchiveMode: config.NoPruning, + TrieTimeLimit: config.TrieTimeout, + SnapshotLimit: config.SnapshotCache, + Preimages: config.Preimages, + StateHistory: config.StateHistory, + MaxDiffLayers: config.MaxDiffLayers, + TrienodeHistory: config.TrienodeHistory, + NodeFullValueCheckpoint: config.NodeFullValueCheckpoint, + StateScheme: scheme, + HistoryPolicy: histPolicy, TxIndexer: &core.TxIndexerConfig{ Limit: min(config.TransactionHistory, math.MaxUint64), }, VmConfig: vm.Config{ EnablePreimageRecording: config.EnablePreimageRecording, - EnableWitnessStats: config.EnableWitnessStats, - StatelessSelfValidation: config.StatelessSelfValidation, }, // Enables file journaling for the trie database. The journal files will be stored // within the data directory. The corresponding paths will be either: @@ -252,6 +257,10 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { // - DATADIR/triedb/verkle.journal TrieJournalDirectory: stack.ResolvePath("triedb"), StateSizeTracking: config.EnableStateSizeTracking, + SlowBlockThreshold: config.SlowBlockThreshold, + + StatelessSelfValidation: config.StatelessSelfValidation, + EnableWitnessStats: config.EnableWitnessStats, } ) if config.VMTrace != "" { @@ -500,6 +509,9 @@ func (s *Ethereum) updateFilterMapsHeads() { if head == nil || newHead.Hash() != head.Hash() { head = newHead chainView := s.newChainView(head) + if chainView == nil { + return + } historyCutoff, _ := s.blockchain.HistoryPruningCutoff() var finalBlock uint64 if fb := s.blockchain.CurrentFinalBlock(); fb != nil { @@ -599,29 +611,3 @@ func (s *Ethereum) Stop() error { return nil } - -// SyncMode retrieves the current sync mode, either explicitly set, or derived -// from the chain status. -func (s *Ethereum) SyncMode() ethconfig.SyncMode { - // If we're in snap sync mode, return that directly - if s.handler.snapSync.Load() { - return ethconfig.SnapSync - } - // We are probably in full sync, but we might have rewound to before the - // snap sync pivot, check if we should re-enable snap sync. - head := s.blockchain.CurrentBlock() - if pivot := rawdb.ReadLastPivotNumber(s.chainDb); pivot != nil { - if head.Number.Uint64() < *pivot { - return ethconfig.SnapSync - } - } - // We are in a full sync, but the associated head state is missing. To complete - // the head state, forcefully rerun the snap sync. Note it doesn't mean the - // persistent state is corrupted, just mismatch with the head block. - if !s.blockchain.HasState(head.Root) { - log.Info("Reenabled snap sync as chain is stateless") - return ethconfig.SnapSync - } - // Nope, we're really full syncing - return ethconfig.FullSync -} diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 6de9901bb3..54439d796b 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -18,6 +18,7 @@ package catalyst import ( + "context" "errors" "fmt" "reflect" @@ -30,13 +31,14 @@ import ( "github.com/ethereum/go-ethereum/beacon/engine" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/internal/telemetry" "github.com/ethereum/go-ethereum/internal/version" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" @@ -45,9 +47,10 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) -// Register adds the engine API to the full node. +// Register adds the engine API and related APIs to the full node. func Register(stack *node.Node, backend *eth.Ethereum) error { stack.RegisterAPIs([]rpc.API{ + newTestingAPI(backend), { Namespace: "engine", Service: NewConsensusAPI(backend), @@ -81,20 +84,6 @@ const ( beaconUpdateWarnFrequency = 5 * time.Minute ) -var ( - // Number of blobs requested via getBlobsV2 - getBlobsRequestedCounter = metrics.NewRegisteredCounter("engine/getblobs/requested", nil) - - // Number of blobs requested via getBlobsV2 that are present in the blobpool - getBlobsAvailableCounter = metrics.NewRegisteredCounter("engine/getblobs/available", nil) - - // Number of times getBlobsV2 responded with “hit” - getBlobsV2RequestHit = metrics.NewRegisteredCounter("engine/getblobs/hit", nil) - - // Number of times getBlobsV2 responded with “miss” - getBlobsV2RequestMiss = metrics.NewRegisteredCounter("engine/getblobs/miss", nil) -) - type ConsensusAPI struct { eth *eth.Ethereum @@ -137,6 +126,9 @@ type ConsensusAPI struct { // NewConsensusAPI creates a new consensus api for the given backend. // The underlying blockchain needs to have a valid terminal total difficulty set. +// +// This function creates a long-lived object with an attached background thread. +// For testing or other short-term use cases, please use newConsensusAPIWithoutHeartbeat. func NewConsensusAPI(eth *eth.Ethereum) *ConsensusAPI { api := newConsensusAPIWithoutHeartbeat(eth) go api.heartbeat() @@ -163,9 +155,6 @@ func newConsensusAPIWithoutHeartbeat(eth *eth.Ethereum) *ConsensusAPI { // // We try to set our blockchain to the headBlock. // -// If the method is called with an empty head block: we return success, which can be used -// to check if the engine API is enabled. -// // If the total difficulty was not reached: we return INVALID. // // If the finalizedBlockHash is set: we check if we have the finalizedBlockHash in our db, @@ -173,7 +162,7 @@ func newConsensusAPIWithoutHeartbeat(eth *eth.Ethereum) *ConsensusAPI { // // If there are payloadAttributes: we try to assemble a block with the payloadAttributes // and return its payloadID. -func (api *ConsensusAPI) ForkchoiceUpdatedV1(update engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { +func (api *ConsensusAPI) ForkchoiceUpdatedV1(ctx context.Context, update engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { if payloadAttributes != nil { switch { case payloadAttributes.Withdrawals != nil || payloadAttributes.BeaconRoot != nil: @@ -182,12 +171,12 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV1(update engine.ForkchoiceStateV1, pa return engine.STATUS_INVALID, paramsErr("fcuV1 called post-shanghai") } } - return api.forkchoiceUpdated(update, payloadAttributes, engine.PayloadV1, false) + return api.forkchoiceUpdated(ctx, update, payloadAttributes, engine.PayloadV1, false) } // ForkchoiceUpdatedV2 is equivalent to V1 with the addition of withdrawals in the payload // attributes. It supports both PayloadAttributesV1 and PayloadAttributesV2. -func (api *ConsensusAPI) ForkchoiceUpdatedV2(update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { +func (api *ConsensusAPI) ForkchoiceUpdatedV2(ctx context.Context, update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { if params != nil { switch { case params.BeaconRoot != nil: @@ -200,12 +189,12 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV2(update engine.ForkchoiceStateV1, pa return engine.STATUS_INVALID, unsupportedForkErr("fcuV2 must only be called with paris or shanghai payloads") } } - return api.forkchoiceUpdated(update, params, engine.PayloadV2, false) + return api.forkchoiceUpdated(ctx, update, params, engine.PayloadV2, false) } // ForkchoiceUpdatedV3 is equivalent to V2 with the addition of parent beacon block root // in the payload attributes. It supports only PayloadAttributesV3. -func (api *ConsensusAPI) ForkchoiceUpdatedV3(update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { +func (api *ConsensusAPI) ForkchoiceUpdatedV3(ctx context.Context, update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { if params != nil { switch { case params.Withdrawals == nil: @@ -220,10 +209,34 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV3(update engine.ForkchoiceStateV1, pa // hash, even if params are wrong. To do this we need to split up // forkchoiceUpdate into a function that only updates the head and then a // function that kicks off block construction. - return api.forkchoiceUpdated(update, params, engine.PayloadV3, false) + return api.forkchoiceUpdated(ctx, update, params, engine.PayloadV3, false) +} + +// ForkchoiceUpdatedV4 is equivalent to V3 with the addition of slot number +// in the payload attributes. It supports only PayloadAttributesV4. +func (api *ConsensusAPI) ForkchoiceUpdatedV4(ctx context.Context, update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { + if params != nil { + switch { + case params.Withdrawals == nil: + return engine.STATUS_INVALID, attributesErr("missing withdrawals") + case params.BeaconRoot == nil: + return engine.STATUS_INVALID, attributesErr("missing beacon root") + case params.SlotNumber == nil: + return engine.STATUS_INVALID, attributesErr("missing slot number") + case !api.checkFork(params.Timestamp, forks.Amsterdam): + return engine.STATUS_INVALID, unsupportedForkErr("fcuV4 must only be called for amsterdam payloads") + } + } + // TODO(matt): the spec requires that fcu is applied when called on a valid + // hash, even if params are wrong. To do this we need to split up + // forkchoiceUpdate into a function that only updates the head and then a + // function that kicks off block construction. + return api.forkchoiceUpdated(ctx, update, params, engine.PayloadV4, false) } -func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes, payloadVersion engine.PayloadVersion, payloadWitness bool) (engine.ForkChoiceResponse, error) { +func (api *ConsensusAPI) forkchoiceUpdated(ctx context.Context, update engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes, payloadVersion engine.PayloadVersion, payloadWitness bool) (result engine.ForkChoiceResponse, err error) { + ctx, _, spanEnd := telemetry.StartSpan(ctx, "engine.forkchoiceUpdated") + defer spanEnd(&err) api.forkchoiceLock.Lock() defer api.forkchoiceLock.Unlock() @@ -244,12 +257,9 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl if res := api.checkInvalidAncestor(update.HeadBlockHash, update.HeadBlockHash); res != nil { return engine.ForkChoiceResponse{PayloadStatus: *res, PayloadID: nil}, nil } - // If the head hash is unknown (was not given to us in a newPayload request), - // we cannot resolve the header, so not much to do. This could be extended in - // the future to resolve from the `eth` network, but it's an unexpected case - // that should be fixed, not papered over. header := api.remoteBlocks.get(update.HeadBlockHash) if header == nil { + // The head hash is unknown locally, try to resolve it from the `eth` network log.Warn("Fetching the unknown forkchoice head from network", "hash", update.HeadBlockHash) retrievedHead, err := api.eth.Downloader().GetHeader(update.HeadBlockHash) if err != nil { @@ -262,7 +272,9 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl // If the finalized hash is known, we can direct the downloader to move // potentially more data to the freezer from the get go. finalized := api.remoteBlocks.get(update.FinalizedBlockHash) - + if finalized == nil { + finalized = api.eth.BlockChain().GetHeaderByHash(update.FinalizedBlockHash) + } // Header advertised via a past newPayload request. Start syncing to it. context := []interface{}{"number", header.Number, "hash", header.Hash()} if update.FinalizedBlockHash != (common.Hash{}) { @@ -273,7 +285,7 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl } } log.Info("Forkchoice requested sync to new head", context...) - if err := api.eth.Downloader().BeaconSync(api.eth.SyncMode(), header, finalized); err != nil { + if err := api.eth.Downloader().BeaconSync(header, finalized); err != nil { return engine.STATUS_SYNCING, err } return engine.STATUS_SYNCING, nil @@ -356,6 +368,7 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl Random: payloadAttributes.Random, Withdrawals: payloadAttributes.Withdrawals, BeaconRoot: payloadAttributes.BeaconRoot, + SlotNum: payloadAttributes.SlotNumber, Version: payloadVersion, } id := args.Id() @@ -364,7 +377,7 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl if api.localBlocks.has(id) { return valid(&id), nil } - payload, err := api.eth.Miner().BuildPayload(args, payloadWitness) + payload, err := api.eth.Miner().BuildPayload(ctx, args, payloadWitness) if err != nil { log.Error("Failed to build payload", "err", err) return valid(nil), engine.InvalidPayloadAttributes.With(err) @@ -405,10 +418,12 @@ func (api *ConsensusAPI) ExchangeTransitionConfigurationV1(config engine.Transit // GetPayloadV1 returns a cached payload by id. func (api *ConsensusAPI) GetPayloadV1(payloadID engine.PayloadID) (*engine.ExecutableData, error) { - if !payloadID.Is(engine.PayloadV1) { - return nil, engine.UnsupportedFork - } - data, err := api.getPayload(payloadID, false) + data, err := api.getPayload( + payloadID, + false, + []engine.PayloadVersion{engine.PayloadV1}, + nil, + ) if err != nil { return nil, err } @@ -417,35 +432,34 @@ func (api *ConsensusAPI) GetPayloadV1(payloadID engine.PayloadID) (*engine.Execu // GetPayloadV2 returns a cached payload by id. func (api *ConsensusAPI) GetPayloadV2(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) { - // executionPayload: ExecutionPayloadV1 | ExecutionPayloadV2 where: - // - // - ExecutionPayloadV1 MUST be returned if the payload timestamp is lower - // than the Shanghai timestamp - // - // - ExecutionPayloadV2 MUST be returned if the payload timestamp is greater - // or equal to the Shanghai timestamp - if !payloadID.Is(engine.PayloadV1, engine.PayloadV2) { - return nil, engine.UnsupportedFork - } - return api.getPayload(payloadID, false) + return api.getPayload( + payloadID, + false, + []engine.PayloadVersion{engine.PayloadV1, engine.PayloadV2}, + []forks.Fork{forks.Paris, forks.Shanghai}, + ) } // GetPayloadV3 returns a cached payload by id. This endpoint should only // be used for the Cancun fork. func (api *ConsensusAPI) GetPayloadV3(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) { - if !payloadID.Is(engine.PayloadV3) { - return nil, engine.UnsupportedFork - } - return api.getPayload(payloadID, false) + return api.getPayload( + payloadID, + false, + []engine.PayloadVersion{engine.PayloadV3}, + []forks.Fork{forks.Cancun}, + ) } // GetPayloadV4 returns a cached payload by id. This endpoint should only // be used for the Prague fork. func (api *ConsensusAPI) GetPayloadV4(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) { - if !payloadID.Is(engine.PayloadV3) { - return nil, engine.UnsupportedFork - } - return api.getPayload(payloadID, false) + return api.getPayload( + payloadID, + false, + []engine.PayloadVersion{engine.PayloadV3}, + []forks.Fork{forks.Prague}, + ) } // GetPayloadV5 returns a cached payload by id. This endpoint should only @@ -454,18 +468,49 @@ func (api *ConsensusAPI) GetPayloadV4(payloadID engine.PayloadID) (*engine.Execu // This method follows the same specification as engine_getPayloadV4 with // changes of returning BlobsBundleV2 with BlobSidecar version 1. func (api *ConsensusAPI) GetPayloadV5(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) { - if !payloadID.Is(engine.PayloadV3) { - return nil, engine.UnsupportedFork - } - return api.getPayload(payloadID, false) + return api.getPayload( + payloadID, + false, + []engine.PayloadVersion{engine.PayloadV3}, + []forks.Fork{ + forks.Osaka, + forks.BPO1, + forks.BPO2, + forks.BPO3, + forks.BPO4, + forks.BPO5, + }) } -func (api *ConsensusAPI) getPayload(payloadID engine.PayloadID, full bool) (*engine.ExecutionPayloadEnvelope, error) { +// GetPayloadV6 returns a cached payload by id. This endpoint should only +// be used after the Amsterdam fork. +func (api *ConsensusAPI) GetPayloadV6(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) { + return api.getPayload( + payloadID, + false, + []engine.PayloadVersion{engine.PayloadV4}, + []forks.Fork{ + forks.Amsterdam, + }) +} + +// getPayload will retrieve the specified payload and verify it conforms to the +// endpoint's allowed payload versions and forks. +// +// Note passing nil `forks`, `versions` disables the respective check. +func (api *ConsensusAPI) getPayload(payloadID engine.PayloadID, full bool, versions []engine.PayloadVersion, forks []forks.Fork) (*engine.ExecutionPayloadEnvelope, error) { log.Trace("Engine API request received", "method", "GetPayload", "id", payloadID) + if versions != nil && !payloadID.Is(versions...) { + return nil, engine.UnsupportedFork + } data := api.localBlocks.get(payloadID, full) if data == nil { return nil, engine.UnknownPayload } + if forks != nil && !api.checkFork(data.ExecutionPayload.Timestamp, forks...) { + return nil, engine.UnsupportedFork + } + return data, nil } @@ -500,7 +545,7 @@ func (api *ConsensusAPI) GetBlobsV1(hashes []common.Hash) ([]*engine.BlobAndProo if len(hashes) > 128 { return nil, engine.TooLargeRequest.With(fmt.Errorf("requested blob count too large: %v", len(hashes))) } - blobs, _, proofs, err := api.eth.BlobTxPool().GetBlobs(hashes, types.BlobSidecarVersion0, false) + blobs, _, proofs, err := api.eth.BlobTxPool().GetBlobs(hashes, types.BlobSidecarVersion0) if err != nil { return nil, engine.InvalidParams.With(err) } @@ -546,8 +591,25 @@ func (api *ConsensusAPI) GetBlobsV1(hashes []common.Hash) ([]*engine.BlobAndProo func (api *ConsensusAPI) GetBlobsV2(hashes []common.Hash) ([]*engine.BlobAndProofV2, error) { head := api.eth.BlockChain().CurrentHeader() if api.config().LatestFork(head.Time, types.DeserializeHeaderExtraInformation(head).ArbOSFormatVersion) < forks.Osaka { - return nil, unsupportedForkErr("engine_getBlobsV2 is not available before Osaka fork") + return nil, nil + } + return api.getBlobs(hashes, true) +} + +// GetBlobsV3 returns a set of blobs from the transaction pool. Same as +// GetBlobsV2, except will return partial responses in case there is a missing +// blob. +func (api *ConsensusAPI) GetBlobsV3(hashes []common.Hash) ([]*engine.BlobAndProofV2, error) { + head := api.eth.BlockChain().CurrentHeader() + if api.config().LatestFork(head.Time, types.DeserializeHeaderExtraInformation(head).ArbOSFormatVersion) < forks.Osaka { + return nil, nil } + return api.getBlobs(hashes, false) +} + +// getBlobs returns all available blobs. In v2, partial responses are not allowed, +// while v3 supports partial responses. +func (api *ConsensusAPI) getBlobs(hashes []common.Hash, v2 bool) ([]*engine.BlobAndProofV2, error) { if len(hashes) > 128 { return nil, engine.TooLargeRequest.With(fmt.Errorf("requested blob count too large: %v", len(hashes))) } @@ -555,28 +617,30 @@ func (api *ConsensusAPI) GetBlobsV2(hashes []common.Hash) ([]*engine.BlobAndProo getBlobsRequestedCounter.Inc(int64(len(hashes))) getBlobsAvailableCounter.Inc(int64(available)) - // Optimization: check first if all blobs are available, if not, return empty response - if available != len(hashes) { - getBlobsV2RequestMiss.Inc(1) + // Short circuit if partial response is not allowed + if v2 && available != len(hashes) { + getBlobsRequestMiss.Inc(1) return nil, nil } - - blobs, _, proofs, err := api.eth.BlobTxPool().GetBlobs(hashes, types.BlobSidecarVersion1, false) + // Retrieve blobs from the pool. This operation is expensive and may involve + // heavy disk I/O. + blobs, _, proofs, err := api.eth.BlobTxPool().GetBlobs(hashes, types.BlobSidecarVersion1) if err != nil { return nil, engine.InvalidParams.With(err) } - - // To comply with API spec, check again that we really got all data needed - for _, blob := range blobs { - if blob == nil { - getBlobsV2RequestMiss.Inc(1) - return nil, nil - } - } - getBlobsV2RequestHit.Inc(1) - + // Validate the blobs from the pool and assemble the response res := make([]*engine.BlobAndProofV2, len(hashes)) - for i := 0; i < len(blobs); i++ { + for i := range blobs { + // The blob has been evicted since the last AvailableBlobs call. + // Return null if partial response is not allowed. + if blobs[i] == nil { + if !v2 { + continue + } else { + getBlobsRequestMiss.Inc(1) + return nil, nil + } + } var cellProofs []hexutil.Bytes for _, proof := range proofs[i] { cellProofs = append(cellProofs, proof[:]) @@ -586,6 +650,13 @@ func (api *ConsensusAPI) GetBlobsV2(hashes []common.Hash) ([]*engine.BlobAndProo CellProofs: cellProofs, } } + if len(res) == len(hashes) { + getBlobsRequestCompleteHit.Inc(1) + } else if len(res) > 0 { + getBlobsRequestPartialHit.Inc(1) + } else { + getBlobsRequestMiss.Inc(1) + } return res, nil } @@ -593,15 +664,15 @@ func (api *ConsensusAPI) GetBlobsV2(hashes []common.Hash) ([]*engine.BlobAndProo var invalidStatus = engine.PayloadStatusV1{Status: engine.INVALID} // NewPayloadV1 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. -func (api *ConsensusAPI) NewPayloadV1(params engine.ExecutableData) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) NewPayloadV1(ctx context.Context, params engine.ExecutableData) (engine.PayloadStatusV1, error) { if params.Withdrawals != nil { return invalidStatus, paramsErr("withdrawals not supported in V1") } - return api.newPayload(params, nil, nil, nil, false) + return api.newPayload(ctx, params, nil, nil, nil, false) } // NewPayloadV2 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. -func (api *ConsensusAPI) NewPayloadV2(params engine.ExecutableData) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) NewPayloadV2(ctx context.Context, params engine.ExecutableData) (engine.PayloadStatusV1, error) { arbosVersion := types.DeserializeHeaderExtraInformation(api.eth.BlockChain().CurrentHeader()).ArbOSFormatVersion var ( cancun = api.config().IsCancun(api.config().LondonBlock, params.Timestamp, arbosVersion) @@ -619,11 +690,11 @@ func (api *ConsensusAPI) NewPayloadV2(params engine.ExecutableData) (engine.Payl case params.BlobGasUsed != nil: return invalidStatus, paramsErr("non-nil blobGasUsed pre-cancun") } - return api.newPayload(params, nil, nil, nil, false) + return api.newPayload(ctx, params, nil, nil, nil, false) } // NewPayloadV3 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. -func (api *ConsensusAPI) NewPayloadV3(params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) NewPayloadV3(ctx context.Context, params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash) (engine.PayloadStatusV1, error) { switch { case params.Withdrawals == nil: return invalidStatus, paramsErr("nil withdrawals post-shanghai") @@ -638,11 +709,11 @@ func (api *ConsensusAPI) NewPayloadV3(params engine.ExecutableData, versionedHas case !api.checkFork(params.Timestamp, forks.Cancun): return invalidStatus, unsupportedForkErr("newPayloadV3 must only be called for cancun payloads") } - return api.newPayload(params, versionedHashes, beaconRoot, nil, false) + return api.newPayload(ctx, params, versionedHashes, beaconRoot, nil, false) } // NewPayloadV4 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. -func (api *ConsensusAPI) NewPayloadV4(params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash, executionRequests []hexutil.Bytes) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) NewPayloadV4(ctx context.Context, params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash, executionRequests []hexutil.Bytes) (engine.PayloadStatusV1, error) { switch { case params.Withdrawals == nil: return invalidStatus, paramsErr("nil withdrawals post-shanghai") @@ -663,10 +734,37 @@ func (api *ConsensusAPI) NewPayloadV4(params engine.ExecutableData, versionedHas if err := validateRequests(requests); err != nil { return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(err) } - return api.newPayload(params, versionedHashes, beaconRoot, requests, false) + return api.newPayload(ctx, params, versionedHashes, beaconRoot, requests, false) +} + +// NewPayloadV5 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. +func (api *ConsensusAPI) NewPayloadV5(ctx context.Context, params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash, executionRequests []hexutil.Bytes) (engine.PayloadStatusV1, error) { + switch { + case params.Withdrawals == nil: + return invalidStatus, paramsErr("nil withdrawals post-shanghai") + case params.ExcessBlobGas == nil: + return invalidStatus, paramsErr("nil excessBlobGas post-cancun") + case params.BlobGasUsed == nil: + return invalidStatus, paramsErr("nil blobGasUsed post-cancun") + case versionedHashes == nil: + return invalidStatus, paramsErr("nil versionedHashes post-cancun") + case beaconRoot == nil: + return invalidStatus, paramsErr("nil beaconRoot post-cancun") + case executionRequests == nil: + return invalidStatus, paramsErr("nil executionRequests post-prague") + case params.SlotNumber == nil: + return invalidStatus, paramsErr("nil slotnumber post-amsterdam") + case !api.checkFork(params.Timestamp, forks.Amsterdam): + return invalidStatus, unsupportedForkErr("newPayloadV5 must only be called for amsterdam payloads") + } + requests := convertRequests(executionRequests) + if err := validateRequests(requests); err != nil { + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(err) + } + return api.newPayload(ctx, params, versionedHashes, beaconRoot, requests, false) } -func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash, requests [][]byte, witness bool) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) newPayload(ctx context.Context, params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash, requests [][]byte, witness bool) (result engine.PayloadStatusV1, err error) { // The locking here is, strictly, not required. Without these locks, this can happen: // // 1. NewPayload( execdata-N ) is invoked from the CL. It goes all the way down to @@ -680,6 +778,13 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe // sequentially. // Hence, we use a lock here, to be sure that the previous call has finished before we // check whether we already have the block locally. + var attrs = []telemetry.Attribute{ + telemetry.Int64Attribute("block.number", int64(params.Number)), + telemetry.StringAttribute("block.hash", params.BlockHash.Hex()), + telemetry.Int64Attribute("tx.count", int64(len(params.Transactions))), + } + ctx, _, spanEnd := telemetry.StartSpan(ctx, "engine.newPayload", attrs...) + defer spanEnd(&err) api.newPayloadLock.Lock() defer api.newPayloadLock.Unlock() @@ -694,6 +799,10 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe if params.ExcessBlobGas != nil { ebg = strconv.Itoa(int(*params.ExcessBlobGas)) } + slotNum := "nil" + if params.SlotNumber != nil { + slotNum = strconv.Itoa(int(*params.SlotNumber)) + } log.Warn("Invalid NewPayload params", "params.Number", params.Number, "params.ParentHash", params.ParentHash, @@ -709,6 +818,7 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe "params.BaseFeePerGas", params.BaseFeePerGas, "params.BlobGasUsed", bgu, "params.ExcessBlobGas", ebg, + "params.SlotNumber", slotNum, "len(params.Transactions)", len(params.Transactions), "len(params.Withdrawals)", len(params.Withdrawals), "beaconRoot", beaconRoot, @@ -741,14 +851,14 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe return api.delayPayloadImport(block), nil } if block.Time() <= parent.Time() { - log.Warn("Invalid timestamp", "parent", block.Time(), "block", block.Time()) + log.Warn("Invalid timestamp", "parent", parent.Time(), "block", block.Time()) return api.invalid(errors.New("invalid timestamp"), parent.Header()), nil } // Another corner case: if the node is in snap sync mode, but the CL client // tries to make it import a block. That should be denied as pushing something // into the database directly will conflict with the assumptions of snap sync // that it has an empty db that it can fill itself. - if api.eth.SyncMode() != ethconfig.FullSync { + if api.eth.Downloader().ConfigSyncMode() == ethconfig.SnapSync { return api.delayPayloadImport(block), nil } if !api.eth.BlockChain().HasBlockAndState(block.ParentHash(), block.NumberU64()-1) { @@ -757,7 +867,9 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe return engine.PayloadStatusV1{Status: engine.ACCEPTED}, nil } log.Trace("Inserting block without sethead", "hash", block.Hash(), "number", block.Number()) - proofs, err := api.eth.BlockChain().InsertBlockWithoutSetHead(block, witness) + start := time.Now() + proofs, err := api.eth.BlockChain().InsertBlockWithoutSetHead(ctx, block, witness) + processingTime := time.Since(start) if err != nil { log.Warn("NewPayload: inserting block failed", "error", err) @@ -770,6 +882,13 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe } hash := block.Hash() + // Emit NewPayloadEvent for ethstats reporting + api.eth.BlockChain().SendNewPayloadEvent(core.NewPayloadEvent{ + Hash: hash, + Number: block.NumberU64(), + ProcessingTime: processingTime, + }) + // If witness collection was requested, inject that into the result too var ow *hexutil.Bytes if proofs != nil { @@ -796,16 +915,16 @@ func (api *ConsensusAPI) delayPayloadImport(block *types.Block) engine.PayloadSt // Although we don't want to trigger a sync, if there is one already in // progress, try to extend it with the current payload request to relieve // some strain from the forkchoice update. - err := api.eth.Downloader().BeaconExtend(api.eth.SyncMode(), block.Header()) + err := api.eth.Downloader().BeaconExtend(block.Header()) if err == nil { log.Debug("Payload accepted for sync extension", "number", block.NumberU64(), "hash", block.Hash()) return engine.PayloadStatusV1{Status: engine.SYNCING} } // Either no beacon sync was started yet, or it rejected the delivered - // payload as non-integratable on top of the existing sync. We'll just + // payload as non-integrate on top of the existing sync. We'll just // have to rely on the beacon client to forcefully update the head with // a forkchoice update request. - if api.eth.SyncMode() == ethconfig.FullSync { + if api.eth.Downloader().ConfigSyncMode() == ethconfig.FullSync { // In full sync mode, failure to import a well-formed block can only mean // that the parent state is missing and the syncer rejected extending the // current cycle with the new payload. @@ -900,8 +1019,6 @@ func (api *ConsensusAPI) invalid(err error, latestValid *types.Header) engine.Pa // heartbeat loops indefinitely, and checks if there have been beacon client updates // received in the last while. If not - or if they but strange ones - it warns the // user that something might be off with their consensus node. -// -// TODO(karalabe): Spin this goroutine down somehow func (api *ConsensusAPI) heartbeat() { // Sleep a bit on startup since there's obviously no beacon client yet // attached, so no need to print scary warnings to the user. diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index 793948eebb..d813886ef0 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -190,7 +190,7 @@ func TestEth2PrepareAndGetPayload(t *testing.T) { SafeBlockHash: common.Hash{}, FinalizedBlockHash: common.Hash{}, } - _, err := api.ForkchoiceUpdatedV1(fcState, &blockParams) + _, err := api.ForkchoiceUpdatedV1(context.Background(), fcState, &blockParams) if err != nil { t.Fatalf("error preparing payload, err=%v", err) } @@ -203,7 +203,7 @@ func TestEth2PrepareAndGetPayload(t *testing.T) { BeaconRoot: blockParams.BeaconRoot, Version: engine.PayloadV1, }).Id() - execData, err := api.getPayload(payloadID, true) + execData, err := api.getPayload(payloadID, true, nil, nil) if err != nil { t.Fatalf("error getting payload, err=%v", err) } @@ -270,7 +270,7 @@ func TestInvalidPayloadTimestamp(t *testing.T) { SafeBlockHash: common.Hash{}, FinalizedBlockHash: common.Hash{}, } - _, err := api.ForkchoiceUpdatedV1(fcState, ¶ms) + _, err := api.ForkchoiceUpdatedV1(context.Background(), fcState, ¶ms) if test.shouldErr && err == nil { t.Fatalf("expected error preparing payload with invalid timestamp, err=%v", err) } else if !test.shouldErr && err != nil { @@ -314,7 +314,7 @@ func TestEth2NewBlock(t *testing.T) { if err != nil { t.Fatalf("Failed to convert executable data to block %v", err) } - newResp, err := api.NewPayloadV1(*execData) + newResp, err := api.NewPayloadV1(context.Background(), *execData) switch { case err != nil: t.Fatalf("Failed to insert block: %v", err) @@ -329,7 +329,7 @@ func TestEth2NewBlock(t *testing.T) { SafeBlockHash: block.Hash(), FinalizedBlockHash: block.Hash(), } - if _, err := api.ForkchoiceUpdatedV1(fcState, nil); err != nil { + if _, err := api.ForkchoiceUpdatedV1(context.Background(), fcState, nil); err != nil { t.Fatalf("Failed to insert block: %v", err) } if have, want := ethservice.BlockChain().CurrentBlock().Number.Uint64(), block.NumberU64(); have != want { @@ -356,7 +356,7 @@ func TestEth2NewBlock(t *testing.T) { if err != nil { t.Fatalf("Failed to convert executable data to block %v", err) } - newResp, err := api.NewPayloadV1(*execData) + newResp, err := api.NewPayloadV1(context.Background(), *execData) if err != nil || newResp.Status != "VALID" { t.Fatalf("Failed to insert block: %v", err) } @@ -369,7 +369,7 @@ func TestEth2NewBlock(t *testing.T) { SafeBlockHash: block.Hash(), FinalizedBlockHash: block.Hash(), } - if _, err := api.ForkchoiceUpdatedV1(fcState, nil); err != nil { + if _, err := api.ForkchoiceUpdatedV1(context.Background(), fcState, nil); err != nil { t.Fatalf("Failed to insert block: %v", err) } if ethservice.BlockChain().CurrentBlock().Number.Uint64() != block.NumberU64() { @@ -426,7 +426,7 @@ func TestEth2DeepReorg(t *testing.T) { } // startEthService creates a full node instance for testing. -func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block) (*node.Node, *eth.Ethereum) { +func startEthService(t testing.TB, genesis *core.Genesis, blocks []*types.Block) (*node.Node, *eth.Ethereum) { t.Helper() n, err := node.New(&node.Config{ @@ -502,7 +502,7 @@ func setupBlocks(t *testing.T, ethservice *eth.Ethereum, n int, parent *types.He } envelope := getNewEnvelope(t, api, parent, w, h) - execResp, err := api.newPayload(*envelope.ExecutionPayload, []common.Hash{}, h, envelope.Requests, false) + execResp, err := api.newPayload(context.Background(), *envelope.ExecutionPayload, []common.Hash{}, h, envelope.Requests, false) if err != nil { t.Fatalf("can't execute payload: %v", err) } @@ -515,7 +515,7 @@ func setupBlocks(t *testing.T, ethservice *eth.Ethereum, n int, parent *types.He SafeBlockHash: payload.ParentHash, FinalizedBlockHash: payload.ParentHash, } - if _, err := api.ForkchoiceUpdatedV1(fcState, nil); err != nil { + if _, err := api.ForkchoiceUpdatedV1(context.Background(), fcState, nil); err != nil { t.Fatalf("Failed to insert block: %v", err) } if ethservice.BlockChain().CurrentBlock().Number.Uint64() != payload.Number { @@ -629,14 +629,14 @@ func TestNewPayloadOnInvalidChain(t *testing.T) { err error ) for i := 0; ; i++ { - if resp, err = api.ForkchoiceUpdatedV1(fcState, ¶ms); err != nil { + if resp, err = api.ForkchoiceUpdatedV1(context.Background(), fcState, ¶ms); err != nil { t.Fatalf("error preparing payload, err=%v", err) } if resp.PayloadStatus.Status != engine.VALID { t.Fatalf("error preparing payload, invalid status: %v", resp.PayloadStatus.Status) } // give the payload some time to be built - if payload, err = api.getPayload(*resp.PayloadID, true); err != nil { + if payload, err = api.getPayload(*resp.PayloadID, true, nil, nil); err != nil { t.Fatalf("can't get payload: %v", err) } if len(payload.ExecutionPayload.Transactions) > 0 { @@ -648,7 +648,7 @@ func TestNewPayloadOnInvalidChain(t *testing.T) { t.Fatalf("payload should not be empty") } } - execResp, err := api.NewPayloadV1(*payload.ExecutionPayload) + execResp, err := api.NewPayloadV1(context.Background(), *payload.ExecutionPayload) if err != nil { t.Fatalf("can't execute payload: %v", err) } @@ -660,7 +660,7 @@ func TestNewPayloadOnInvalidChain(t *testing.T) { SafeBlockHash: payload.ExecutionPayload.ParentHash, FinalizedBlockHash: payload.ExecutionPayload.ParentHash, } - if _, err := api.ForkchoiceUpdatedV1(fcState, nil); err != nil { + if _, err := api.ForkchoiceUpdatedV1(context.Background(), fcState, nil); err != nil { t.Fatalf("Failed to insert block: %v", err) } if ethservice.BlockChain().CurrentBlock().Number.Uint64() != payload.ExecutionPayload.Number { @@ -679,7 +679,7 @@ func assembleEnvelope(api *ConsensusAPI, parentHash common.Hash, params *engine. Withdrawals: params.Withdrawals, BeaconRoot: params.BeaconRoot, } - payload, err := api.eth.Miner().BuildPayload(args, false) + payload, err := api.eth.Miner().BuildPayload(context.Background(), args, false) if err != nil { return nil, err } @@ -708,7 +708,7 @@ func TestEmptyBlocks(t *testing.T) { // (1) check LatestValidHash by sending a normal payload (P1'') payload := getNewPayload(t, api, commonAncestor, nil, nil) - status, err := api.NewPayloadV1(*payload) + status, err := api.NewPayloadV1(context.Background(), *payload) if err != nil { t.Fatal(err) } @@ -724,7 +724,7 @@ func TestEmptyBlocks(t *testing.T) { payload.GasUsed += 1 payload = setBlockhash(payload) // Now latestValidHash should be the common ancestor - status, err = api.NewPayloadV1(*payload) + status, err = api.NewPayloadV1(context.Background(), *payload) if err != nil { t.Fatal(err) } @@ -742,7 +742,7 @@ func TestEmptyBlocks(t *testing.T) { payload.ParentHash = common.Hash{1} payload = setBlockhash(payload) // Now latestValidHash should be the common ancestor - status, err = api.NewPayloadV1(*payload) + status, err = api.NewPayloadV1(context.Background(), *payload) if err != nil { t.Fatal(err) } @@ -859,7 +859,7 @@ func TestTrickRemoteBlockCache(t *testing.T) { // feed the payloads to node B for _, payload := range invalidChain { - status, err := apiB.NewPayloadV1(*payload) + status, err := apiB.NewPayloadV1(context.Background(), *payload) if err != nil { panic(err) } @@ -867,7 +867,7 @@ func TestTrickRemoteBlockCache(t *testing.T) { t.Error("invalid status: VALID on an invalid chain") } // Now reorg to the head of the invalid chain - resp, err := apiB.ForkchoiceUpdatedV1(engine.ForkchoiceStateV1{HeadBlockHash: payload.BlockHash, SafeBlockHash: payload.BlockHash, FinalizedBlockHash: payload.ParentHash}, nil) + resp, err := apiB.ForkchoiceUpdatedV1(context.Background(), engine.ForkchoiceStateV1{HeadBlockHash: payload.BlockHash, SafeBlockHash: payload.BlockHash, FinalizedBlockHash: payload.ParentHash}, nil) if err != nil { t.Fatal(err) } @@ -892,7 +892,7 @@ func TestInvalidBloom(t *testing.T) { // (1) check LatestValidHash by sending a normal payload (P1'') payload := getNewPayload(t, api, commonAncestor, nil, nil) payload.LogsBloom = append(payload.LogsBloom, byte(1)) - status, err := api.NewPayloadV1(*payload) + status, err := api.NewPayloadV1(context.Background(), *payload) if err != nil { t.Fatal(err) } @@ -931,7 +931,7 @@ func TestSimultaneousNewBlock(t *testing.T) { for ii := 0; ii < 10; ii++ { go func() { defer wg.Done() - if newResp, err := api.NewPayloadV1(*execData); err != nil { + if newResp, err := api.NewPayloadV1(context.Background(), *execData); err != nil { errMu.Lock() testErr = fmt.Errorf("failed to insert block: %w", err) errMu.Unlock() @@ -970,7 +970,7 @@ func TestSimultaneousNewBlock(t *testing.T) { for ii := 0; ii < 10; ii++ { go func() { defer wg.Done() - if _, err := api.ForkchoiceUpdatedV1(fcState, nil); err != nil { + if _, err := api.ForkchoiceUpdatedV1(context.Background(), fcState, nil); err != nil { errMu.Lock() testErr = fmt.Errorf("failed to insert block: %w", err) errMu.Unlock() @@ -1011,7 +1011,7 @@ func TestWithdrawals(t *testing.T) { fcState := engine.ForkchoiceStateV1{ HeadBlockHash: parent.Hash(), } - resp, err := api.ForkchoiceUpdatedV2(fcState, &blockParams) + resp, err := api.ForkchoiceUpdatedV2(context.Background(), fcState, &blockParams) if err != nil { t.Fatalf("error preparing payload, err=%v", err) } @@ -1038,7 +1038,7 @@ func TestWithdrawals(t *testing.T) { } // 10: verify locally built block - if status, err := api.NewPayloadV2(*execData.ExecutionPayload); err != nil { + if status, err := api.NewPayloadV2(context.Background(), *execData.ExecutionPayload); err != nil { t.Fatalf("error validating payload: %v", err) } else if status.Status != engine.VALID { t.Fatalf("invalid payload") @@ -1063,7 +1063,7 @@ func TestWithdrawals(t *testing.T) { }, } fcState.HeadBlockHash = execData.ExecutionPayload.BlockHash - _, err = api.ForkchoiceUpdatedV2(fcState, &blockParams) + _, err = api.ForkchoiceUpdatedV2(context.Background(), fcState, &blockParams) if err != nil { t.Fatalf("error preparing payload, err=%v", err) } @@ -1082,7 +1082,7 @@ func TestWithdrawals(t *testing.T) { if err != nil { t.Fatalf("error getting payload, err=%v", err) } - if status, err := api.NewPayloadV2(*execData.ExecutionPayload); err != nil { + if status, err := api.NewPayloadV2(context.Background(), *execData.ExecutionPayload); err != nil { t.Fatalf("error validating payload: %v", err) } else if status.Status != engine.VALID { t.Fatalf("invalid payload") @@ -1090,7 +1090,7 @@ func TestWithdrawals(t *testing.T) { // 11: set block as head. fcState.HeadBlockHash = execData.ExecutionPayload.BlockHash - _, err = api.ForkchoiceUpdatedV2(fcState, nil) + _, err = api.ForkchoiceUpdatedV2(context.Background(), fcState, nil) if err != nil { t.Fatalf("error preparing payload, err=%v", err) } @@ -1196,10 +1196,10 @@ func TestNilWithdrawals(t *testing.T) { ) if !shanghai { payloadVersion = engine.PayloadV1 - _, err = api.ForkchoiceUpdatedV1(fcState, &test.blockParams) + _, err = api.ForkchoiceUpdatedV1(context.Background(), fcState, &test.blockParams) } else { payloadVersion = engine.PayloadV2 - _, err = api.ForkchoiceUpdatedV2(fcState, &test.blockParams) + _, err = api.ForkchoiceUpdatedV2(context.Background(), fcState, &test.blockParams) } if test.wantErr { if err == nil { @@ -1219,15 +1219,20 @@ func TestNilWithdrawals(t *testing.T) { Random: test.blockParams.Random, Version: payloadVersion, }).Id() - execData, err := api.GetPayloadV2(payloadID) + if !shanghai { + if _, err := api.GetPayloadV2(payloadID); err != nil { + t.Fatalf("GetPayloadV2 rejected pre-shanghai payload: %v", err) + } + } + execData, err := api.getPayload(payloadID, false, nil, nil) if err != nil { t.Fatalf("error getting payload, err=%v", err) } var status engine.PayloadStatusV1 if !shanghai { - status, err = api.NewPayloadV1(*execData.ExecutionPayload) + status, err = api.NewPayloadV1(context.Background(), *execData.ExecutionPayload) } else { - status, err = api.NewPayloadV2(*execData.ExecutionPayload) + status, err = api.NewPayloadV2(context.Background(), *execData.ExecutionPayload) } if err != nil { t.Fatalf("error validating payload: %v", err.(*engine.EngineAPIError).ErrorData()) @@ -1574,7 +1579,7 @@ func TestParentBeaconBlockRoot(t *testing.T) { fcState := engine.ForkchoiceStateV1{ HeadBlockHash: parent.Hash(), } - resp, err := api.ForkchoiceUpdatedV3(fcState, &blockParams) + resp, err := api.ForkchoiceUpdatedV3(context.Background(), fcState, &blockParams) if err != nil { t.Fatalf("error preparing payload, err=%v", err.(*engine.EngineAPIError).ErrorData()) } @@ -1598,14 +1603,14 @@ func TestParentBeaconBlockRoot(t *testing.T) { } // 11: verify locally built block - if status, err := api.NewPayloadV3(*execData.ExecutionPayload, []common.Hash{}, &common.Hash{42}); err != nil { + if status, err := api.NewPayloadV3(context.Background(), *execData.ExecutionPayload, []common.Hash{}, &common.Hash{42}); err != nil { t.Fatalf("error validating payload: %v", err) } else if status.Status != engine.VALID { t.Fatalf("invalid payload") } fcState.HeadBlockHash = execData.ExecutionPayload.BlockHash - resp, err = api.ForkchoiceUpdatedV3(fcState, nil) + resp, err = api.ForkchoiceUpdatedV3(context.Background(), fcState, nil) if err != nil { t.Fatalf("error preparing payload, err=%v", err.(*engine.EngineAPIError).ErrorData()) } @@ -1661,7 +1666,7 @@ func TestWitnessCreationAndConsumption(t *testing.T) { SafeBlockHash: common.Hash{}, FinalizedBlockHash: common.Hash{}, } - _, err := api.ForkchoiceUpdatedWithWitnessV3(fcState, &blockParams) + _, err := api.ForkchoiceUpdatedWithWitnessV3(context.Background(), fcState, &blockParams) if err != nil { t.Fatalf("error preparing payload, err=%v", err) } @@ -1674,7 +1679,7 @@ func TestWitnessCreationAndConsumption(t *testing.T) { BeaconRoot: blockParams.BeaconRoot, Version: engine.PayloadV3, }).Id() - envelope, err := api.getPayload(payloadID, true) + envelope, err := api.getPayload(payloadID, true, nil, nil) if err != nil { t.Fatalf("error getting payload, err=%v", err) } @@ -1705,7 +1710,7 @@ func TestWitnessCreationAndConsumption(t *testing.T) { envelope.ExecutionPayload.StateRoot = wantStateRoot envelope.ExecutionPayload.ReceiptsRoot = wantReceiptRoot - res2, err := api.NewPayloadWithWitnessV3(*envelope.ExecutionPayload, []common.Hash{}, &common.Hash{42}) + res2, err := api.NewPayloadWithWitnessV3(context.Background(), *envelope.ExecutionPayload, []common.Hash{}, &common.Hash{42}) if err != nil { t.Fatalf("error executing stateless payload witness: %v", err) } @@ -1873,7 +1878,7 @@ func makeMultiBlobTx(chainConfig *params.ChainConfig, nonce uint64, blobCount in return types.MustSignNewTx(key, types.LatestSigner(chainConfig), blobtx) } -func newGetBlobEnv(t *testing.T, version byte) (*node.Node, *ConsensusAPI) { +func newGetBlobEnv(t testing.TB, version byte) (*node.Node, *ConsensusAPI) { var ( // Create a database pre-initialize with a genesis block config = *params.MergedTestChainConfig @@ -2016,7 +2021,7 @@ func TestGetBlobsV1AfterOsakaFork(t *testing.T) { } } -func TestGetBlobsV2(t *testing.T) { +func TestGetBlobsV2And3(t *testing.T) { n, api := newGetBlobEnv(t, 1) defer n.Close() @@ -2045,36 +2050,64 @@ func TestGetBlobsV2(t *testing.T) { }, } for i, suite := range suites { - // Fill the request for retrieving blobs - var ( - vhashes []common.Hash - expect []*engine.BlobAndProofV2 - ) - // fill missing blob - if suite.fillRandom { - vhashes = append(vhashes, testrand.Hash()) - } - for j := suite.start; j < suite.limit; j++ { - vhashes = append(vhashes, testBlobVHashes[j]) - var cellProofs []hexutil.Bytes - for _, proof := range testBlobCellProofs[j] { - cellProofs = append(cellProofs, proof[:]) + runGetBlobs(t, api.GetBlobsV2, suite.start, suite.limit, suite.fillRandom, false, fmt.Sprintf("GetBlobsV2 suite=%d", i)) + runGetBlobs(t, api.GetBlobsV3, suite.start, suite.limit, suite.fillRandom, true, fmt.Sprintf("GetBlobsV3 suite=%d %v", i, suite)) + } +} + +// Benchmark GetBlobsV2 internals +// Note that this is not an RPC-level benchmark, so JSON-RPC overhead is not included. +func BenchmarkGetBlobsV2(b *testing.B) { + n, api := newGetBlobEnv(b, 1) + defer n.Close() + + // for blobs in [1, 2, 4, 6], print string and run benchmark + for _, blobs := range []int{1, 2, 4, 6} { + name := fmt.Sprintf("blobs=%d", blobs) + b.Run(name, func(b *testing.B) { + for b.Loop() { + runGetBlobs(b, api.GetBlobsV2, 0, blobs, false, false, name) } - expect = append(expect, &engine.BlobAndProofV2{ - Blob: testBlobs[j][:], - CellProofs: cellProofs, - }) - } - result, err := api.GetBlobsV2(vhashes) - if err != nil { - t.Errorf("Unexpected error for case %d, %v", i, err) - } - // null is responded if any blob is missing - if suite.fillRandom { + }) + } +} + +type getBlobsFn func(hashes []common.Hash) ([]*engine.BlobAndProofV2, error) + +func runGetBlobs(t testing.TB, getBlobs getBlobsFn, start, limit int, fillRandom bool, expectPartialResponse bool, name string) { + // Fill the request for retrieving blobs + var ( + vhashes []common.Hash + expect []*engine.BlobAndProofV2 + ) + for j := start; j < limit; j++ { + vhashes = append(vhashes, testBlobVHashes[j]) + var cellProofs []hexutil.Bytes + for _, proof := range testBlobCellProofs[j] { + cellProofs = append(cellProofs, proof[:]) + } + expect = append(expect, &engine.BlobAndProofV2{ + Blob: testBlobs[j][:], + CellProofs: cellProofs, + }) + } + // fill missing blob + if fillRandom { + vhashes = append(vhashes, testrand.Hash()) + } + result, err := getBlobs(vhashes) + if err != nil { + t.Errorf("Unexpected error for case %s, %v", name, err) + } + if fillRandom { + if expectPartialResponse { + expect = append(expect, nil) + } else { + // Nil is expected if getBlobs can not return a partial response expect = nil } - if !reflect.DeepEqual(result, expect) { - t.Fatalf("Unexpected result for case %d", i) - } + } + if !reflect.DeepEqual(result, expect) { + t.Fatalf("Unexpected result for case %s", name) } } diff --git a/eth/catalyst/api_testing.go b/eth/catalyst/api_testing.go new file mode 100644 index 0000000000..8586029468 --- /dev/null +++ b/eth/catalyst/api_testing.go @@ -0,0 +1,79 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package catalyst + +import ( + "errors" + + "github.com/ethereum/go-ethereum/beacon/engine" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/miner" + "github.com/ethereum/go-ethereum/rpc" +) + +// testingAPI implements the testing_ namespace. +// It's an engine-API adjacent namespace for testing purposes. +type testingAPI struct { + eth *eth.Ethereum +} + +func newTestingAPI(backend *eth.Ethereum) rpc.API { + return rpc.API{ + Namespace: "testing", + Service: &testingAPI{backend}, + Version: "1.0", + Authenticated: false, + } +} + +func (api *testingAPI) BuildBlockV1(parentHash common.Hash, payloadAttributes engine.PayloadAttributes, transactions *[]hexutil.Bytes, extraData *hexutil.Bytes) (*engine.ExecutionPayloadEnvelope, error) { + if api.eth.BlockChain().CurrentBlock().Hash() != parentHash { + return nil, errors.New("parentHash is not current head") + } + // If transactions is empty but not nil, build an empty block + // If the transactions is nil, build a block with the current transactions from the txpool + // If the transactions is not nil and not empty, build a block with the transactions + buildEmpty := transactions != nil && len(*transactions) == 0 + var txs []*types.Transaction + if transactions != nil { + dec := make([][]byte, 0, len(*transactions)) + for _, tx := range *transactions { + dec = append(dec, tx) + } + var err error + txs, err = engine.DecodeTransactions(dec) + if err != nil { + return nil, err + } + } + extra := make([]byte, 0) + if extraData != nil { + extra = *extraData + } + args := &miner.BuildPayloadArgs{ + Parent: parentHash, + Timestamp: payloadAttributes.Timestamp, + FeeRecipient: payloadAttributes.SuggestedFeeRecipient, + Random: payloadAttributes.Random, + Withdrawals: payloadAttributes.Withdrawals, + BeaconRoot: payloadAttributes.BeaconRoot, + } + return api.eth.Miner().BuildTestingPayload(args, txs, buildEmpty, extra) +} diff --git a/eth/catalyst/api_testing_test.go b/eth/catalyst/api_testing_test.go new file mode 100644 index 0000000000..fd4d28d26a --- /dev/null +++ b/eth/catalyst/api_testing_test.go @@ -0,0 +1,121 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package catalyst + +import ( + "context" + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/beacon/engine" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/params" +) + +func TestBuildBlockV1(t *testing.T) { + genesis, blocks := generateMergeChain(5, true) + n, ethservice := startEthService(t, genesis, blocks) + defer n.Close() + + parent := ethservice.BlockChain().CurrentBlock() + attrs := engine.PayloadAttributes{ + Timestamp: parent.Time + 1, + Random: crypto.Keccak256Hash([]byte("test")), + SuggestedFeeRecipient: parent.Coinbase, + Withdrawals: nil, + BeaconRoot: nil, + } + + currentNonce, _ := ethservice.APIBackend.GetPoolNonce(context.Background(), testAddr) + tx, _ := types.SignTx(types.NewTransaction(currentNonce, testAddr, big.NewInt(1), params.TxGas, big.NewInt(params.InitialBaseFee*2), nil), types.LatestSigner(ethservice.BlockChain().Config()), testKey) + + api := &testingAPI{eth: ethservice} + + t.Run("buildOnCurrentHead", func(t *testing.T) { + envelope, err := api.BuildBlockV1(parent.Hash(), attrs, nil, nil) + if err != nil { + t.Fatalf("BuildBlockV1 failed: %v", err) + } + if envelope == nil || envelope.ExecutionPayload == nil { + t.Fatal("expected non-nil envelope and payload") + } + payload := envelope.ExecutionPayload + if payload.ParentHash != parent.Hash() { + t.Errorf("parent hash mismatch: got %x want %x", payload.ParentHash, parent.Hash()) + } + if payload.Number != parent.Number.Uint64()+1 { + t.Errorf("block number mismatch: got %d want %d", payload.Number, parent.Number.Uint64()+1) + } + if payload.Timestamp != attrs.Timestamp { + t.Errorf("timestamp mismatch: got %d want %d", payload.Timestamp, attrs.Timestamp) + } + if payload.FeeRecipient != attrs.SuggestedFeeRecipient { + t.Errorf("fee recipient mismatch: got %x want %x", payload.FeeRecipient, attrs.SuggestedFeeRecipient) + } + }) + + t.Run("wrongParentHash", func(t *testing.T) { + wrongParent := common.Hash{0x01} + _, err := api.BuildBlockV1(wrongParent, attrs, nil, nil) + if err == nil { + t.Fatal("expected error when parentHash is not current head") + } + if err.Error() != "parentHash is not current head" { + t.Errorf("unexpected error: %v", err) + } + }) + + t.Run("buildEmptyBlock", func(t *testing.T) { + emptyTxs := []hexutil.Bytes{} + envelope, err := api.BuildBlockV1(parent.Hash(), attrs, &emptyTxs, nil) + if err != nil { + t.Fatalf("BuildBlockV1 with empty txs failed: %v", err) + } + if envelope == nil || envelope.ExecutionPayload == nil { + t.Fatal("expected non-nil envelope and payload") + } + if len(envelope.ExecutionPayload.Transactions) != 0 { + t.Errorf("expected empty block, got %d transactions", len(envelope.ExecutionPayload.Transactions)) + } + }) + + t.Run("buildBlockWithTransactions", func(t *testing.T) { + enc, _ := tx.MarshalBinary() + txs := []hexutil.Bytes{enc} + envelope, err := api.BuildBlockV1(parent.Hash(), attrs, &txs, nil) + if err != nil { + t.Fatalf("BuildBlockV1 with transaction failed: %v", err) + } + if len(envelope.ExecutionPayload.Transactions) != 1 { + t.Errorf("expected 1 transaction, got %d", len(envelope.ExecutionPayload.Transactions)) + } + }) + + t.Run("buildBlockWithTransactionsFromTxPool", func(t *testing.T) { + ethservice.TxPool().Add([]*types.Transaction{tx}, true) + envelope, err := api.BuildBlockV1(parent.Hash(), attrs, nil, nil) + if err != nil { + t.Fatalf("BuildBlockV1 with transaction failed: %v", err) + } + if len(envelope.ExecutionPayload.Transactions) != 1 { + t.Errorf("expected 1 transaction, got %d", len(envelope.ExecutionPayload.Transactions)) + } + }) +} diff --git a/eth/catalyst/metrics.go b/eth/catalyst/metrics.go new file mode 100644 index 0000000000..01a24191b0 --- /dev/null +++ b/eth/catalyst/metrics.go @@ -0,0 +1,37 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package catalyst + +import "github.com/ethereum/go-ethereum/metrics" + +var ( + // Number of blobs requested via getBlobsV2 + getBlobsRequestedCounter = metrics.NewRegisteredCounter("engine/getblobs/requested", nil) + + // Number of blobs requested via getBlobsV2 that are present in the blobpool + getBlobsAvailableCounter = metrics.NewRegisteredCounter("engine/getblobs/available", nil) + + // Number of times getBlobsV2/V3 responded with all blobs + getBlobsRequestCompleteHit = metrics.NewRegisteredCounter("engine/getblobs/hit", nil) + + // Number of times getBlobsV2/V3 responded with no blobs. V2 will return no + // blobs if it doesn't have all the blobs (all or nothing). + getBlobsRequestMiss = metrics.NewRegisteredCounter("engine/getblobs/miss", nil) + + // Number of times getBlobsV3 responded with some, but not all, blobs + getBlobsRequestPartialHit = metrics.NewRegisteredCounter("engine/getblobs/partial", nil) +) diff --git a/eth/catalyst/simulated_beacon.go b/eth/catalyst/simulated_beacon.go index 6fee56d37b..66ba7e9e52 100644 --- a/eth/catalyst/simulated_beacon.go +++ b/eth/catalyst/simulated_beacon.go @@ -17,6 +17,7 @@ package catalyst import ( + "context" "crypto/rand" "crypto/sha256" "errors" @@ -32,11 +33,13 @@ import ( "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/internal/telemetry" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params/forks" "github.com/ethereum/go-ethereum/rpc" + "go.opentelemetry.io/otel" ) const devEpochLength = 32 @@ -102,6 +105,8 @@ type SimulatedBeacon struct { func payloadVersion(config *params.ChainConfig, time uint64, header *types.Header) engine.PayloadVersion { switch config.LatestFork(time, types.DeserializeHeaderExtraInformation(header).ArbOSFormatVersion) { + case forks.Amsterdam: + return engine.PayloadV4 case forks.BPO5, forks.BPO4, forks.BPO3, forks.BPO2, forks.BPO1, forks.Osaka, forks.Prague, forks.Cancun: return engine.PayloadV3 case forks.Paris, forks.Shanghai: @@ -123,7 +128,7 @@ func NewSimulatedBeacon(period uint64, feeRecipient common.Address, eth *eth.Eth // if genesis block, send forkchoiceUpdated to trigger transition to PoS if block.Number.Sign() == 0 { version := payloadVersion(eth.BlockChain().Config(), block.Time, block) - if _, err := engineAPI.forkchoiceUpdated(current, nil, version, false); err != nil { + if _, err := engineAPI.forkchoiceUpdated(context.Background(), current, nil, version, false); err != nil { return nil, err } } @@ -195,17 +200,34 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal, timestamp u return fmt.Errorf("failed to sync txpool: %w", err) } - version := payloadVersion(c.eth.BlockChain().Config(), timestamp, c.eth.BlockChain().CurrentBlock()) + header := c.eth.BlockChain().CurrentBlock() + version := payloadVersion(c.eth.BlockChain().Config(), timestamp, header) + tracer := otel.Tracer("") var random [32]byte rand.Read(random[:]) - fcResponse, err := c.engineAPI.forkchoiceUpdated(c.curForkchoiceState, &engine.PayloadAttributes{ + + attribute := &engine.PayloadAttributes{ Timestamp: timestamp, SuggestedFeeRecipient: feeRecipient, Withdrawals: withdrawals, Random: random, BeaconRoot: &common.Hash{}, - }, version, false) + } + if c.eth.BlockChain().Config().LatestFork(timestamp, types.DeserializeHeaderExtraInformation(header).ArbOSFormatVersion) == forks.Amsterdam { + slotNumber := uint64(0) + attribute.SlotNumber = &slotNumber + } + + // Create a server span for forkchoiceUpdated with payload attributes, + // simulating an incoming engine API request from a real consensus client. + fcCtx, fcSpanEnd := telemetry.StartServerSpan(context.Background(), tracer, telemetry.RPCInfo{ + System: "jsonrpc", + Service: "engine", + Method: "forkchoiceUpdatedV" + fmt.Sprintf("%d", version), + }) + fcResponse, err := c.engineAPI.forkchoiceUpdated(fcCtx, c.curForkchoiceState, attribute, version, false) + fcSpanEnd(&err) if err != nil { return err } @@ -219,7 +241,15 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal, timestamp u return nil } - envelope, err := c.engineAPI.getPayload(*fcResponse.PayloadID, true) + // Create a server span for getPayload, simulating the consensus client + // coming back to retrieve the built payload. + _, gpSpanEnd := telemetry.StartServerSpan(context.Background(), tracer, telemetry.RPCInfo{ + System: "jsonrpc", + Service: "engine", + Method: "getPayloadV" + fmt.Sprintf("%d", version), + }) + envelope, err := c.engineAPI.getPayload(*fcResponse.PayloadID, true, nil, nil) + gpSpanEnd(&err) if err != nil { return err } @@ -260,15 +290,32 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal, timestamp u requests = envelope.Requests } + // Create a server span for newPayload, simulating the consensus client + // sending the execution payload for validation. + npCtx, npSpanEnd := telemetry.StartServerSpan(context.Background(), tracer, telemetry.RPCInfo{ + System: "jsonrpc", + Service: "engine", + Method: "newPayloadV" + fmt.Sprintf("%d", version), + }) + // Mark the payload as canon - _, err = c.engineAPI.newPayload(*payload, blobHashes, beaconRoot, requests, false) + _, err = c.engineAPI.newPayload(npCtx, *payload, blobHashes, beaconRoot, requests, false) + npSpanEnd(&err) if err != nil { return err } c.setCurrentState(payload.BlockHash, finalizedHash) - // Mark the block containing the payload as canonical - if _, err = c.engineAPI.forkchoiceUpdated(c.curForkchoiceState, nil, version, false); err != nil { + // Create a server span for the final forkchoiceUpdated (no payload attributes), + // which sets the new block as the canonical chain head. + fcuCtx, fcuSpanEnd := telemetry.StartServerSpan(context.Background(), tracer, telemetry.RPCInfo{ + System: "jsonrpc", + Service: "engine", + Method: "forkchoiceUpdatedV" + fmt.Sprintf("%d", version), + }) + _, err = c.engineAPI.forkchoiceUpdated(fcuCtx, c.curForkchoiceState, nil, version, false) + fcuSpanEnd(&err) + if err != nil { return err } c.lastBlockTime = payload.Timestamp @@ -289,9 +336,8 @@ func (c *SimulatedBeacon) loop() { case <-timer.C: if err := c.sealBlock(c.withdrawals.pop(10), uint64(time.Now().Unix())); err != nil { log.Warn("Error performing sealing work", "err", err) - } else { - timer.Reset(time.Second * time.Duration(c.period)) } + timer.Reset(time.Second * time.Duration(c.period)) } } } @@ -339,7 +385,7 @@ func (c *SimulatedBeacon) Rollback() { func (c *SimulatedBeacon) Fork(parentHash common.Hash) error { // Ensure no pending transactions. c.eth.TxPool().Sync() - if len(c.eth.TxPool().Pending(txpool.PendingFilter{})) != 0 { + if pending, _ := c.eth.TxPool().Pending(txpool.PendingFilter{}); len(pending) != 0 { return errors.New("pending block dirty") } @@ -353,7 +399,7 @@ func (c *SimulatedBeacon) Fork(parentHash common.Hash) error { // AdjustTime creates a new block with an adjusted timestamp. func (c *SimulatedBeacon) AdjustTime(adjustment time.Duration) error { - if len(c.eth.TxPool().Pending(txpool.PendingFilter{})) != 0 { + if pending, _ := c.eth.TxPool().Pending(txpool.PendingFilter{}); len(pending) != 0 { return errors.New("could not adjust time on non-empty block") } parent := c.eth.BlockChain().CurrentBlock() @@ -374,5 +420,6 @@ func RegisterSimulatedBeaconAPIs(stack *node.Node, sim *SimulatedBeacon) { Service: api, Version: "1.0", }, + newTestingAPI(sim.eth), }) } diff --git a/eth/catalyst/witness.go b/eth/catalyst/witness.go index a519516036..7cb6438c27 100644 --- a/eth/catalyst/witness.go +++ b/eth/catalyst/witness.go @@ -17,6 +17,7 @@ package catalyst import ( + "context" "errors" "strconv" "time" @@ -35,7 +36,7 @@ import ( // ForkchoiceUpdatedWithWitnessV1 is analogous to ForkchoiceUpdatedV1, only it // generates an execution witness too if block building was requested. -func (api *ConsensusAPI) ForkchoiceUpdatedWithWitnessV1(update engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { +func (api *ConsensusAPI) ForkchoiceUpdatedWithWitnessV1(ctx context.Context, update engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { if payloadAttributes != nil { switch { case payloadAttributes.Withdrawals != nil || payloadAttributes.BeaconRoot != nil: @@ -44,12 +45,12 @@ func (api *ConsensusAPI) ForkchoiceUpdatedWithWitnessV1(update engine.Forkchoice return engine.STATUS_INVALID, paramsErr("fcuV1 called post-shanghai") } } - return api.forkchoiceUpdated(update, payloadAttributes, engine.PayloadV1, true) + return api.forkchoiceUpdated(ctx, update, payloadAttributes, engine.PayloadV1, true) } // ForkchoiceUpdatedWithWitnessV2 is analogous to ForkchoiceUpdatedV2, only it // generates an execution witness too if block building was requested. -func (api *ConsensusAPI) ForkchoiceUpdatedWithWitnessV2(update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { +func (api *ConsensusAPI) ForkchoiceUpdatedWithWitnessV2(ctx context.Context, update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { if params != nil { switch { case params.BeaconRoot != nil: @@ -62,12 +63,12 @@ func (api *ConsensusAPI) ForkchoiceUpdatedWithWitnessV2(update engine.Forkchoice return engine.STATUS_INVALID, unsupportedForkErr("fcuV2 must only be called with paris or shanghai payloads") } } - return api.forkchoiceUpdated(update, params, engine.PayloadV2, true) + return api.forkchoiceUpdated(ctx, update, params, engine.PayloadV2, true) } // ForkchoiceUpdatedWithWitnessV3 is analogous to ForkchoiceUpdatedV3, only it // generates an execution witness too if block building was requested. -func (api *ConsensusAPI) ForkchoiceUpdatedWithWitnessV3(update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { +func (api *ConsensusAPI) ForkchoiceUpdatedWithWitnessV3(ctx context.Context, update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { if params != nil { switch { case params.Withdrawals == nil: @@ -82,21 +83,21 @@ func (api *ConsensusAPI) ForkchoiceUpdatedWithWitnessV3(update engine.Forkchoice // hash, even if params are wrong. To do this we need to split up // forkchoiceUpdate into a function that only updates the head and then a // function that kicks off block construction. - return api.forkchoiceUpdated(update, params, engine.PayloadV3, true) + return api.forkchoiceUpdated(ctx, update, params, engine.PayloadV3, true) } // NewPayloadWithWitnessV1 is analogous to NewPayloadV1, only it also generates // and returns a stateless witness after running the payload. -func (api *ConsensusAPI) NewPayloadWithWitnessV1(params engine.ExecutableData) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) NewPayloadWithWitnessV1(ctx context.Context, params engine.ExecutableData) (engine.PayloadStatusV1, error) { if params.Withdrawals != nil { return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("withdrawals not supported in V1")) } - return api.newPayload(params, nil, nil, nil, true) + return api.newPayload(ctx, params, nil, nil, nil, true) } // NewPayloadWithWitnessV2 is analogous to NewPayloadV2, only it also generates // and returns a stateless witness after running the payload. -func (api *ConsensusAPI) NewPayloadWithWitnessV2(params engine.ExecutableData) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) NewPayloadWithWitnessV2(ctx context.Context, params engine.ExecutableData) (engine.PayloadStatusV1, error) { arbosVersion := types.DeserializeHeaderExtraInformation(api.eth.BlockChain().CurrentHeader()).ArbOSFormatVersion var ( cancun = api.config().IsCancun(api.config().LondonBlock, params.Timestamp, arbosVersion) @@ -114,12 +115,12 @@ func (api *ConsensusAPI) NewPayloadWithWitnessV2(params engine.ExecutableData) ( case params.BlobGasUsed != nil: return invalidStatus, paramsErr("non-nil blobGasUsed pre-cancun") } - return api.newPayload(params, nil, nil, nil, true) + return api.newPayload(ctx, params, nil, nil, nil, true) } // NewPayloadWithWitnessV3 is analogous to NewPayloadV3, only it also generates // and returns a stateless witness after running the payload. -func (api *ConsensusAPI) NewPayloadWithWitnessV3(params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) NewPayloadWithWitnessV3(ctx context.Context, params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash) (engine.PayloadStatusV1, error) { switch { case params.Withdrawals == nil: return invalidStatus, paramsErr("nil withdrawals post-shanghai") @@ -134,12 +135,12 @@ func (api *ConsensusAPI) NewPayloadWithWitnessV3(params engine.ExecutableData, v case !api.checkFork(params.Timestamp, forks.Cancun): return invalidStatus, unsupportedForkErr("newPayloadV3 must only be called for cancun payloads") } - return api.newPayload(params, versionedHashes, beaconRoot, nil, true) + return api.newPayload(ctx, params, versionedHashes, beaconRoot, nil, true) } // NewPayloadWithWitnessV4 is analogous to NewPayloadV4, only it also generates // and returns a stateless witness after running the payload. -func (api *ConsensusAPI) NewPayloadWithWitnessV4(params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash, executionRequests []hexutil.Bytes) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) NewPayloadWithWitnessV4(ctx context.Context, params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash, executionRequests []hexutil.Bytes) (engine.PayloadStatusV1, error) { switch { case params.Withdrawals == nil: return invalidStatus, paramsErr("nil withdrawals post-shanghai") @@ -160,7 +161,7 @@ func (api *ConsensusAPI) NewPayloadWithWitnessV4(params engine.ExecutableData, v if err := validateRequests(requests); err != nil { return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(err) } - return api.newPayload(params, versionedHashes, beaconRoot, requests, true) + return api.newPayload(ctx, params, versionedHashes, beaconRoot, requests, true) } // ExecuteStatelessPayloadV1 is analogous to NewPayloadV1, only it operates in @@ -286,7 +287,7 @@ func (api *ConsensusAPI) executeStatelessPayload(params engine.ExecutableData, v api.lastNewPayloadUpdate.Store(time.Now().Unix()) log.Trace("Executing block statelessly", "number", block.Number(), "hash", params.BlockHash) - stateRoot, receiptRoot, err := core.ExecuteStateless(api.config(), vm.Config{}, block, witness) + stateRoot, receiptRoot, err := core.ExecuteStateless(context.Background(), api.config(), vm.Config{}, block, witness) if err != nil { log.Warn("ExecuteStatelessPayload: execution failed", "err", err) errorMsg := err.Error() diff --git a/eth/downloader/beacondevsync.go b/eth/downloader/beacondevsync.go index 03f17b1a52..52e43f86b4 100644 --- a/eth/downloader/beacondevsync.go +++ b/eth/downloader/beacondevsync.go @@ -33,14 +33,14 @@ import ( // Note, this must not be used in live code. If the forkchcoice endpoint where // to use this instead of giving us the payload first, then essentially nobody // in the network would have the block yet that we'd attempt to retrieve. -func (d *Downloader) BeaconDevSync(mode SyncMode, header *types.Header) error { +func (d *Downloader) BeaconDevSync(header *types.Header) error { // Be very loud that this code should not be used in a live node log.Warn("----------------------------------") log.Warn("Beacon syncing with hash as target", "number", header.Number, "hash", header.Hash()) log.Warn("This is unhealthy for a live node!") log.Warn("This is incompatible with the consensus layer!") log.Warn("----------------------------------") - return d.BeaconSync(mode, header, header) + return d.BeaconSync(header, header) } // GetHeader tries to retrieve the header with a given hash from a random peer. diff --git a/eth/downloader/beaconsync.go b/eth/downloader/beaconsync.go index 12b74a1ba9..914e1dfada 100644 --- a/eth/downloader/beaconsync.go +++ b/eth/downloader/beaconsync.go @@ -34,10 +34,8 @@ import ( // directed by the skeleton sync's head/tail events. type beaconBackfiller struct { downloader *Downloader // Downloader to direct via this callback implementation - syncMode SyncMode // Sync mode to use for backfilling the skeleton chains success func() // Callback to run on successful sync cycle completion filling bool // Flag whether the downloader is backfilling or not - filled *types.Header // Last header filled by the last terminated sync loop started chan struct{} // Notification channel whether the downloader inited lock sync.Mutex // Mutex protecting the sync lock } @@ -57,12 +55,15 @@ func (b *beaconBackfiller) suspend() *types.Header { // If no filling is running, don't waste cycles b.lock.Lock() filling := b.filling - filled := b.filled started := b.started b.lock.Unlock() if !filling { - return filled // Return the filled header on the previous sync completion + // Sync cycle was inactive, retrieve and return the latest snap block + // as the filled header. + log.Debug("Backfiller was inactive") + + return b.downloader.blockchain.CurrentSnapBlock() } // A previous filling should be running, though it may happen that it hasn't // yet started (being done on a new goroutine). Many concurrent beacon head @@ -74,9 +75,9 @@ func (b *beaconBackfiller) suspend() *types.Header { // Now that we're sure the downloader successfully started up, we can cancel // it safely without running the risk of data races. b.downloader.Cancel() + log.Debug("Backfiller has been suspended") // Sync cycle was just terminated, retrieve and return the last filled header. - // Can't use `filled` as that contains a stale value from before cancellation. return b.downloader.blockchain.CurrentSnapBlock() } @@ -87,12 +88,11 @@ func (b *beaconBackfiller) resume() { // If a previous filling cycle is still running, just ignore this start // request. // TODO(karalabe): We should make this channel driven b.lock.Unlock() + log.Debug("Backfiller is running") return } b.filling = true - b.filled = nil b.started = make(chan struct{}) - mode := b.syncMode b.lock.Unlock() // Start the backfilling on its own thread since the downloader does not have @@ -102,42 +102,22 @@ func (b *beaconBackfiller) resume() { defer func() { b.lock.Lock() b.filling = false - b.filled = b.downloader.blockchain.CurrentSnapBlock() b.lock.Unlock() }() // If the downloader fails, report an error as in beacon chain mode there // should be no errors as long as the chain we're syncing to is valid. - if err := b.downloader.synchronise(mode, b.started); err != nil { + if err := b.downloader.synchronise(b.started); err != nil { log.Error("Beacon backfilling failed", "err", err) return } // Synchronization succeeded. Since this happens async, notify the outer - // context to disable snap syncing and enable transaction propagation. + // context to enable transaction propagation. if b.success != nil { b.success() } + log.Debug("Backfilling completed") }() -} - -// setMode updates the sync mode from the current one to the requested one. If -// there's an active sync in progress, it will be cancelled and restarted. -func (b *beaconBackfiller) setMode(mode SyncMode) { - // Update the old sync mode and track if it was changed - b.lock.Lock() - oldMode := b.syncMode - updated := oldMode != mode - filling := b.filling - b.syncMode = mode - b.lock.Unlock() - - // If the sync mode was changed mid-sync, restart. This should never ever - // really happen, we just handle it to detect programming errors. - if !updated || !filling { - return - } - log.Error("Downloader sync mode changed mid-run", "old", oldMode.String(), "new", mode.String()) - b.suspend() - b.resume() + log.Debug("Backfilling started") } // SetBadBlockCallback sets the callback to run when a bad block is hit by the @@ -153,8 +133,8 @@ func (d *Downloader) SetBadBlockCallback(onBadBlock badBlockFn) { // // Internally backfilling and state sync is done the same way, but the header // retrieval and scheduling is replaced. -func (d *Downloader) BeaconSync(mode SyncMode, head *types.Header, final *types.Header) error { - return d.beaconSync(mode, head, final, true) +func (d *Downloader) BeaconSync(head *types.Header, final *types.Header) error { + return d.beaconSync(head, final, true) } // BeaconExtend is an optimistic version of BeaconSync, where an attempt is made @@ -163,8 +143,8 @@ func (d *Downloader) BeaconSync(mode SyncMode, head *types.Header, final *types. // // This is useful if a beacon client is feeding us large chunks of payloads to run, // but is not setting the head after each. -func (d *Downloader) BeaconExtend(mode SyncMode, head *types.Header) error { - return d.beaconSync(mode, head, nil, false) +func (d *Downloader) BeaconExtend(head *types.Header) error { + return d.beaconSync(head, nil, false) } // beaconSync is the post-merge version of the chain synchronization, where the @@ -173,20 +153,9 @@ func (d *Downloader) BeaconExtend(mode SyncMode, head *types.Header) error { // // Internally backfilling and state sync is done the same way, but the header // retrieval and scheduling is replaced. -func (d *Downloader) beaconSync(mode SyncMode, head *types.Header, final *types.Header, force bool) error { - // When the downloader starts a sync cycle, it needs to be aware of the sync - // mode to use (full, snap). To keep the skeleton chain oblivious, inject the - // mode into the backfiller directly. - // - // Super crazy dangerous type cast. Should be fine (TM), we're only using a - // different backfiller implementation for skeleton tests. - d.skeleton.filler.(*beaconBackfiller).setMode(mode) - +func (d *Downloader) beaconSync(head *types.Header, final *types.Header, force bool) error { // Signal the skeleton sync to switch to a new head, however it wants - if err := d.skeleton.Sync(head, final, force); err != nil { - return err - } - return nil + return d.skeleton.Sync(head, final, force) } // findBeaconAncestor tries to locate the common ancestor link of the local chain @@ -217,6 +186,8 @@ func (d *Downloader) findBeaconAncestor() (uint64, error) { log.Error("Failed to retrieve beacon bounds", "err", err) return 0, err } + log.Debug("Searching beacon ancestor", "local", number, "beaconhead", beaconHead.Number, "beacontail", beaconTail.Number) + var linked bool switch d.getMode() { case ethconfig.FullSync: @@ -270,6 +241,7 @@ func (d *Downloader) findBeaconAncestor() (uint64, error) { } start = check } + log.Debug("Found beacon ancestor", "number", start) return start, nil } diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 09837a3045..1de0933842 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -97,8 +97,9 @@ type headerTask struct { } type Downloader struct { - mode atomic.Uint32 // Synchronisation mode defining the strategy used (per sync cycle), use d.getMode() to get the SyncMode - mux *event.TypeMux // Event multiplexer to announce sync operation events + mode atomic.Uint32 // Synchronisation mode defining the strategy used (per sync cycle), use d.getMode() to get the SyncMode + moder *syncModer // Sync mode management, deliver the appropriate sync mode choice for each cycle + mux *event.TypeMux // Event multiplexer to announce sync operation events queue *queue // Scheduler for selecting the hashes to download peers *peerSet // Set of active peers from which download can proceed @@ -165,6 +166,9 @@ type BlockChain interface { // HasHeader verifies a header's presence in the local chain. HasHeader(common.Hash, uint64) bool + // HasState checks if state trie is fully present in the database or not. + HasState(root common.Hash) bool + // GetHeaderByHash retrieves a header from the local chain. GetHeaderByHash(common.Hash) *types.Header @@ -189,8 +193,12 @@ type BlockChain interface { // CurrentSnapBlock retrieves the head snap block from the local chain. CurrentSnapBlock() *types.Header - // SnapSyncCommitHead directly commits the head block to a certain entity. - SnapSyncCommitHead(common.Hash) error + // SnapSyncStart explicitly notifies the chain that snap sync is scheduled and + // marks chain mutations as disallowed. + SnapSyncStart() error + + // SnapSyncComplete directly commits the head block to a certain entity. + SnapSyncComplete(common.Hash) error // InsertHeadersBeforeCutoff inserts a batch of headers before the configured // chain cutoff into the ancient store. @@ -221,10 +229,11 @@ type BlockChain interface { } // New creates a new downloader to fetch hashes and blocks from remote peers. -func New(stateDb ethdb.Database, mux *event.TypeMux, chain BlockChain, dropPeer peerDropFn, success func()) *Downloader { +func New(stateDb ethdb.Database, mode ethconfig.SyncMode, mux *event.TypeMux, chain BlockChain, dropPeer peerDropFn, success func()) *Downloader { cutoffNumber, cutoffHash := chain.HistoryPruningCutoff() dl := &Downloader{ stateDB: stateDb, + moder: newSyncModer(mode, chain, stateDb), mux: mux, queue: newQueue(blockCacheMaxItems, blockCacheInitialItems), peers: newPeerSet(), @@ -239,7 +248,7 @@ func New(stateDb ethdb.Database, mux *event.TypeMux, chain BlockChain, dropPeer syncStartBlock: chain.CurrentSnapBlock().Number.Uint64(), } // Create the post-merge skeleton syncer and start the process - dl.skeleton = newSkeleton(stateDb, dl.peers, dropPeer, newBeaconBackfiller(dl, success)) + dl.skeleton = newSkeleton(stateDb, dl.peers, dropPeer, newBeaconBackfiller(dl, success), chain) go dl.stateFetcher() return dl @@ -331,7 +340,7 @@ func (d *Downloader) UnregisterPeer(id string) error { // synchronise will select the peer and use it for synchronising. If an empty string is given // it will use the best peer possible and synchronize if its TD is higher than our own. If any of the // checks fail an error will be returned. This method is synchronous -func (d *Downloader) synchronise(mode SyncMode, beaconPing chan struct{}) error { +func (d *Downloader) synchronise(beaconPing chan struct{}) (err error) { // The beacon header syncer is async. It will start this synchronization and // will continue doing other tasks. However, if synchronization needs to be // cancelled, the syncer needs to know if we reached the startup point (and @@ -356,21 +365,21 @@ func (d *Downloader) synchronise(mode SyncMode, beaconPing chan struct{}) error if d.notified.CompareAndSwap(false, true) { log.Info("Block synchronisation started") } - if mode == ethconfig.SnapSync { - // Snap sync will directly modify the persistent state, making the entire - // trie database unusable until the state is fully synced. To prevent any - // subsequent state reads, explicitly disable the trie database and state - // syncer is responsible to address and correct any state missing. - if d.blockchain.TrieDB().Scheme() == rawdb.PathScheme { - if err := d.blockchain.TrieDB().Disable(); err != nil { - return err - } + + // Obtain the synchronized used in this cycle + mode := d.moder.get(true) + defer func() { + if err == nil && mode == ethconfig.SnapSync { + d.moder.disableSnap() + log.Info("Disabled snap-sync after the initial sync cycle") } - // Snap sync uses the snapshot namespace to store potentially flaky data until - // sync completely heals and finishes. Pause snapshot maintenance in the mean- - // time to prevent access. - if snapshots := d.blockchain.Snapshots(); snapshots != nil { // Only nil in tests - snapshots.Disable() + }() + + // Disable chain mutations when snap sync is selected, ensuring the + // downloader is the sole mutator. + if mode == ethconfig.SnapSync { + if err := d.blockchain.SnapSyncStart(); err != nil { + return err } } // Reset the queue, peer set and wake channels to clean any internal leftover state @@ -399,6 +408,7 @@ func (d *Downloader) synchronise(mode SyncMode, beaconPing chan struct{}) error // Atomically set the requested sync mode d.mode.Store(uint32(mode)) + defer d.mode.Store(0) if beaconPing != nil { close(beaconPing) @@ -406,10 +416,17 @@ func (d *Downloader) synchronise(mode SyncMode, beaconPing chan struct{}) error return d.syncToHead() } +// getMode returns the sync mode used within current cycle. func (d *Downloader) getMode() SyncMode { return SyncMode(d.mode.Load()) } +// ConfigSyncMode returns the sync mode configured for the node. +// The actual running sync mode can differ from this. +func (d *Downloader) ConfigSyncMode() SyncMode { + return d.moder.get(false) +} + // syncToHead starts a block synchronization based on the hash chain from // the specified head hash. func (d *Downloader) syncToHead() (err error) { @@ -942,29 +959,6 @@ func (d *Downloader) processSnapSyncContent() error { } else { // results already piled up, consume before handling pivot move results = append(append([]*fetchResult{oldPivot}, oldTail...), results...) } - // Split around the pivot block and process the two sides via snap/full sync - if !d.committed.Load() { - latest := results[len(results)-1].Header - // If the height is above the pivot block by 2 sets, it means the pivot - // became stale in the network, and it was garbage collected, move to a - // new pivot. - // - // Note, we have `reorgProtHeaderDelay` number of blocks withheld, Those - // need to be taken into account, otherwise we're detecting the pivot move - // late and will drop peers due to unavailable state!!! - if height := latest.Number.Uint64(); height >= pivot.Number.Uint64()+2*uint64(fsMinFullBlocks)-uint64(reorgProtHeaderDelay) { - log.Warn("Pivot became stale, moving", "old", pivot.Number.Uint64(), "new", height-uint64(fsMinFullBlocks)+uint64(reorgProtHeaderDelay)) - pivot = results[len(results)-1-fsMinFullBlocks+reorgProtHeaderDelay].Header // must exist as lower old pivot is uncommitted - - d.pivotLock.Lock() - d.pivotHeader = pivot - d.pivotLock.Unlock() - - // Write out the pivot into the database so a rollback beyond it will - // reenable snap sync - rawdb.WriteLastPivotNumber(d.stateDB, pivot.Number.Uint64()) - } - } P, beforeP, afterP := splitAroundPivot(pivot.Number.Uint64(), results) if err := d.commitSnapSyncData(beforeP, sync); err != nil { return err @@ -1066,7 +1060,7 @@ func (d *Downloader) commitPivotBlock(result *fetchResult) error { if _, err := d.blockchain.InsertReceiptChain([]*types.Block{block}, []rlp.RawValue{result.Receipts}, d.ancientLimit); err != nil { return err } - if err := d.blockchain.SnapSyncCommitHead(block.Hash()); err != nil { + if err := d.blockchain.SnapSyncComplete(block.Hash()); err != nil { return err } d.committed.Store(true) diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 12d33495d2..84b3e06b16 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -17,7 +17,6 @@ package downloader import ( - "fmt" "math/big" "sync" "sync/atomic" @@ -30,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/eth/protocols/snap" "github.com/ethereum/go-ethereum/event" @@ -49,12 +49,12 @@ type downloadTester struct { } // newTester creates a new downloader test mocker. -func newTester(t *testing.T) *downloadTester { - return newTesterWithNotification(t, nil) +func newTester(t *testing.T, mode ethconfig.SyncMode) *downloadTester { + return newTesterWithNotification(t, mode, nil) } // newTesterWithNotification creates a new downloader test mocker. -func newTesterWithNotification(t *testing.T, success func()) *downloadTester { +func newTesterWithNotification(t *testing.T, mode ethconfig.SyncMode, success func()) *downloadTester { db, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{}) if err != nil { panic(err) @@ -75,7 +75,7 @@ func newTesterWithNotification(t *testing.T, success func()) *downloadTester { chain: chain, peers: make(map[string]*downloadTesterPeer), } - tester.downloader = New(db, new(event.TypeMux), tester.chain, tester.dropPeer, success) + tester.downloader = New(db, mode, new(event.TypeMux), tester.chain, tester.dropPeer, success) return tester } @@ -213,10 +213,12 @@ func (dlp *downloadTesterPeer) RequestHeadersByNumber(origin uint64, amount int, func (dlp *downloadTesterPeer) RequestBodies(hashes []common.Hash, sink chan *eth.Response) (*eth.Request, error) { blobs := eth.ServiceGetBlockBodiesQuery(dlp.chain, hashes) - bodies := make([]*eth.BlockBody, len(blobs)) + bodies := make([]*types.Body, len(blobs)) + ethbodies := make([]eth.BlockBody, len(blobs)) for i, blob := range blobs { - bodies[i] = new(eth.BlockBody) + bodies[i] = new(types.Body) rlp.DecodeBytes(blob, bodies[i]) + rlp.DecodeBytes(blob, ðbodies[i]) } var ( txsHashes = make([]common.Hash, len(bodies)) @@ -238,9 +240,13 @@ func (dlp *downloadTesterPeer) RequestBodies(hashes []common.Hash, sink chan *et Peer: dlp.id, } res := ð.Response{ - Req: req, - Res: (*eth.BlockBodiesResponse)(&bodies), - Meta: [][]common.Hash{txsHashes, uncleHashes, withdrawalHashes}, + Req: req, + Res: (*eth.BlockBodiesResponse)(ðbodies), + Meta: eth.BlockBodyHashes{ + TransactionRoots: txsHashes, + UncleHashes: uncleHashes, + WithdrawalRoots: withdrawalHashes, + }, Time: 1, Done: make(chan error, 1), // Ignore the returned status } @@ -254,23 +260,24 @@ func (dlp *downloadTesterPeer) RequestBodies(hashes []common.Hash, sink chan *et // peer in the download tester. The returned function can be used to retrieve // batches of block receipts from the particularly requested peer. func (dlp *downloadTesterPeer) RequestReceipts(hashes []common.Hash, sink chan *eth.Response) (*eth.Request, error) { - blobs := eth.ServiceGetReceiptsQuery68(dlp.chain, hashes) + blobs := eth.ServiceGetReceiptsQuery(dlp.chain, hashes) + receipts := make([]types.Receipts, blobs.Len()) - receipts := make([]types.Receipts, len(blobs)) - for i, blob := range blobs { - rlp.DecodeBytes(blob, &receipts[i]) - } + // compute hashes + hashes = make([]common.Hash, blobs.Len()) hasher := trie.NewStackTrie(nil) - hashes = make([]common.Hash, len(receipts)) - for i, receipt := range receipts { - hashes[i] = types.DeriveSha(receipt, hasher) + receiptLists, err := blobs.Items() + if err != nil { + panic(err) } - req := ð.Request{ - Peer: dlp.id, + for i, rl := range receiptLists { + hashes[i] = types.DeriveSha(rl.Derivable(), hasher) } + + // deliver the response right away resp := eth.ReceiptsRLPResponse(types.EncodeBlockReceiptLists(receipts)) res := ð.Response{ - Req: req, + Req: ð.Request{Peer: dlp.id}, Res: &resp, Meta: hashes, Time: 1, @@ -279,7 +286,7 @@ func (dlp *downloadTesterPeer) RequestReceipts(hashes []common.Hash, sink chan * go func() { sink <- res }() - return req, nil + return res.Req, nil } // ID retrieves the peer's unique identifier. @@ -289,14 +296,14 @@ func (dlp *downloadTesterPeer) ID() string { // RequestAccountRange fetches a batch of accounts rooted in a specific account // trie, starting with the origin. -func (dlp *downloadTesterPeer) RequestAccountRange(id uint64, root, origin, limit common.Hash, bytes uint64) error { +func (dlp *downloadTesterPeer) RequestAccountRange(id uint64, root, origin, limit common.Hash, bytes int) error { // Create the request and service it req := &snap.GetAccountRangePacket{ ID: id, Root: root, Origin: origin, Limit: limit, - Bytes: bytes, + Bytes: uint64(bytes), } slimaccs, proofs := snap.ServiceGetAccountRangeQuery(dlp.chain, req) @@ -315,7 +322,7 @@ func (dlp *downloadTesterPeer) RequestAccountRange(id uint64, root, origin, limi // RequestStorageRanges fetches a batch of storage slots belonging to one or // more accounts. If slots from only one account is requested, an origin marker // may also be used to retrieve from there. -func (dlp *downloadTesterPeer) RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes uint64) error { +func (dlp *downloadTesterPeer) RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes int) error { // Create the request and service it req := &snap.GetStorageRangesPacket{ ID: id, @@ -323,7 +330,7 @@ func (dlp *downloadTesterPeer) RequestStorageRanges(id uint64, root common.Hash, Root: root, Origin: origin, Limit: limit, - Bytes: bytes, + Bytes: uint64(bytes), } storage, proofs := snap.ServiceGetStorageRangesQuery(dlp.chain, req) @@ -340,25 +347,28 @@ func (dlp *downloadTesterPeer) RequestStorageRanges(id uint64, root common.Hash, } // RequestByteCodes fetches a batch of bytecodes by hash. -func (dlp *downloadTesterPeer) RequestByteCodes(id uint64, hashes []common.Hash, bytes uint64) error { +func (dlp *downloadTesterPeer) RequestByteCodes(id uint64, hashes []common.Hash, bytes int) error { req := &snap.GetByteCodesPacket{ ID: id, Hashes: hashes, - Bytes: bytes, + Bytes: uint64(bytes), } codes := snap.ServiceGetByteCodesQuery(dlp.chain, req) go dlp.dl.downloader.SnapSyncer.OnByteCodes(dlp, id, codes) return nil } -// RequestTrieNodes fetches a batch of account or storage trie nodes rooted in -// a specific state trie. -func (dlp *downloadTesterPeer) RequestTrieNodes(id uint64, root common.Hash, paths []snap.TrieNodePathSet, bytes uint64) error { +// RequestTrieNodes fetches a batch of account or storage trie nodes. +func (dlp *downloadTesterPeer) RequestTrieNodes(id uint64, root common.Hash, count int, paths []snap.TrieNodePathSet, bytes int) error { + encPaths, err := rlp.EncodeToRawList(paths) + if err != nil { + panic(err) + } req := &snap.GetTrieNodesPacket{ ID: id, Root: root, - Paths: paths, - Bytes: bytes, + Paths: encPaths, + Bytes: uint64(bytes), } nodes, _ := snap.ServiceGetTrieNodesQuery(dlp.chain, req, time.Now()) go dlp.dl.downloader.SnapSyncer.OnTrieNodes(dlp, id, nodes) @@ -388,13 +398,12 @@ func assertOwnChain(t *testing.T, tester *downloadTester, length int) { } } -func TestCanonicalSynchronisation68Full(t *testing.T) { testCanonSync(t, eth.ETH68, FullSync) } - -// func TestCanonicalSynchronisation68Snap(t *testing.T) { testCanonSync(t, eth.ETH68, SnapSync) } +func TestCanonicalSynchronisationFull(t *testing.T) { testCanonSync(t, eth.ETH69, FullSync) } +func TestCanonicalSynchronisationSnap(t *testing.T) { testCanonSync(t, eth.ETH69, SnapSync) } func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { success := make(chan struct{}) - tester := newTesterWithNotification(t, func() { + tester := newTesterWithNotification(t, mode, func() { close(success) }) defer tester.terminate() @@ -404,7 +413,7 @@ func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { tester.newPeer("peer", protocol, chain.blocks[1:]) // Synchronise with the peer and make sure all relevant data was retrieved - if err := tester.downloader.BeaconSync(mode, chain.blocks[len(chain.blocks)-1].Header(), nil); err != nil { + if err := tester.downloader.BeaconSync(chain.blocks[len(chain.blocks)-1].Header(), nil); err != nil { t.Fatalf("failed to beacon-sync chain: %v", err) } select { @@ -417,12 +426,11 @@ func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that if a large batch of blocks are being downloaded, it is throttled // until the cached blocks are retrieved. -func TestThrottling68Full(t *testing.T) { testThrottling(t, eth.ETH68, FullSync) } - -// func TestThrottling68Snap(t *testing.T) { testThrottling(t, eth.ETH68, SnapSync) } +func TestThrottlingFull(t *testing.T) { testThrottling(t, eth.ETH69, FullSync) } +func TestThrottlingSnap(t *testing.T) { testThrottling(t, eth.ETH69, SnapSync) } func testThrottling(t *testing.T, protocol uint, mode SyncMode) { - tester := newTester(t) + tester := newTester(t, mode) defer tester.terminate() // Create a long block chain to download and the tester @@ -439,7 +447,7 @@ func testThrottling(t *testing.T, protocol uint, mode SyncMode) { // Start a synchronisation concurrently errc := make(chan error, 1) go func() { - errc <- tester.downloader.BeaconSync(mode, testChainBase.blocks[len(testChainBase.blocks)-1].Header(), nil) + errc <- tester.downloader.BeaconSync(testChainBase.blocks[len(testChainBase.blocks)-1].Header(), nil) }() // Iteratively take some blocks, always checking the retrieval count for { @@ -496,16 +504,15 @@ func testThrottling(t *testing.T, protocol uint, mode SyncMode) { } // Tests that a canceled download wipes all previously accumulated state. -func TestCancel68Full(t *testing.T) { testCancel(t, eth.ETH68, FullSync) } - -// func TestCancel68Snap(t *testing.T) { testCancel(t, eth.ETH68, SnapSync) } +func TestCancelFull(t *testing.T) { testCancel(t, eth.ETH69, FullSync) } +func TestCancelSnap(t *testing.T) { testCancel(t, eth.ETH69, SnapSync) } func testCancel(t *testing.T, protocol uint, mode SyncMode) { complete := make(chan struct{}) success := func() { close(complete) } - tester := newTesterWithNotification(t, success) + tester := newTesterWithNotification(t, mode, success) defer tester.terminate() chain := testChainBase.shorten(MaxHeaderFetch) @@ -517,7 +524,7 @@ func testCancel(t *testing.T, protocol uint, mode SyncMode) { t.Errorf("download queue not idle") } // Synchronise with the peer, but cancel afterwards - if err := tester.downloader.BeaconSync(mode, chain.blocks[len(chain.blocks)-1].Header(), nil); err != nil { + if err := tester.downloader.BeaconSync(chain.blocks[len(chain.blocks)-1].Header(), nil); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } <-complete @@ -527,55 +534,14 @@ func testCancel(t *testing.T, protocol uint, mode SyncMode) { } } -// Tests that synchronisations behave well in multi-version protocol environments -// and not wreak havoc on other nodes in the network. -func TestMultiProtoSynchronisation68Full(t *testing.T) { testMultiProtoSync(t, eth.ETH68, FullSync) } - -// func TestMultiProtoSynchronisation68Snap(t *testing.T) { testMultiProtoSync(t, eth.ETH68, SnapSync) } - -func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { - complete := make(chan struct{}) - success := func() { - close(complete) - } - tester := newTesterWithNotification(t, success) - defer tester.terminate() - - // Create a small enough block chain to download - chain := testChainBase.shorten(blockCacheMaxItems - 15) - - // Create peers of every type - tester.newPeer("peer 68", eth.ETH68, chain.blocks[1:]) - - if err := tester.downloader.BeaconSync(mode, chain.blocks[len(chain.blocks)-1].Header(), nil); err != nil { - t.Fatalf("failed to start beacon sync: %v", err) - } - select { - case <-complete: - break - case <-time.NewTimer(time.Second * 3).C: - t.Fatalf("Failed to sync chain in three seconds") - } - assertOwnChain(t, tester, len(chain.blocks)) - - // Check that no peers have been dropped off - for _, version := range []int{68} { - peer := fmt.Sprintf("peer %d", version) - if _, ok := tester.peers[peer]; !ok { - t.Errorf("%s dropped", peer) - } - } -} - // Tests that if a block is empty (e.g. header only), no body request should be // made, and instead the header should be assembled into a whole block in itself. -func TestEmptyShortCircuit68Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH68, FullSync) } - -// func TestEmptyShortCircuit68Snap(t *testing.T) { testEmptyShortCircuit(t, eth.ETH68, SnapSync) } +func TestEmptyShortCircuitFull(t *testing.T) { testEmptyShortCircuit(t, eth.ETH69, FullSync) } +func TestEmptyShortCircuitSnap(t *testing.T) { testEmptyShortCircuit(t, eth.ETH69, SnapSync) } func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { success := make(chan struct{}) - tester := newTesterWithNotification(t, func() { + tester := newTesterWithNotification(t, mode, func() { close(success) }) defer tester.terminate() @@ -593,7 +559,7 @@ func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { receiptsHave.Add(int32(len(headers))) } - if err := tester.downloader.BeaconSync(mode, chain.blocks[len(chain.blocks)-1].Header(), nil); err != nil { + if err := tester.downloader.BeaconSync(chain.blocks[len(chain.blocks)-1].Header(), nil); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } select { @@ -639,9 +605,8 @@ func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.Sync // Tests that peers below a pre-configured checkpoint block are prevented from // being fast-synced from, avoiding potential cheap eclipse attacks. -func TestBeaconSync68Full(t *testing.T) { testBeaconSync(t, eth.ETH68, FullSync) } - -// func TestBeaconSync68Snap(t *testing.T) { testBeaconSync(t, eth.ETH68, SnapSync) } +func TestBeaconSyncFull(t *testing.T) { testBeaconSync(t, eth.ETH69, FullSync) } +func TestBeaconSyncSnap(t *testing.T) { testBeaconSync(t, eth.ETH69, SnapSync) } func testBeaconSync(t *testing.T, protocol uint, mode SyncMode) { var cases = []struct { @@ -656,7 +621,7 @@ func testBeaconSync(t *testing.T, protocol uint, mode SyncMode) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { success := make(chan struct{}) - tester := newTesterWithNotification(t, func() { + tester := newTesterWithNotification(t, mode, func() { close(success) }) defer tester.terminate() @@ -668,7 +633,7 @@ func testBeaconSync(t *testing.T, protocol uint, mode SyncMode) { if c.local > 0 { tester.chain.InsertChain(chain.blocks[1 : c.local+1]) } - if err := tester.downloader.BeaconSync(mode, chain.blocks[len(chain.blocks)-1].Header(), nil); err != nil { + if err := tester.downloader.BeaconSync(chain.blocks[len(chain.blocks)-1].Header(), nil); err != nil { t.Fatalf("Failed to beacon sync chain %v %v", c.name, err) } select { @@ -686,13 +651,12 @@ func testBeaconSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that synchronisation progress (origin block number, current block number // and highest block number) is tracked and updated correctly. -func TestSyncProgress68Full(t *testing.T) { testSyncProgress(t, eth.ETH68, FullSync) } - -// func TestSyncProgress68Snap(t *testing.T) { testSyncProgress(t, eth.ETH68, SnapSync) } +func TestSyncProgressFull(t *testing.T) { testSyncProgress(t, eth.ETH69, FullSync) } +func TestSyncProgressSnap(t *testing.T) { testSyncProgress(t, eth.ETH69, SnapSync) } func testSyncProgress(t *testing.T, protocol uint, mode SyncMode) { success := make(chan struct{}) - tester := newTesterWithNotification(t, func() { + tester := newTesterWithNotification(t, mode, func() { success <- struct{}{} }) defer tester.terminate() @@ -707,7 +671,7 @@ func testSyncProgress(t *testing.T, protocol uint, mode SyncMode) { faultyPeer.withholdBodies[header.Hash()] = struct{}{} } - if err := tester.downloader.BeaconSync(mode, chain.blocks[len(chain.blocks)/2-1].Header(), nil); err != nil { + if err := tester.downloader.BeaconSync(chain.blocks[len(chain.blocks)/2-1].Header(), nil); err != nil { t.Fatalf("failed to beacon-sync chain: %v", err) } select { @@ -723,7 +687,7 @@ func testSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // Synchronise all the blocks and check continuation progress tester.newPeer("peer-full", protocol, chain.blocks[1:]) - if err := tester.downloader.BeaconSync(mode, chain.blocks[len(chain.blocks)-1].Header(), nil); err != nil { + if err := tester.downloader.BeaconSync(chain.blocks[len(chain.blocks)-1].Header(), nil); err != nil { t.Fatalf("failed to beacon-sync chain: %v", err) } startingBlock := uint64(len(chain.blocks)/2 - 1) diff --git a/eth/downloader/fetchers_concurrent_bodies.go b/eth/downloader/fetchers_concurrent_bodies.go index 56359b33c9..6a8eb35219 100644 --- a/eth/downloader/fetchers_concurrent_bodies.go +++ b/eth/downloader/fetchers_concurrent_bodies.go @@ -88,15 +88,14 @@ func (q *bodyQueue) request(peer *peerConnection, req *fetchRequest, resCh chan // deliver is responsible for taking a generic response packet from the concurrent // fetcher, unpacking the body data and delivering it to the downloader's queue. func (q *bodyQueue) deliver(peer *peerConnection, packet *eth.Response) (int, error) { - txs, uncles, withdrawals := packet.Res.(*eth.BlockBodiesResponse).Unpack() - hashsets := packet.Meta.([][]common.Hash) // {txs hashes, uncle hashes, withdrawal hashes} - - accepted, err := q.queue.DeliverBodies(peer.id, txs, hashsets[0], uncles, hashsets[1], withdrawals, hashsets[2]) + resp := packet.Res.(*eth.BlockBodiesResponse) + meta := packet.Meta.(eth.BlockBodyHashes) + accepted, err := q.queue.DeliverBodies(peer.id, meta, *resp) switch { - case err == nil && len(txs) == 0: + case err == nil && len(*resp) == 0: peer.log.Trace("Requested bodies delivered") case err == nil: - peer.log.Trace("Delivered new batch of bodies", "count", len(txs), "accepted", accepted) + peer.log.Trace("Delivered new batch of bodies", "count", len(*resp), "accepted", accepted) default: peer.log.Debug("Failed to deliver retrieved bodies", "err", err) } diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index 9fe169d5f7..c0cb9b174a 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -29,11 +29,10 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/prque" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" - "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" ) @@ -418,7 +417,7 @@ func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common skip := make([]*types.Header, 0) progress := false throttled := false - for proc := 0; len(send) < count && !taskQueue.Empty(); proc++ { + for len(send) < count && !taskQueue.Empty() { // the task queue will pop items in order, so the highest prio block // is also the lowest block number. header, _ := taskQueue.Peek() @@ -433,7 +432,6 @@ func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common taskQueue.PopItem() progress = true delete(taskPool, header.Hash()) - proc = proc - 1 log.Error("Fetch reservation already delivered", "number", header.Number.Uint64()) continue } @@ -455,7 +453,6 @@ func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common // If it's a noop, we can skip this task delete(taskPool, header.Hash()) taskQueue.PopItem() - proc = proc - 1 progress = true continue } @@ -561,63 +558,54 @@ func (q *queue) expire(peer string, pendPool map[string]*fetchRequest, taskQueue // DeliverBodies injects a block body retrieval response into the results queue. // The method returns the number of blocks bodies accepted from the delivery and // also wakes any threads waiting for data delivery. -func (q *queue) DeliverBodies(id string, txLists [][]*types.Transaction, txListHashes []common.Hash, - uncleLists [][]*types.Header, uncleListHashes []common.Hash, - withdrawalLists [][]*types.Withdrawal, withdrawalListHashes []common.Hash, -) (int, error) { +func (q *queue) DeliverBodies(id string, hashes eth.BlockBodyHashes, bodies []eth.BlockBody) (int, error) { q.lock.Lock() defer q.lock.Unlock() + var txLists [][]*types.Transaction + var uncleLists [][]*types.Header + var withdrawalLists [][]*types.Withdrawal + validate := func(index int, header *types.Header) error { - if txListHashes[index] != header.TxHash { + if hashes.TransactionRoots[index] != header.TxHash { return errInvalidBody } - if uncleListHashes[index] != header.UncleHash { + if hashes.UncleHashes[index] != header.UncleHash { return errInvalidBody } if header.WithdrawalsHash == nil { // nil hash means that withdrawals should not be present in body - if withdrawalLists[index] != nil { + if bodies[index].Withdrawals != nil { return errInvalidBody } } else { // non-nil hash: body must have withdrawals - if withdrawalLists[index] == nil { + if bodies[index].Withdrawals == nil { return errInvalidBody } - if withdrawalListHashes[index] != *header.WithdrawalsHash { + if hashes.WithdrawalRoots[index] != *header.WithdrawalsHash { return errInvalidBody } } - // Blocks must have a number of blobs corresponding to the header gas usage, - // and zero before the Cancun hardfork. - var blobs int - for _, tx := range txLists[index] { - // Count the number of blobs to validate against the header's blobGasUsed - blobs += len(tx.BlobHashes()) - - // Validate the data blobs individually too - if tx.Type() == types.BlobTxType { - if len(tx.BlobHashes()) == 0 { - return errInvalidBody - } - for _, hash := range tx.BlobHashes() { - if !kzg4844.IsValidVersionedHash(hash[:]) { - return errInvalidBody - } - } - if tx.BlobTxSidecar() != nil { - return errInvalidBody - } - } + + // decode + txs, err := bodies[index].Transactions.Items() + if err != nil { + return fmt.Errorf("%w: bad transactions: %v", errInvalidBody, err) } - if header.BlobGasUsed != nil { - if want := *header.BlobGasUsed / params.BlobTxBlobGasPerBlob; uint64(blobs) != want { // div because the header is surely good vs the body might be bloated - return errInvalidBody + txLists = append(txLists, txs) + uncles, err := bodies[index].Uncles.Items() + if err != nil { + return fmt.Errorf("%w: bad uncles: %v", errInvalidBody, err) + } + uncleLists = append(uncleLists, uncles) + if bodies[index].Withdrawals != nil { + withdrawals, err := bodies[index].Withdrawals.Items() + if err != nil { + return fmt.Errorf("%w: bad withdrawals: %v", errInvalidBody, err) } + withdrawalLists = append(withdrawalLists, withdrawals) } else { - if blobs != 0 { - return errInvalidBody - } + withdrawalLists = append(withdrawalLists, nil) } return nil } @@ -628,8 +616,9 @@ func (q *queue) DeliverBodies(id string, txLists [][]*types.Transaction, txListH result.Withdrawals = withdrawalLists[index] result.SetBodyDone() } + nresults := len(hashes.TransactionRoots) return q.deliver(id, q.blockTaskPool, q.blockTaskQueue, q.blockPendPool, - bodyReqTimer, bodyInMeter, bodyDropMeter, len(txLists), validate, reconstruct) + bodyReqTimer, bodyInMeter, bodyDropMeter, nresults, validate, reconstruct) } // DeliverReceipts injects a receipt retrieval response into the results queue. diff --git a/eth/downloader/queue_test.go b/eth/downloader/queue_test.go index c990e5b950..97584341ec 100644 --- a/eth/downloader/queue_test.go +++ b/eth/downloader/queue_test.go @@ -30,8 +30,10 @@ import ( "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" ) @@ -323,26 +325,31 @@ func XTestDelivery(t *testing.T) { emptyList []*types.Header txset [][]*types.Transaction uncleset [][]*types.Header + bodies []eth.BlockBody ) numToSkip := rand.Intn(len(f.Headers)) for _, hdr := range f.Headers[0 : len(f.Headers)-numToSkip] { - txset = append(txset, world.getTransactions(hdr.Number.Uint64())) + txs := world.getTransactions(hdr.Number.Uint64()) + txset = append(txset, txs) uncleset = append(uncleset, emptyList) + txsList, _ := rlp.EncodeToRawList(txs) + bodies = append(bodies, eth.BlockBody{Transactions: txsList}) + } + hashes := eth.BlockBodyHashes{ + TransactionRoots: make([]common.Hash, len(txset)), + UncleHashes: make([]common.Hash, len(uncleset)), + WithdrawalRoots: make([]common.Hash, len(txset)), } - var ( - txsHashes = make([]common.Hash, len(txset)) - uncleHashes = make([]common.Hash, len(uncleset)) - ) hasher := trie.NewStackTrie(nil) for i, txs := range txset { - txsHashes[i] = types.DeriveSha(types.Transactions(txs), hasher) + hashes.TransactionRoots[i] = types.DeriveSha(types.Transactions(txs), hasher) } for i, uncles := range uncleset { - uncleHashes[i] = types.CalcUncleHash(uncles) + hashes.UncleHashes[i] = types.CalcUncleHash(uncles) } + time.Sleep(100 * time.Millisecond) - _, err := q.DeliverBodies(peer.id, txset, txsHashes, uncleset, uncleHashes, nil, nil) - if err != nil { + if _, err := q.DeliverBodies(peer.id, hashes, bodies); err != nil { fmt.Printf("delivered %d bodies %v\n", len(txset), err) } } else { diff --git a/eth/downloader/skeleton.go b/eth/downloader/skeleton.go index 2cf9c4672b..e693bfc066 100644 --- a/eth/downloader/skeleton.go +++ b/eth/downloader/skeleton.go @@ -64,6 +64,12 @@ var errSyncMerged = errors.New("sync merged") // should abort and restart with the new state. var errSyncReorged = errors.New("sync reorged") +// errSyncTrimmed is an internal helper error to signal that the local chain +// has been trimmed (e.g, via debug_setHead explicitly) and the skeleton chain +// is no longer linked with the local chain. In this case, the skeleton sync +// should be re-scheduled again. +var errSyncTrimmed = errors.New("sync trimmed") + // errTerminated is returned if the sync mechanism was terminated for this run of // the process. This is usually the case when Geth is shutting down and some events // might still be propagating. @@ -201,6 +207,7 @@ type backfiller interface { type skeleton struct { db ethdb.Database // Database backing the skeleton filler backfiller // Chain syncer suspended/resumed by head events + chain chainReader // Underlying block chain peers *peerSet // Set of peers we can sync from idles map[string]*peerConnection // Set of idle peers in the current sync cycle @@ -225,12 +232,19 @@ type skeleton struct { syncStarting func() // callback triggered after a sync cycle is inited but before started } +// chainReader wraps the method to retrieve the head of the local chain. +type chainReader interface { + // CurrentSnapBlock retrieves the head snap block from the local chain. + CurrentSnapBlock() *types.Header +} + // newSkeleton creates a new sync skeleton that tracks a potentially dangling // header chain until it's linked into an existing set of blocks. -func newSkeleton(db ethdb.Database, peers *peerSet, drop peerDropFn, filler backfiller) *skeleton { +func newSkeleton(db ethdb.Database, peers *peerSet, drop peerDropFn, filler backfiller, chain chainReader) *skeleton { sk := &skeleton{ db: db, filler: filler, + chain: chain, peers: peers, drop: drop, requests: make(map[uint64]*headerRequest), @@ -296,6 +310,11 @@ func (s *skeleton) startup() { // head to force a cleanup. head = newhead + case err == errSyncTrimmed: + // The skeleton chain is not linked with the local chain anymore, + // restart the sync. + head = nil + case err == errTerminated: // Sync was requested to be terminated from within, stop and // return (no need to pass a message, was already done internally) @@ -343,6 +362,29 @@ func (s *skeleton) Sync(head *types.Header, final *types.Header, force bool) err } } +// linked returns the flag indicating whether the skeleton has been linked with +// the local chain. +func (s *skeleton) linked(number uint64, hash common.Hash) bool { + linked := rawdb.HasHeader(s.db, hash, number) && + rawdb.HasBody(s.db, hash, number) && + rawdb.HasReceipts(s.db, hash, number) + + // Ensure the skeleton chain links to the local chain below the chain head. + // This accounts for edge cases where leftover chain segments above the head + // may still link to the skeleton chain. In such cases, synchronization is + // likely to fail due to potentially missing segments in the middle. + // + // You can try to produce the edge case by these steps: + // - sync the chain + // - debug.setHead(`0x1`) + // - kill the geth process (the chain segment will be left with chain head rewound) + // - restart + if s.chain.CurrentSnapBlock() != nil { + linked = linked && s.chain.CurrentSnapBlock().Number.Uint64() >= number + } + return linked +} + // sync is the internal version of Sync that executes a single sync cycle, either // until some termination condition is reached, or until the current cycle merges // with a previously aborted run. @@ -367,10 +409,7 @@ func (s *skeleton) sync(head *types.Header) (*types.Header, error) { // If the sync is already done, resume the backfiller. When the loop stops, // terminate the backfiller too. - linked := len(s.progress.Subchains) == 1 && - rawdb.HasHeader(s.db, s.progress.Subchains[0].Next, s.scratchHead) && - rawdb.HasBody(s.db, s.progress.Subchains[0].Next, s.scratchHead) && - rawdb.HasReceipts(s.db, s.progress.Subchains[0].Next, s.scratchHead) + linked := len(s.progress.Subchains) == 1 && s.linked(s.scratchHead, s.progress.Subchains[0].Next) if linked { s.filler.resume() } @@ -486,7 +525,17 @@ func (s *skeleton) sync(head *types.Header) (*types.Header, error) { // is still running, it will pick it up. If it already terminated, // a new cycle needs to be spun up. if linked { - s.filler.resume() + if len(s.progress.Subchains) == 1 && s.linked(s.scratchHead, s.progress.Subchains[0].Next) { + // The skeleton chain has been extended and is still linked with the local + // chain, try to re-schedule the backfiller if it's already terminated. + s.filler.resume() + } else { + // The skeleton chain is no longer linked to the local chain for some reason + // (e.g. debug_setHead was used to trim the local chain). Re-schedule the + // skeleton sync to fill the chain gap. + log.Warn("Local chain has been trimmed", "tailnumber", s.scratchHead, "tailhash", s.progress.Subchains[0].Next) + return nil, errSyncTrimmed + } } case req := <-requestFails: @@ -649,9 +698,19 @@ func (s *skeleton) processNewHead(head *types.Header, final *types.Header) error // Not a noop / double head announce, abort with a reorg return fmt.Errorf("%w, tail: %d, head: %d, newHead: %d", errChainReorged, lastchain.Tail, lastchain.Head, number) } + // Terminate the sync if the chain head is gapped if lastchain.Head+1 < number { return fmt.Errorf("%w, head: %d, newHead: %d", errChainGapped, lastchain.Head, number) } + // Ignore the duplicated beacon header announcement + if lastchain.Head == number { + local := rawdb.ReadSkeletonHeader(s.db, number) + if local != nil && local.Hash() == head.Hash() { + log.Debug("Ignored the identical beacon header", "number", number, "hash", local.Hash()) + return nil + } + } + // Terminate the sync if the chain head is forked if parent := rawdb.ReadSkeletonHeader(s.db, number-1); parent.Hash() != head.ParentHash { return fmt.Errorf("%w, ancestor: %d, hash: %s, want: %s", errChainForked, number-1, parent.Hash(), head.ParentHash) } @@ -669,6 +728,7 @@ func (s *skeleton) processNewHead(head *types.Header, final *types.Header) error if err := batch.Write(); err != nil { log.Crit("Failed to write skeleton sync status", "err", err) } + log.Debug("Extended beacon header chain", "number", head.Number, "hash", head.Hash()) return nil } @@ -909,6 +969,45 @@ func (s *skeleton) revertRequest(req *headerRequest) { s.scratchOwners[(s.scratchHead-req.head)/requestHeaders] = "" } +// mergeSubchains is invoked once certain beacon headers have been persisted locally +// and the subchains should be merged in case there are some overlaps between. An +// indicator will be returned if the last subchain is merged with previous subchain. +func (s *skeleton) mergeSubchains() bool { + // If the subchain extended into the next subchain, we need to handle + // the overlap. Since there could be many overlaps, do this in a loop. + var merged bool + for len(s.progress.Subchains) > 1 && s.progress.Subchains[1].Head >= s.progress.Subchains[0].Tail { + // Extract some stats from the second subchain + head := s.progress.Subchains[1].Head + tail := s.progress.Subchains[1].Tail + next := s.progress.Subchains[1].Next + + // Since we just overwrote part of the next subchain, we need to trim + // its head independent of matching or mismatching content + if s.progress.Subchains[1].Tail >= s.progress.Subchains[0].Tail { + // Fully overwritten, get rid of the subchain as a whole + log.Debug("Previous subchain fully overwritten", "head", head, "tail", tail, "next", next) + s.progress.Subchains = append(s.progress.Subchains[:1], s.progress.Subchains[2:]...) + continue + } else { + // Partially overwritten, trim the head to the overwritten size + log.Debug("Previous subchain partially overwritten", "head", head, "tail", tail, "next", next) + s.progress.Subchains[1].Head = s.progress.Subchains[0].Tail - 1 + } + // If the old subchain is an extension of the new one, merge the two + // and let the skeleton syncer restart (to clean internal state) + if rawdb.ReadSkeletonHeader(s.db, s.progress.Subchains[1].Head).Hash() == s.progress.Subchains[0].Next { + log.Debug("Previous subchain merged", "head", head, "tail", tail, "next", next) + s.progress.Subchains[0].Tail = s.progress.Subchains[1].Tail + s.progress.Subchains[0].Next = s.progress.Subchains[1].Next + + s.progress.Subchains = append(s.progress.Subchains[:1], s.progress.Subchains[2:]...) + merged = true + } + } + return merged +} + func (s *skeleton) processResponse(res *headerResponse) (linked bool, merged bool) { res.peer.log.Trace("Processing header response", "head", res.headers[0].Number, "hash", res.headers[0].Hash(), "count", len(res.headers)) @@ -982,10 +1081,9 @@ func (s *skeleton) processResponse(res *headerResponse) (linked bool, merged boo // processing is done, so it's just one more "needless" check. // // The weird cascading checks are done to minimize the database reads. - linked = rawdb.HasHeader(s.db, header.ParentHash, header.Number.Uint64()-1) && - rawdb.HasBody(s.db, header.ParentHash, header.Number.Uint64()-1) && - rawdb.HasReceipts(s.db, header.ParentHash, header.Number.Uint64()-1) + linked = s.linked(header.Number.Uint64()-1, header.ParentHash) if linked { + log.Debug("Primary subchain linked", "number", header.Number.Uint64()-1, "hash", header.ParentHash) break } } @@ -999,6 +1097,9 @@ func (s *skeleton) processResponse(res *headerResponse) (linked bool, merged boo // If the beacon chain was linked to the local chain, completely swap out // all internal progress and abort header synchronization. if linked { + // Merge all overlapped subchains beforehand + s.mergeSubchains() + // Linking into the local chain should also mean that there are no // leftover subchains, but in the case of importing the blocks via // the engine API, we will not push the subchains forward. This will @@ -1056,41 +1157,10 @@ func (s *skeleton) processResponse(res *headerResponse) (linked bool, merged boo s.scratchHead -= uint64(consumed) - // If the subchain extended into the next subchain, we need to handle - // the overlap. Since there could be many overlaps (come on), do this - // in a loop. - for len(s.progress.Subchains) > 1 && s.progress.Subchains[1].Head >= s.progress.Subchains[0].Tail { - // Extract some stats from the second subchain - head := s.progress.Subchains[1].Head - tail := s.progress.Subchains[1].Tail - next := s.progress.Subchains[1].Next - - // Since we just overwrote part of the next subchain, we need to trim - // its head independent of matching or mismatching content - if s.progress.Subchains[1].Tail >= s.progress.Subchains[0].Tail { - // Fully overwritten, get rid of the subchain as a whole - log.Debug("Previous subchain fully overwritten", "head", head, "tail", tail, "next", next) - s.progress.Subchains = append(s.progress.Subchains[:1], s.progress.Subchains[2:]...) - continue - } else { - // Partially overwritten, trim the head to the overwritten size - log.Debug("Previous subchain partially overwritten", "head", head, "tail", tail, "next", next) - s.progress.Subchains[1].Head = s.progress.Subchains[0].Tail - 1 - } - // If the old subchain is an extension of the new one, merge the two - // and let the skeleton syncer restart (to clean internal state) - if rawdb.ReadSkeletonHeader(s.db, s.progress.Subchains[1].Head).Hash() == s.progress.Subchains[0].Next { - log.Debug("Previous subchain merged", "head", head, "tail", tail, "next", next) - s.progress.Subchains[0].Tail = s.progress.Subchains[1].Tail - s.progress.Subchains[0].Next = s.progress.Subchains[1].Next - - s.progress.Subchains = append(s.progress.Subchains[:1], s.progress.Subchains[2:]...) - merged = true - } - } // If subchains were merged, all further available headers in the scratch // space are invalid since we skipped ahead. Stop processing the scratch // space to avoid dropping peers thinking they delivered invalid data. + merged = s.mergeSubchains() if merged { break } @@ -1121,15 +1191,17 @@ func (s *skeleton) processResponse(res *headerResponse) (linked bool, merged boo // due to the downloader backfilling past the tracked tail. func (s *skeleton) cleanStales(filled *types.Header) error { number := filled.Number.Uint64() - log.Trace("Cleaning stale beacon headers", "filled", number, "hash", filled.Hash()) + log.Debug("Cleaning stale beacon headers", "filled", number, "hash", filled.Hash()) - // If the filled header is below the linked subchain, something's corrupted - // internally. Report and error and refuse to do anything. + // If the filled header is below the subchain, it means the skeleton is not + // linked with local chain yet, don't bother to do cleanup. if number+1 < s.progress.Subchains[0].Tail { - return fmt.Errorf("filled header below beacon header tail: %d < %d", number, s.progress.Subchains[0].Tail) + log.Debug("filled header below beacon header tail", "filled", number, "tail", s.progress.Subchains[0].Tail) + return nil } // If nothing in subchain is filled, don't bother to do cleanup. if number+1 == s.progress.Subchains[0].Tail { + log.Debug("Skeleton chain not yet consumed", "filled", number, "hash", filled.Hash(), "tail", s.progress.Subchains[0].Tail) return nil } // If the latest fill was on a different subchain, it means the backfiller @@ -1206,6 +1278,7 @@ func (s *skeleton) cleanStales(filled *types.Header) error { if err := batch.Write(); err != nil { log.Crit("Failed to write beacon trim data", "err", err) } + log.Debug("Cleaned stale beacon headers", "start", start, "end", end) return nil } diff --git a/eth/downloader/skeleton_test.go b/eth/downloader/skeleton_test.go index 4aa97cf1f7..8c38e9d0c5 100644 --- a/eth/downloader/skeleton_test.go +++ b/eth/downloader/skeleton_test.go @@ -20,6 +20,7 @@ import ( "encoding/json" "errors" "fmt" + "math" "math/big" "sync/atomic" "testing" @@ -71,6 +72,12 @@ func (hf *hookedBackfiller) resume() { } } +type fakeChainReader struct{} + +func (fc *fakeChainReader) CurrentSnapBlock() *types.Header { + return &types.Header{Number: big.NewInt(math.MaxInt64)} +} + // skeletonTestPeer is a mock peer that can only serve header requests from a // pre-perated header chain (which may be arbitrarily wrong for testing). // @@ -369,7 +376,7 @@ func TestSkeletonSyncInit(t *testing.T) { // Create a skeleton sync and run a cycle wait := make(chan struct{}) - skeleton := newSkeleton(db, newPeerSet(), nil, newHookedBackfiller()) + skeleton := newSkeleton(db, newPeerSet(), nil, newHookedBackfiller(), &fakeChainReader{}) skeleton.syncStarting = func() { close(wait) } skeleton.Sync(tt.head, nil, true) @@ -472,7 +479,7 @@ func TestSkeletonSyncExtend(t *testing.T) { // Create a skeleton sync and run a cycle wait := make(chan struct{}) - skeleton := newSkeleton(db, newPeerSet(), nil, newHookedBackfiller()) + skeleton := newSkeleton(db, newPeerSet(), nil, newHookedBackfiller(), &fakeChainReader{}) skeleton.syncStarting = func() { close(wait) } skeleton.Sync(tt.head, nil, true) @@ -838,7 +845,7 @@ func TestSkeletonSyncRetrievals(t *testing.T) { // Create a peer set to feed headers through peerset := newPeerSet() for _, peer := range tt.peers { - peerset.Register(newPeerConnection(peer.id, eth.ETH68, peer, log.New("id", peer.id))) + peerset.Register(newPeerConnection(peer.id, eth.ETH69, peer, log.New("id", peer.id))) } // Create a peer dropper to track malicious peers dropped := make(map[string]int) @@ -885,7 +892,7 @@ func TestSkeletonSyncRetrievals(t *testing.T) { } } // Create a skeleton sync and run a cycle - skeleton := newSkeleton(db, peerset, drop, filler) + skeleton := newSkeleton(db, peerset, drop, filler, &fakeChainReader{}) skeleton.Sync(tt.head, nil, true) // Wait a bit (bleah) for the initial sync loop to go to idle. This might @@ -905,7 +912,7 @@ func TestSkeletonSyncRetrievals(t *testing.T) { // Apply the post-init events if there's any endpeers := tt.peers if tt.newPeer != nil { - if err := peerset.Register(newPeerConnection(tt.newPeer.id, eth.ETH68, tt.newPeer, log.New("id", tt.newPeer.id))); err != nil { + if err := peerset.Register(newPeerConnection(tt.newPeer.id, eth.ETH69, tt.newPeer, log.New("id", tt.newPeer.id))); err != nil { t.Errorf("test %d: failed to register new peer: %v", i, err) } time.Sleep(time.Millisecond * 50) // given time for peer registration diff --git a/eth/downloader/syncmode.go b/eth/downloader/syncmode.go new file mode 100644 index 0000000000..036119ce3d --- /dev/null +++ b/eth/downloader/syncmode.go @@ -0,0 +1,115 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package downloader + +import ( + "sync" + + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" +) + +// syncModer is responsible for managing the downloader's sync mode. It takes the +// user's preference at startup and then determines the appropriate sync mode +// based on the current chain status. +type syncModer struct { + mode ethconfig.SyncMode + chain BlockChain + disk ethdb.KeyValueReader + lock sync.Mutex +} + +func newSyncModer(mode ethconfig.SyncMode, chain BlockChain, disk ethdb.KeyValueReader) *syncModer { + if mode == ethconfig.FullSync { + // The database seems empty as the current block is the genesis. Yet the snap + // block is ahead, so snap sync was enabled for this node at a certain point. + // The scenarios where this can happen is + // * if the user manually (or via a bad block) rolled back a snap sync node + // below the sync point. + // * the last snap sync is not finished while user specifies a full sync this + // time. But we don't have any recent state for full sync. + // In these cases however it's safe to reenable snap sync. + fullBlock, snapBlock := chain.CurrentBlock(), chain.CurrentSnapBlock() + if fullBlock.Number.Uint64() == 0 && snapBlock.Number.Uint64() > 0 { + mode = ethconfig.SnapSync + log.Warn("Switching from full-sync to snap-sync", "reason", "snap-sync incomplete") + } else if !chain.HasState(fullBlock.Root) { + mode = ethconfig.SnapSync + log.Warn("Switching from full-sync to snap-sync", "reason", "head state missing") + } else { + // Grant the full sync mode + log.Info("Enabled full-sync", "head", fullBlock.Number, "hash", fullBlock.Hash()) + } + } else { + head := chain.CurrentBlock() + if head.Number.Uint64() > 0 && chain.HasState(head.Root) { + mode = ethconfig.FullSync + log.Info("Switching from snap-sync to full-sync", "reason", "snap-sync complete") + } else { + // If snap sync was requested and our database is empty, grant it + log.Info("Enabled snap-sync", "head", head.Number, "hash", head.Hash()) + } + } + return &syncModer{ + mode: mode, + chain: chain, + disk: disk, + } +} + +// get retrieves the current sync mode, either explicitly set, or derived +// from the chain status. +func (m *syncModer) get(report bool) ethconfig.SyncMode { + m.lock.Lock() + defer m.lock.Unlock() + + // If we're in snap sync mode, return that directly + if m.mode == ethconfig.SnapSync { + return ethconfig.SnapSync + } + logger := log.Debug + if report { + logger = log.Info + } + // We are probably in full sync, but we might have rewound to before the + // snap sync pivot, check if we should re-enable snap sync. + head := m.chain.CurrentBlock() + if pivot := rawdb.ReadLastPivotNumber(m.disk); pivot != nil { + if head.Number.Uint64() < *pivot { + logger("Reenabled snap-sync as chain is lagging behind the pivot", "head", head.Number, "pivot", pivot) + return ethconfig.SnapSync + } + } + // We are in a full sync, but the associated head state is missing. To complete + // the head state, forcefully rerun the snap sync. Note it doesn't mean the + // persistent state is corrupted, just mismatch with the head block. + if !m.chain.HasState(head.Root) { + logger("Reenabled snap-sync as chain is stateless") + return ethconfig.SnapSync + } + // Nope, we're really full syncing + return ethconfig.FullSync +} + +// disableSnap disables the snap sync mode, usually it's called after a successful snap sync. +func (m *syncModer) disableSnap() { + m.lock.Lock() + m.mode = ethconfig.FullSync + m.lock.Unlock() +} diff --git a/eth/dropper.go b/eth/dropper.go index 51f2a7a95a..dada5d07c0 100644 --- a/eth/dropper.go +++ b/eth/dropper.go @@ -145,6 +145,9 @@ func randomDuration(min, max time.Duration) time.Duration { if min > max { panic("min duration must be less than or equal to max duration") } + if min == max { + return min + } return time.Duration(mrand.Int63n(int64(max-min)) + int64(min)) } diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 7b8cd78fa6..1657bb3602 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -50,30 +50,34 @@ var FullNodeGPO = gasprice.Config{ // Defaults contains default settings for use on the Ethereum main net. var Defaults = Config{ - HistoryMode: history.KeepAll, - SyncMode: SnapSync, - NetworkId: 0, // enable auto configuration of networkID == chainID - TxLookupLimit: 2350000, - TransactionHistory: 2350000, - LogHistory: 2350000, - StateHistory: params.FullImmutabilityThreshold, - MaxDiffLayers: pathdb.DefaultMaxDiffLayers, - DatabaseCache: 512, - TrieCleanCache: 154, - TrieDirtyCache: 256, - TrieTimeout: 60 * time.Minute, - SnapshotCache: 102, - FilterLogCacheSize: 32, - LogQueryLimit: 1000, - Miner: miner.DefaultConfig, - TxPool: legacypool.DefaultConfig, - BlobPool: blobpool.DefaultConfig, - RPCGasCap: 50000000, - RPCEVMTimeout: 5 * time.Second, - GPO: FullNodeGPO, - RPCTxFeeCap: 1, // 1 ether - TxSyncDefaultTimeout: 20 * time.Second, - TxSyncMaxTimeout: 1 * time.Minute, + HistoryMode: history.KeepAll, + SyncMode: SnapSync, + NetworkId: 0, // enable auto configuration of networkID == chainID + TxLookupLimit: 2350000, + TransactionHistory: 2350000, + LogHistory: 2350000, + StateHistory: pathdb.Defaults.StateHistory, + MaxDiffLayers: pathdb.DefaultMaxDiffLayers, + TrienodeHistory: pathdb.Defaults.TrienodeHistory, + NodeFullValueCheckpoint: pathdb.Defaults.FullValueCheckpoint, + DatabaseCache: 2048, + TrieCleanCache: 614, + TrieDirtyCache: 1024, + SnapshotCache: 409, + TrieTimeout: 60 * time.Minute, + FilterLogCacheSize: 32, + LogQueryLimit: 1000, + Miner: miner.DefaultConfig, + TxPool: legacypool.DefaultConfig, + BlobPool: blobpool.DefaultConfig, + RPCGasCap: 50000000, + RPCEVMTimeout: 5 * time.Second, + GPO: FullNodeGPO, + RPCTxFeeCap: 1, // 1 ether + TxSyncDefaultTimeout: 20 * time.Second, + TxSyncMaxTimeout: 1 * time.Minute, + SlowBlockThreshold: -1, // Disabled by default; set via --debug.logslowblock flag + RangeLimit: 0, } //go:generate go run github.com/fjl/gencodec -type Config -formats toml -out gen_config.go @@ -110,6 +114,13 @@ type Config struct { LogExportCheckpoints string // export log index checkpoints to file StateHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose state histories are reserved. MaxDiffLayers int `toml:",omitempty"` // Maximum diff layers allowed in the layer tree. + TrienodeHistory int64 `toml:",omitempty"` // Number of blocks from the chain head for which trienode histories are retained + + // The frequency of full-value encoding. For example, a value of 16 means + // that, on average, for a given trie node across its 16 consecutive historical + // versions, only one version is stored in full format, while the others + // are stored in diff mode for storage compression. + NodeFullValueCheckpoint uint32 `toml:",omitempty"` // State scheme represents the scheme used to store ethereum states and trie // nodes on top. It can be 'hash', 'path', or none which means use the scheme @@ -121,6 +132,11 @@ type Config struct { // presence of these blocks for every new peer connection. RequiredBlocks map[uint64]common.Hash `toml:"-"` + // SlowBlockThreshold is the block execution time threshold beyond which + // detailed statistics are logged. Negative means disabled (default), zero + // logs all blocks, positive filters by execution time. + SlowBlockThreshold time.Duration `toml:",omitempty"` + // Database options SkipBcVersionCheck bool `toml:"-"` DatabaseHandles int `toml:"-"` @@ -192,6 +208,9 @@ type Config struct { // EIP-7966: eth_sendRawTransactionSync timeouts TxSyncDefaultTimeout time.Duration `toml:",omitempty"` TxSyncMaxTimeout time.Duration `toml:",omitempty"` + + // RangeLimit restricts the maximum range (end - start) for range queries. + RangeLimit uint64 `toml:",omitempty"` } // CreateConsensusEngine creates a consensus engine for the given chain config. diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 6f18dc34c5..6f94a409e5 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -31,8 +31,11 @@ func (c Config) MarshalTOML() (interface{}, error) { LogNoHistory bool `toml:",omitempty"` LogExportCheckpoints string StateHistory uint64 `toml:",omitempty"` + TrienodeHistory int64 `toml:",omitempty"` + NodeFullValueCheckpoint uint32 `toml:",omitempty"` StateScheme string `toml:",omitempty"` RequiredBlocks map[uint64]common.Hash `toml:"-"` + SlowBlockThreshold time.Duration `toml:",omitempty"` SkipBcVersionCheck bool `toml:"-"` DatabaseHandles int `toml:"-"` DatabaseCache int @@ -64,6 +67,7 @@ func (c Config) MarshalTOML() (interface{}, error) { OverrideVerkle *uint64 `toml:",omitempty"` TxSyncDefaultTimeout time.Duration `toml:",omitempty"` TxSyncMaxTimeout time.Duration `toml:",omitempty"` + RangeLimit uint64 `toml:",omitempty"` } var enc Config enc.Genesis = c.Genesis @@ -80,8 +84,11 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.LogNoHistory = c.LogNoHistory enc.LogExportCheckpoints = c.LogExportCheckpoints enc.StateHistory = c.StateHistory + enc.TrienodeHistory = c.TrienodeHistory + enc.NodeFullValueCheckpoint = c.NodeFullValueCheckpoint enc.StateScheme = c.StateScheme enc.RequiredBlocks = c.RequiredBlocks + enc.SlowBlockThreshold = c.SlowBlockThreshold enc.SkipBcVersionCheck = c.SkipBcVersionCheck enc.DatabaseHandles = c.DatabaseHandles enc.DatabaseCache = c.DatabaseCache @@ -113,6 +120,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.OverrideVerkle = c.OverrideVerkle enc.TxSyncDefaultTimeout = c.TxSyncDefaultTimeout enc.TxSyncMaxTimeout = c.TxSyncMaxTimeout + enc.RangeLimit = c.RangeLimit return &enc, nil } @@ -133,8 +141,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { LogNoHistory *bool `toml:",omitempty"` LogExportCheckpoints *string StateHistory *uint64 `toml:",omitempty"` + TrienodeHistory *int64 `toml:",omitempty"` + NodeFullValueCheckpoint *uint32 `toml:",omitempty"` StateScheme *string `toml:",omitempty"` RequiredBlocks map[uint64]common.Hash `toml:"-"` + SlowBlockThreshold *time.Duration `toml:",omitempty"` SkipBcVersionCheck *bool `toml:"-"` DatabaseHandles *int `toml:"-"` DatabaseCache *int @@ -166,6 +177,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { OverrideVerkle *uint64 `toml:",omitempty"` TxSyncDefaultTimeout *time.Duration `toml:",omitempty"` TxSyncMaxTimeout *time.Duration `toml:",omitempty"` + RangeLimit *uint64 `toml:",omitempty"` } var dec Config if err := unmarshal(&dec); err != nil { @@ -213,12 +225,21 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.StateHistory != nil { c.StateHistory = *dec.StateHistory } + if dec.TrienodeHistory != nil { + c.TrienodeHistory = *dec.TrienodeHistory + } + if dec.NodeFullValueCheckpoint != nil { + c.NodeFullValueCheckpoint = *dec.NodeFullValueCheckpoint + } if dec.StateScheme != nil { c.StateScheme = *dec.StateScheme } if dec.RequiredBlocks != nil { c.RequiredBlocks = dec.RequiredBlocks } + if dec.SlowBlockThreshold != nil { + c.SlowBlockThreshold = *dec.SlowBlockThreshold + } if dec.SkipBcVersionCheck != nil { c.SkipBcVersionCheck = *dec.SkipBcVersionCheck } @@ -312,5 +333,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.TxSyncMaxTimeout != nil { c.TxSyncMaxTimeout = *dec.TxSyncMaxTimeout } + if dec.RangeLimit != nil { + c.RangeLimit = *dec.RangeLimit + } return nil } diff --git a/eth/fetcher/metrics.go b/eth/fetcher/metrics.go index fd1678dd30..3c0d6a8fd8 100644 --- a/eth/fetcher/metrics.go +++ b/eth/fetcher/metrics.go @@ -24,6 +24,7 @@ var ( txAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/announces/in", nil) txAnnounceKnownMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/announces/known", nil) txAnnounceUnderpricedMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/announces/underpriced", nil) + txAnnounceOnchainMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/announces/onchain", nil) txAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/announces/dos", nil) txBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/broadcasts/in", nil) diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go index 4954c71819..5817dfbcf5 100644 --- a/eth/fetcher/tx_fetcher.go +++ b/eth/fetcher/tx_fetcher.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" @@ -71,6 +72,11 @@ const ( // addTxsBatchSize it the max number of transactions to add in a single batch from a peer. addTxsBatchSize = 128 + + // txOnChainCacheLimit is number of on-chain transactions to keep in a cache to avoid + // re-fetching them soon after they are mined. + // Approx 1MB for 30 minutes of transactions at 18 tps + txOnChainCacheLimit = 32768 ) var ( @@ -152,6 +158,9 @@ type TxFetcher struct { txSeq uint64 // Unique transaction sequence number underpriced *lru.Cache[common.Hash, time.Time] // Transactions discarded as too cheap (don't re-fetch) + chain *core.BlockChain // Blockchain interface for on-chain checks + txOnChainCache *lru.Cache[common.Hash, struct{}] // Cache to avoid fetching once the tx gets on chain + // Stage 1: Waiting lists for newly discovered transactions that might be // broadcast without needing explicit request/reply round trips. waitlist map[common.Hash]map[string]struct{} // Transactions waiting for an potential broadcast @@ -171,10 +180,10 @@ type TxFetcher struct { alternates map[common.Hash]map[string]struct{} // In-flight transaction alternate origins if retrieval fails // Callbacks - hasTx func(common.Hash) bool // Retrieves a tx from the local txpool - addTxs func([]*types.Transaction) []error // Insert a batch of transactions into local txpool - fetchTxs func(string, []common.Hash) error // Retrieves a set of txs from a remote peer - dropPeer func(string) // Drops a peer in case of announcement violation + validateMeta func(common.Hash, byte) error // Validate a tx metadata based on the local txpool + addTxs func([]*types.Transaction) []error // Insert a batch of transactions into local txpool + fetchTxs func(string, []common.Hash) error // Retrieves a set of txs from a remote peer + dropPeer func(string) // Drops a peer in case of announcement violation step chan struct{} // Notification channel when the fetcher loop iterates clock mclock.Clock // Monotonic clock or simulated clock for tests @@ -184,36 +193,40 @@ type TxFetcher struct { // NewTxFetcher creates a transaction fetcher to retrieve transaction // based on hash announcements. -func NewTxFetcher(hasTx func(common.Hash) bool, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, dropPeer func(string)) *TxFetcher { - return NewTxFetcherForTests(hasTx, addTxs, fetchTxs, dropPeer, mclock.System{}, time.Now, nil) +// Chain can be nil to disable on-chain checks. +func NewTxFetcher(chain *core.BlockChain, validateMeta func(common.Hash, byte) error, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, dropPeer func(string)) *TxFetcher { + return NewTxFetcherForTests(chain, validateMeta, addTxs, fetchTxs, dropPeer, mclock.System{}, time.Now, nil) } // NewTxFetcherForTests is a testing method to mock out the realtime clock with // a simulated version and the internal randomness with a deterministic one. +// Chain can be nil to disable on-chain checks. func NewTxFetcherForTests( - hasTx func(common.Hash) bool, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, dropPeer func(string), + chain *core.BlockChain, validateMeta func(common.Hash, byte) error, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, dropPeer func(string), clock mclock.Clock, realTime func() time.Time, rand *mrand.Rand) *TxFetcher { return &TxFetcher{ - notify: make(chan *txAnnounce), - cleanup: make(chan *txDelivery), - drop: make(chan *txDrop), - quit: make(chan struct{}), - waitlist: make(map[common.Hash]map[string]struct{}), - waittime: make(map[common.Hash]mclock.AbsTime), - waitslots: make(map[string]map[common.Hash]*txMetadataWithSeq), - announces: make(map[string]map[common.Hash]*txMetadataWithSeq), - announced: make(map[common.Hash]map[string]struct{}), - fetching: make(map[common.Hash]string), - requests: make(map[string]*txRequest), - alternates: make(map[common.Hash]map[string]struct{}), - underpriced: lru.NewCache[common.Hash, time.Time](maxTxUnderpricedSetSize), - hasTx: hasTx, - addTxs: addTxs, - fetchTxs: fetchTxs, - dropPeer: dropPeer, - clock: clock, - realTime: realTime, - rand: rand, + notify: make(chan *txAnnounce), + cleanup: make(chan *txDelivery), + drop: make(chan *txDrop), + quit: make(chan struct{}), + waitlist: make(map[common.Hash]map[string]struct{}), + waittime: make(map[common.Hash]mclock.AbsTime), + waitslots: make(map[string]map[common.Hash]*txMetadataWithSeq), + announces: make(map[string]map[common.Hash]*txMetadataWithSeq), + announced: make(map[common.Hash]map[string]struct{}), + fetching: make(map[common.Hash]string), + requests: make(map[string]*txRequest), + alternates: make(map[common.Hash]map[string]struct{}), + underpriced: lru.NewCache[common.Hash, time.Time](maxTxUnderpricedSetSize), + txOnChainCache: lru.NewCache[common.Hash, struct{}](txOnChainCacheLimit), + chain: chain, + validateMeta: validateMeta, + addTxs: addTxs, + fetchTxs: fetchTxs, + dropPeer: dropPeer, + clock: clock, + realTime: realTime, + rand: rand, } } @@ -233,25 +246,40 @@ func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []c unknownMetas = make([]txMetadata, 0, len(hashes)) duplicate int64 + onchain int64 underpriced int64 ) for i, hash := range hashes { - switch { - case f.hasTx(hash): + err := f.validateMeta(hash, types[i]) + if errors.Is(err, txpool.ErrAlreadyKnown) { duplicate++ - case f.isKnownUnderpriced(hash): - underpriced++ - default: - unknownHashes = append(unknownHashes, hash) + continue + } + if err != nil { + continue + } - // Transaction metadata has been available since eth68, and all - // legacy eth protocols (prior to eth68) have been deprecated. - // Therefore, metadata is always expected in the announcement. - unknownMetas = append(unknownMetas, txMetadata{kind: types[i], size: sizes[i]}) + // check on chain as well (no need to check limbo separately, as chain checks limbo too) + if _, exist := f.txOnChainCache.Get(hash); exist { + onchain++ + continue + } + + if f.isKnownUnderpriced(hash) { + underpriced++ + continue } + + unknownHashes = append(unknownHashes, hash) + + // Transaction metadata has been available since eth68, and all + // legacy eth protocols (prior to eth68) have been deprecated. + // Therefore, metadata is always expected in the announcement. + unknownMetas = append(unknownMetas, txMetadata{kind: types[i], size: sizes[i]}) } txAnnounceKnownMeter.Mark(duplicate) txAnnounceUnderpricedMeter.Mark(underpriced) + txAnnounceOnchainMeter.Mark(onchain) // If anything's left to announce, push it into the internal loop if len(unknownHashes) == 0 { @@ -358,9 +386,9 @@ func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool) otherRejectMeter.Mark(otherreject) // If 'other reject' is >25% of the deliveries in any batch, sleep a bit. - if otherreject > addTxsBatchSize/4 { + if otherreject > int64((len(batch)+3)/4) { + log.Debug("Peer delivering stale or invalid transactions", "peer", peer, "rejected", otherreject) time.Sleep(200 * time.Millisecond) - log.Debug("Peer delivering stale transactions", "peer", peer, "rejected", otherreject) } // If we encountered a protocol violation, disconnect this peer. if violation != nil { @@ -405,7 +433,20 @@ func (f *TxFetcher) loop() { waitTrigger = make(chan struct{}, 1) timeoutTrigger = make(chan struct{}, 1) + + oldHead *types.Header ) + + // Subscribe to chain events to know when transactions are added to chain + var headEventCh chan core.ChainEvent + if f.chain != nil { + headEventCh = make(chan core.ChainEvent, 10) + sub := f.chain.SubscribeChainEvent(headEventCh) + if sub != nil { + defer sub.Unsubscribe() + } + } + for { select { case ann := <-f.notify: @@ -830,6 +871,21 @@ func (f *TxFetcher) loop() { f.rescheduleTimeout(timeoutTimer, timeoutTrigger) } + case ev := <-headEventCh: + // New head(s) added + newHead := ev.Header + if oldHead != nil && newHead.ParentHash != oldHead.Hash() { + // Reorg or setHead detected, clear the cache. We could be smarter here and + // only remove/add the diff, but this is simpler and not being exact here + // only results in a few more fetches. + f.txOnChainCache.Purge() + } + oldHead = newHead + // Add all transactions from the new block to the on-chain cache + for _, tx := range ev.Transactions { + f.txOnChainCache.Add(tx.Hash(), struct{}{}) + } + case <-f.quit: return } diff --git a/eth/fetcher/tx_fetcher_test.go b/eth/fetcher/tx_fetcher_test.go index e685d50c5c..6c2719631e 100644 --- a/eth/fetcher/tx_fetcher_test.go +++ b/eth/fetcher/tx_fetcher_test.go @@ -87,6 +87,20 @@ type txFetcherTest struct { steps []interface{} } +// newTestTxFetcher creates a tx fetcher with noop callbacks, simulated clock, +// and deterministic randomness. +func newTestTxFetcher() *TxFetcher { + return NewTxFetcher( + nil, + func(common.Hash, byte) error { return nil }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + nil, + ) +} + // Tests that transaction announcements with associated metadata are added to a // waitlist, and none of them are scheduled for retrieval until the wait expires. // @@ -95,14 +109,7 @@ type txFetcherTest struct { // with all the useless extra fields. func TestTransactionFetcherWaiting(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - nil, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Initial announcement to get something into the waitlist doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{111, 222}}, @@ -297,14 +304,7 @@ func TestTransactionFetcherWaiting(t *testing.T) { // already scheduled. func TestTransactionFetcherSkipWaiting(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - nil, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Push an initial announcement through to the scheduled stage doTxNotify{ @@ -387,14 +387,7 @@ func TestTransactionFetcherSkipWaiting(t *testing.T) { // and subsequent announces block or get allotted to someone else. func TestTransactionFetcherSingletonRequesting(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - nil, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Push an initial announcement through to the scheduled stage doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{111, 222}}, @@ -493,15 +486,12 @@ func TestTransactionFetcherFailedRescheduling(t *testing.T) { proceed := make(chan struct{}) testTransactionFetcherParallel(t, txFetcherTest{ init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - nil, - func(origin string, hashes []common.Hash) error { - <-proceed - return errors.New("peer disconnected") - }, - nil, - ) + f := newTestTxFetcher() + f.fetchTxs = func(origin string, hashes []common.Hash) error { + <-proceed + return errors.New("peer disconnected") + } + return f }, steps: []interface{}{ // Push an initial announcement through to the scheduled stage @@ -576,16 +566,7 @@ func TestTransactionFetcherFailedRescheduling(t *testing.T) { // are cleaned up. func TestTransactionFetcherCleanup(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Push an initial announcement through to the scheduled stage doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, @@ -620,16 +601,7 @@ func TestTransactionFetcherCleanup(t *testing.T) { // this was a bug)). func TestTransactionFetcherCleanupEmpty(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Push an initial announcement through to the scheduled stage doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, @@ -663,16 +635,7 @@ func TestTransactionFetcherCleanupEmpty(t *testing.T) { // different peer, or self if they are after the cutoff point. func TestTransactionFetcherMissingRescheduling(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Push an initial announcement through to the scheduled stage doTxNotify{peer: "A", @@ -724,16 +687,7 @@ func TestTransactionFetcherMissingRescheduling(t *testing.T) { // delivered, the peer gets properly cleaned out from the internal state. func TestTransactionFetcherMissingCleanup(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Push an initial announcement through to the scheduled stage doTxNotify{peer: "A", @@ -773,16 +727,7 @@ func TestTransactionFetcherMissingCleanup(t *testing.T) { // Tests that transaction broadcasts properly clean up announcements. func TestTransactionFetcherBroadcasts(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Set up three transactions to be in different stats, waiting, queued and fetching doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, @@ -829,14 +774,7 @@ func TestTransactionFetcherBroadcasts(t *testing.T) { // Tests that the waiting list timers properly reset and reschedule. func TestTransactionFetcherWaitTimerResets(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - nil, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}, types: []byte{types.LegacyTxType}, sizes: []uint32{111}}, isWaiting(map[string][]announce{ @@ -899,16 +837,7 @@ func TestTransactionFetcherWaitTimerResets(t *testing.T) { // out and be re-scheduled for someone else. func TestTransactionFetcherTimeoutRescheduling(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Push an initial announcement through to the scheduled stage doTxNotify{ @@ -977,14 +906,7 @@ func TestTransactionFetcherTimeoutRescheduling(t *testing.T) { // Tests that the fetching timeout timers properly reset and reschedule. func TestTransactionFetcherTimeoutTimerResets(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - nil, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}, types: []byte{types.LegacyTxType}, sizes: []uint32{111}}, doWait{time: txArriveTimeout, step: true}, @@ -1055,14 +977,7 @@ func TestTransactionFetcherRateLimiting(t *testing.T) { }) } testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - nil, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Announce all the transactions, wait a bit and ensure only a small // percentage gets requested @@ -1085,14 +1000,7 @@ func TestTransactionFetcherRateLimiting(t *testing.T) { // be requested at a time, to keep the responses below a reasonable level. func TestTransactionFetcherBandwidthLimiting(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - nil, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Announce mid size transactions from A to verify that multiple // ones can be piled into a single request. @@ -1202,14 +1110,7 @@ func TestTransactionFetcherDoSProtection(t *testing.T) { }) } testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - nil, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Announce half of the transaction and wait for them to be scheduled doTxNotify{peer: "A", hashes: hashesA[:maxTxAnnounces/2], types: typesA[:maxTxAnnounces/2], sizes: sizesA[:maxTxAnnounces/2]}, @@ -1270,24 +1171,21 @@ func TestTransactionFetcherDoSProtection(t *testing.T) { func TestTransactionFetcherUnderpricedDedup(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - errs := make([]error, len(txs)) - for i := 0; i < len(errs); i++ { - if i%3 == 0 { - errs[i] = txpool.ErrUnderpriced - } else if i%3 == 1 { - errs[i] = txpool.ErrReplaceUnderpriced - } else { - errs[i] = txpool.ErrTxGasPriceTooLow - } + f := newTestTxFetcher() + f.addTxs = func(txs []*types.Transaction) []error { + errs := make([]error, len(txs)) + for i := 0; i < len(errs); i++ { + if i%3 == 0 { + errs[i] = txpool.ErrUnderpriced + } else if i%3 == 1 { + errs[i] = txpool.ErrReplaceUnderpriced + } else { + errs[i] = txpool.ErrTxGasPriceTooLow } - return errs - }, - func(string, []common.Hash) error { return nil }, - nil, - ) + } + return errs + } + return f }, steps: []interface{}{ // Deliver a transaction through the fetcher, but reject as underpriced @@ -1371,18 +1269,15 @@ func TestTransactionFetcherUnderpricedDoSProtection(t *testing.T) { } testTransactionFetcher(t, txFetcherTest{ init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - errs := make([]error, len(txs)) - for i := 0; i < len(errs); i++ { - errs[i] = txpool.ErrUnderpriced - } - return errs - }, - func(string, []common.Hash) error { return nil }, - nil, - ) + f := newTestTxFetcher() + f.addTxs = func(txs []*types.Transaction) []error { + errs := make([]error, len(txs)) + for i := 0; i < len(errs); i++ { + errs[i] = txpool.ErrUnderpriced + } + return errs + } + return f }, steps: append(steps, []interface{}{ // The preparation of the test has already been done in `steps`, add the last check @@ -1402,16 +1297,7 @@ func TestTransactionFetcherUnderpricedDoSProtection(t *testing.T) { // Tests that unexpected deliveries don't corrupt the internal state. func TestTransactionFetcherOutOfBoundDeliveries(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Deliver something out of the blue isWaiting(nil), @@ -1461,16 +1347,7 @@ func TestTransactionFetcherOutOfBoundDeliveries(t *testing.T) { // live or dangling stages. func TestTransactionFetcherDrop(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Set up a few hashes into various stages doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}, types: []byte{types.LegacyTxType}, sizes: []uint32{111}}, @@ -1535,16 +1412,7 @@ func TestTransactionFetcherDrop(t *testing.T) { // available peer. func TestTransactionFetcherDropRescheduling(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Set up a few hashes into various stages doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}, types: []byte{types.LegacyTxType}, sizes: []uint32{111}}, @@ -1582,14 +1450,9 @@ func TestInvalidAnnounceMetadata(t *testing.T) { drop := make(chan string, 2) testTransactionFetcherParallel(t, txFetcherTest{ init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - func(peer string) { drop <- peer }, - ) + f := newTestTxFetcher() + f.dropPeer = func(peer string) { drop <- peer } + return f }, steps: []interface{}{ // Initial announcement to get something into the waitlist @@ -1664,16 +1527,7 @@ func TestInvalidAnnounceMetadata(t *testing.T) { // announced one. func TestTransactionFetcherFuzzCrash01(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Get a transaction into fetching mode and make it dangling with a broadcast doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, @@ -1692,16 +1546,7 @@ func TestTransactionFetcherFuzzCrash01(t *testing.T) { // concurrently announced one. func TestTransactionFetcherFuzzCrash02(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Get a transaction into fetching mode and make it dangling with a broadcast doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, @@ -1722,16 +1567,7 @@ func TestTransactionFetcherFuzzCrash02(t *testing.T) { // with a concurrent notify. func TestTransactionFetcherFuzzCrash03(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Get a transaction into fetching mode and make it dangling with a broadcast doTxNotify{ @@ -1762,17 +1598,12 @@ func TestTransactionFetcherFuzzCrash04(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { - <-proceed - return errors.New("peer disconnected") - }, - nil, - ) + f := newTestTxFetcher() + f.fetchTxs = func(string, []common.Hash) error { + <-proceed + return errors.New("peer disconnected") + } + return f }, steps: []interface{}{ // Get a transaction into fetching mode and make it dangling with a broadcast @@ -1796,14 +1627,7 @@ func TestTransactionFetcherFuzzCrash04(t *testing.T) { // once they are announced in the network. func TestBlobTransactionAnnounce(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - nil, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Initial announcement to get something into the waitlist doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{111, 222}}, @@ -1864,16 +1688,7 @@ func TestBlobTransactionAnnounce(t *testing.T) { func TestTransactionFetcherDropAlternates(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, doWait{time: txArriveTimeout, step: true}, @@ -1912,6 +1727,30 @@ func TestTransactionFetcherDropAlternates(t *testing.T) { }) } +func TestTransactionFetcherWrongMetadata(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + f := newTestTxFetcher() + f.validateMeta = func(name common.Hash, kind byte) error { + switch kind { + case types.LegacyTxType, types.AccessListTxType, types.DynamicFeeTxType, types.BlobTxType, types.SetCodeTxType: + return nil + } + return types.ErrTxTypeNotSupported + } + return f + }, + steps: []interface{}{ + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}, types: []byte{0xff, types.LegacyTxType}, sizes: []uint32{111, 222}}, + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, + }), + }, + }) +} + func makeInvalidBlobTx() *types.Transaction { key, _ := crypto.GenerateKey() blob := &kzg4844.Blob{byte(0xa)} @@ -1947,20 +1786,16 @@ func TestTransactionProtocolViolation(t *testing.T) { ) testTransactionFetcherParallel(t, txFetcherTest{ init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - var errs []error - for range txs { - errs = append(errs, txpool.ErrKZGVerificationError) - } - return errs - }, - func(a string, b []common.Hash) error { - return nil - }, - func(peer string) { drop <- struct{}{} }, - ) + f := newTestTxFetcher() + f.addTxs = func(txs []*types.Transaction) []error { + var errs []error + for range txs { + errs = append(errs, txpool.ErrKZGVerificationError) + } + return errs + } + f.dropPeer = func(string) { drop <- struct{}{} } + return f }, steps: []interface{}{ // Initial announcement to get something into the waitlist @@ -2357,7 +2192,8 @@ func TestTransactionForgotten(t *testing.T) { } fetcher := NewTxFetcherForTests( - func(common.Hash) bool { return false }, + nil, + func(common.Hash, byte) error { return nil }, func(txs []*types.Transaction) []error { errs := make([]error, len(txs)) for i := 0; i < len(errs); i++ { diff --git a/eth/filters/api.go b/eth/filters/api.go index 9b418bd151..a49dd639f4 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -36,17 +36,29 @@ import ( ) var ( - errInvalidTopic = errors.New("invalid topic(s)") - errFilterNotFound = errors.New("filter not found") - errInvalidBlockRange = errors.New("invalid block range params") + errInvalidTopic = invalidParamsErr("invalid topic(s)") + errInvalidBlockRange = invalidParamsErr("invalid block range params") + errBlockRangeIntoFuture = invalidParamsErr("block range extends beyond current head block") + errBlockHashWithRange = invalidParamsErr("can't specify fromBlock/toBlock with blockHash") + errPendingLogsUnsupported = invalidParamsErr("pending logs are not supported") errUnknownBlock = errors.New("unknown block") - errBlockHashWithRange = errors.New("can't specify fromBlock/toBlock with blockHash") - errPendingLogsUnsupported = errors.New("pending logs are not supported") + errFilterNotFound = errors.New("filter not found") errExceedMaxTopics = errors.New("exceed max topics") errExceedLogQueryLimit = errors.New("exceed max addresses or topics per search position") errExceedMaxTxHashes = errors.New("exceed max number of transaction hashes allowed per transactionReceipts subscription") ) +type invalidParamsError struct { + err error +} + +func (e invalidParamsError) Error() string { return e.err.Error() } +func (e invalidParamsError) ErrorCode() int { return -32602 } + +func invalidParamsErr(format string, args ...any) error { + return invalidParamsError{fmt.Errorf(format, args...)} +} + const ( // The maximum number of topic criteria allowed, vm.LOG4 - vm.LOG0 maxTopics = 4 @@ -78,6 +90,7 @@ type FilterAPI struct { filters map[rpc.ID]*filter timeout time.Duration logQueryLimit int + rangeLimit uint64 } // NewFilterAPI returns a new FilterAPI instance. @@ -88,6 +101,7 @@ func NewFilterAPI(system *FilterSystem) *FilterAPI { filters: make(map[rpc.ID]*filter), timeout: system.cfg.Timeout, logQueryLimit: system.cfg.LogQueryLimit, + rangeLimit: system.cfg.RangeLimit, } go api.timeoutLoop(system.cfg.Timeout) @@ -174,11 +188,13 @@ func (api *FilterAPI) NewPendingTransactions(ctx context.Context, fullTx *bool) return &rpc.Subscription{}, rpc.ErrNotificationsUnsupported } - rpcSub := notifier.CreateSubscription() + var ( + rpcSub = notifier.CreateSubscription() + txs = make(chan []*types.Transaction, 128) + pendingTxSub = api.events.SubscribePendingTxs(txs) + ) go func() { - txs := make(chan []*types.Transaction, 128) - pendingTxSub := api.events.SubscribePendingTxs(txs) defer pendingTxSub.Unsubscribe() chainConfig := api.sys.backend.ChainConfig() @@ -247,11 +263,13 @@ func (api *FilterAPI) NewHeads(ctx context.Context) (*rpc.Subscription, error) { return &rpc.Subscription{}, rpc.ErrNotificationsUnsupported } - rpcSub := notifier.CreateSubscription() + var ( + rpcSub = notifier.CreateSubscription() + headers = make(chan *types.Header) + headersSub = api.events.SubscribeNewHeads(headers) + ) go func() { - headers := make(chan *types.Header) - headersSub := api.events.SubscribeNewHeads(headers) defer headersSub.Unsubscribe() for { @@ -479,7 +497,7 @@ func (api *FilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([]*type return nil, &history.PrunedHistoryError{} } // Construct the range filter - filter = api.sys.NewRangeFilter(begin, end, crit.Addresses, crit.Topics) + filter = api.sys.NewRangeFilter(begin, end, crit.Addresses, crit.Topics, api.rangeLimit) } // Run the filter and return all the logs @@ -530,8 +548,11 @@ func (api *FilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*types.Lo if f.crit.ToBlock != nil { end = f.crit.ToBlock.Int64() } + if begin >= 0 && begin < int64(api.events.backend.HistoryPruningCutoff()) { + return nil, &history.PrunedHistoryError{} + } // Construct the range filter - filter = api.sys.NewRangeFilter(begin, end, f.crit.Addresses, f.crit.Topics) + filter = api.sys.NewRangeFilter(begin, end, f.crit.Addresses, f.crit.Topics, api.rangeLimit) } // Run the filter and return all the logs logs, err := filter.Logs(ctx) diff --git a/eth/filters/filter.go b/eth/filters/filter.go index ae7cd1c298..92c16f9bf0 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -19,6 +19,7 @@ package filters import ( "context" "errors" + "fmt" "math" "time" @@ -42,15 +43,17 @@ type Filter struct { begin, end int64 // Range interval if filtering multiple blocks rangeLogsTestHook chan rangeLogsTestEvent + rangeLimit uint64 } // NewRangeFilter creates a new filter which uses a bloom filter on blocks to // figure out whether a particular block is interesting or not. -func (sys *FilterSystem) NewRangeFilter(begin, end int64, addresses []common.Address, topics [][]common.Hash) *Filter { +func (sys *FilterSystem) NewRangeFilter(begin, end int64, addresses []common.Address, topics [][]common.Hash, rangeLimit uint64) *Filter { // Create a generic filter and convert it into a range filter filter := newFilter(sys, addresses, topics) filter.begin = begin filter.end = end + filter.rangeLimit = rangeLimit return filter } @@ -141,6 +144,9 @@ func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { if err != nil { return nil, err } + if f.rangeLimit != 0 && (end-begin) > f.rangeLimit { + return nil, fmt.Errorf("exceed maximum block range: %d", f.rangeLimit) + } return f.rangeLogs(ctx, begin, end) } @@ -219,9 +225,12 @@ func (s *searchSession) updateChainView() error { if lastBlock == math.MaxUint64 { lastBlock = head } - if firstBlock > lastBlock || lastBlock > head { + if firstBlock > lastBlock { return errInvalidBlockRange } + if lastBlock > head { + return errBlockRangeIntoFuture + } s.searchRange = common.NewRange(firstBlock, lastBlock+1-firstBlock) // Trim existing match set in case a reorg may have invalidated some results @@ -379,7 +388,7 @@ func (f *Filter) rangeLogs(ctx context.Context, firstBlock, lastBlock uint64) ([ } if firstBlock > lastBlock { - return nil, nil + return nil, errInvalidBlockRange } mb := f.sys.backend.NewMatcherBackend() defer mb.Close() @@ -526,7 +535,7 @@ type ReceiptWithTx struct { // In addition to returning receipts, it also returns the corresponding transactions. // This is because receipts only contain low-level data, while user-facing data // may require additional information from the Transaction. -func filterReceipts(txHashes map[common.Hash]bool, ev core.ChainEvent) []*ReceiptWithTx { +func filterReceipts(txHashes map[common.Hash]struct{}, ev core.ChainEvent) []*ReceiptWithTx { var ret []*ReceiptWithTx receipts := ev.Receipts @@ -548,7 +557,7 @@ func filterReceipts(txHashes map[common.Hash]bool, ev core.ChainEvent) []*Receip } } else { for i, receipt := range receipts { - if txHashes[receipt.TxHash] { + if _, ok := txHashes[receipt.TxHash]; ok { ret = append(ret, &ReceiptWithTx{ Receipt: receipt, Transaction: txs[i], diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index 1d69d57ac8..5788cac53d 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -44,6 +44,7 @@ type Config struct { LogCacheSize int // maximum number of cached blocks (default: 32) Timeout time.Duration // how long filters stay active (default: 5min) LogQueryLimit int // maximum number of addresses allowed in filter criteria (default: 1000) + RangeLimit uint64 // maximum block range allowed in filter criteria (default: 0) } func (cfg Config) withDefaults() Config { @@ -187,9 +188,9 @@ type subscription struct { txs chan []*types.Transaction headers chan *types.Header receipts chan []*ReceiptWithTx - txHashes map[common.Hash]bool // contains transaction hashes for transactionReceipts subscription filtering - installed chan struct{} // closed when the filter is installed - err chan error // closed when the filter is uninstalled + txHashes map[common.Hash]struct{} // contains transaction hashes for transactionReceipts subscription filtering + installed chan struct{} // closed when the filter is installed + err chan error // closed when the filter is uninstalled } // EventSystem creates subscriptions, processes events and broadcasts them to the @@ -405,9 +406,9 @@ func (es *EventSystem) SubscribePendingTxs(txs chan []*types.Transaction) *Subsc // transactions when they are included in blocks. If txHashes is provided, only receipts // for those specific transaction hashes will be delivered. func (es *EventSystem) SubscribeTransactionReceipts(txHashes []common.Hash, receipts chan []*ReceiptWithTx) *Subscription { - hashSet := make(map[common.Hash]bool) + hashSet := make(map[common.Hash]struct{}, len(txHashes)) for _, h := range txHashes { - hashSet[h] = true + hashSet[h] = struct{}{} } sub := &subscription{ id: rpc.NewID(), diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index eeaa1397bf..d503da6ec6 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -550,7 +550,7 @@ func TestExceedLogQueryLimit(t *testing.T) { } ) - _, err := gspec.Commit(db, triedb.NewDatabase(db, nil)) + _, err := gspec.Commit(db, triedb.NewDatabase(db, nil), nil) if err != nil { t.Fatal(err) } diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index c5a955f13b..3ad88d8c10 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -109,7 +109,7 @@ func benchmarkFilters(b *testing.B, history uint64, noHistory bool) { backend.startFilterMaps(history, noHistory, filtermaps.DefaultParams) defer backend.stopFilterMaps() - filter := sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), []common.Address{addr1, addr2, addr3, addr4}, nil) + filter := sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), []common.Address{addr1, addr2, addr3, addr4}, nil, 0) for b.Loop() { filter.begin = 0 logs, _ := filter.Logs(context.Background()) @@ -205,7 +205,7 @@ func testFilters(t *testing.T, history uint64, noHistory bool) { // Hack: GenerateChainWithGenesis creates a new db. // Commit the genesis manually and use GenerateChain. - _, err = gspec.Commit(db, triedb.NewDatabase(db, nil)) + _, err = gspec.Commit(db, triedb.NewDatabase(db, nil), nil) if err != nil { t.Fatal(err) } @@ -319,70 +319,71 @@ func testFilters(t *testing.T, history uint64, noHistory bool) { want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0xdefe471992a07a02acdfbe33edaae22fbb86d7d3cec3f1b8e4e77702fb3acc1d","transactionIndex":"0x0","blockHash":"0x7a7556792ca7d37882882e2b001fe14833eaf81c2c7f865c9c771ec37a024f6b","blockTimestamp":"0x1e","logIndex":"0x0","removed":false}]`, }, { - f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), []common.Address{contract}, [][]common.Hash{{hash1, hash2, hash3, hash4}}), + f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), []common.Address{contract}, [][]common.Hash{{hash1, hash2, hash3, hash4}}, 0), want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x2","transactionHash":"0xa8028c655b6423204c8edfbc339f57b042d6bec2b6a61145d76b7c08b4cccd42","transactionIndex":"0x0","blockHash":"0x24417bb49ce44cfad65da68f33b510bf2a129c0d89ccf06acb6958b8585ccf34","blockTimestamp":"0x14","logIndex":"0x0","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0xdefe471992a07a02acdfbe33edaae22fbb86d7d3cec3f1b8e4e77702fb3acc1d","transactionIndex":"0x0","blockHash":"0x7a7556792ca7d37882882e2b001fe14833eaf81c2c7f865c9c771ec37a024f6b","blockTimestamp":"0x1e","logIndex":"0x0","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696334"],"data":"0x","blockNumber":"0x3e8","transactionHash":"0x9a87842100a638dfa5da8842b4beda691d2fd77b0c84b57f24ecfa9fb208f747","transactionIndex":"0x0","blockHash":"0xb360bad5265261c075ece02d3bf0e39498a6a76310482cdfd90588748e6c5ee0","blockTimestamp":"0x2710","logIndex":"0x0","removed":false}]`, }, { - f: sys.NewRangeFilter(900, 999, []common.Address{contract}, [][]common.Hash{{hash3}}), + f: sys.NewRangeFilter(900, 999, []common.Address{contract}, [][]common.Hash{{hash3}}, 0), }, { - f: sys.NewRangeFilter(990, int64(rpc.LatestBlockNumber), []common.Address{contract2}, [][]common.Hash{{hash3}}), + f: sys.NewRangeFilter(990, int64(rpc.LatestBlockNumber), []common.Address{contract2}, [][]common.Hash{{hash3}}, 0), want: `[{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696333"],"data":"0x","blockNumber":"0x3e7","transactionHash":"0x53e3675800c6908424b61b35a44e51ca4c73ca603e58a65b32c67968b4f42200","transactionIndex":"0x0","blockHash":"0x2e4620a2b426b0612ec6cad9603f466723edaed87f98c9137405dd4f7a2409ff","blockTimestamp":"0x2706","logIndex":"0x0","removed":false}]`, }, { - f: sys.NewRangeFilter(1, 10, []common.Address{contract}, [][]common.Hash{{hash2}, {hash1}}), + f: sys.NewRangeFilter(1, 10, []common.Address{contract}, [][]common.Hash{{hash2}, {hash1}}, 0), want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0xdefe471992a07a02acdfbe33edaae22fbb86d7d3cec3f1b8e4e77702fb3acc1d","transactionIndex":"0x0","blockHash":"0x7a7556792ca7d37882882e2b001fe14833eaf81c2c7f865c9c771ec37a024f6b","blockTimestamp":"0x1e","logIndex":"0x0","removed":false}]`, }, { - f: sys.NewRangeFilter(1, 10, nil, [][]common.Hash{{hash1, hash2}}), + f: sys.NewRangeFilter(1, 10, nil, [][]common.Hash{{hash1, hash2}}, 0), want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x2","transactionHash":"0xa8028c655b6423204c8edfbc339f57b042d6bec2b6a61145d76b7c08b4cccd42","transactionIndex":"0x0","blockHash":"0x24417bb49ce44cfad65da68f33b510bf2a129c0d89ccf06acb6958b8585ccf34","blockTimestamp":"0x14","logIndex":"0x0","removed":false},{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x2","transactionHash":"0xdba3e2ea9a7d690b722d70ee605fd67ba4c00d1d3aecd5cf187a7b92ad8eb3df","transactionIndex":"0x1","blockHash":"0x24417bb49ce44cfad65da68f33b510bf2a129c0d89ccf06acb6958b8585ccf34","blockTimestamp":"0x14","logIndex":"0x1","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0xdefe471992a07a02acdfbe33edaae22fbb86d7d3cec3f1b8e4e77702fb3acc1d","transactionIndex":"0x0","blockHash":"0x7a7556792ca7d37882882e2b001fe14833eaf81c2c7f865c9c771ec37a024f6b","blockTimestamp":"0x1e","logIndex":"0x0","removed":false}]`, }, { - f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), nil, [][]common.Hash{{common.BytesToHash([]byte("fail"))}}), + f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), nil, [][]common.Hash{{common.BytesToHash([]byte("fail"))}}, 0), }, { - f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), []common.Address{common.BytesToAddress([]byte("failmenow"))}, nil), + f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), []common.Address{common.BytesToAddress([]byte("failmenow"))}, nil, 0), }, { - f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), nil, [][]common.Hash{{common.BytesToHash([]byte("fail"))}, {hash1}}), + f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), nil, [][]common.Hash{{common.BytesToHash([]byte("fail"))}, {hash1}}, 0), }, { - f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.LatestBlockNumber), nil, nil), + f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.LatestBlockNumber), nil, nil, 0), want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696334"],"data":"0x","blockNumber":"0x3e8","transactionHash":"0x9a87842100a638dfa5da8842b4beda691d2fd77b0c84b57f24ecfa9fb208f747","transactionIndex":"0x0","blockHash":"0xb360bad5265261c075ece02d3bf0e39498a6a76310482cdfd90588748e6c5ee0","blockTimestamp":"0x2710","logIndex":"0x0","removed":false}]`, }, { - f: sys.NewRangeFilter(int64(rpc.FinalizedBlockNumber), int64(rpc.LatestBlockNumber), nil, nil), + f: sys.NewRangeFilter(int64(rpc.FinalizedBlockNumber), int64(rpc.LatestBlockNumber), nil, nil, 0), want: `[{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696333"],"data":"0x","blockNumber":"0x3e7","transactionHash":"0x53e3675800c6908424b61b35a44e51ca4c73ca603e58a65b32c67968b4f42200","transactionIndex":"0x0","blockHash":"0x2e4620a2b426b0612ec6cad9603f466723edaed87f98c9137405dd4f7a2409ff","blockTimestamp":"0x2706","logIndex":"0x0","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696334"],"data":"0x","blockNumber":"0x3e8","transactionHash":"0x9a87842100a638dfa5da8842b4beda691d2fd77b0c84b57f24ecfa9fb208f747","transactionIndex":"0x0","blockHash":"0xb360bad5265261c075ece02d3bf0e39498a6a76310482cdfd90588748e6c5ee0","blockTimestamp":"0x2710","logIndex":"0x0","removed":false}]`, }, { - f: sys.NewRangeFilter(int64(rpc.FinalizedBlockNumber), int64(rpc.FinalizedBlockNumber), nil, nil), + f: sys.NewRangeFilter(int64(rpc.FinalizedBlockNumber), int64(rpc.FinalizedBlockNumber), nil, nil, 0), want: `[{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696333"],"data":"0x","blockNumber":"0x3e7","transactionHash":"0x53e3675800c6908424b61b35a44e51ca4c73ca603e58a65b32c67968b4f42200","transactionIndex":"0x0","blockHash":"0x2e4620a2b426b0612ec6cad9603f466723edaed87f98c9137405dd4f7a2409ff","blockTimestamp":"0x2706","logIndex":"0x0","removed":false}]`, }, { - f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.FinalizedBlockNumber), nil, nil), + f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.FinalizedBlockNumber), nil, nil, 0), + err: errInvalidBlockRange.Error(), }, { - f: sys.NewRangeFilter(int64(rpc.SafeBlockNumber), int64(rpc.LatestBlockNumber), nil, nil), + f: sys.NewRangeFilter(int64(rpc.SafeBlockNumber), int64(rpc.LatestBlockNumber), nil, nil, 0), err: "safe header not found", }, { - f: sys.NewRangeFilter(int64(rpc.SafeBlockNumber), int64(rpc.SafeBlockNumber), nil, nil), + f: sys.NewRangeFilter(int64(rpc.SafeBlockNumber), int64(rpc.SafeBlockNumber), nil, nil, 0), err: "safe header not found", }, { - f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.SafeBlockNumber), nil, nil), + f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.SafeBlockNumber), nil, nil, 0), err: "safe header not found", }, { - f: sys.NewRangeFilter(int64(rpc.PendingBlockNumber), int64(rpc.PendingBlockNumber), nil, nil), + f: sys.NewRangeFilter(int64(rpc.PendingBlockNumber), int64(rpc.PendingBlockNumber), nil, nil, 0), err: errPendingLogsUnsupported.Error(), }, { - f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.PendingBlockNumber), nil, nil), + f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.PendingBlockNumber), nil, nil, 0), err: errPendingLogsUnsupported.Error(), }, { - f: sys.NewRangeFilter(int64(rpc.PendingBlockNumber), int64(rpc.LatestBlockNumber), nil, nil), + f: sys.NewRangeFilter(int64(rpc.PendingBlockNumber), int64(rpc.LatestBlockNumber), nil, nil, 0), err: errPendingLogsUnsupported.Error(), }, } { @@ -405,7 +406,7 @@ func testFilters(t *testing.T, history uint64, noHistory bool) { } t.Run("timeout", func(t *testing.T) { - f := sys.NewRangeFilter(0, rpc.LatestBlockNumber.Int64(), nil, nil) + f := sys.NewRangeFilter(0, rpc.LatestBlockNumber.Int64(), nil, nil, 0) ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Hour)) defer cancel() _, err := f.Logs(ctx) @@ -428,7 +429,7 @@ func TestRangeLogs(t *testing.T) { BaseFee: big.NewInt(params.InitialBaseFee), } ) - _, err := gspec.Commit(db, triedb.NewDatabase(db, nil)) + _, err := gspec.Commit(db, triedb.NewDatabase(db, nil), nil) if err != nil { t.Fatal(err) } @@ -468,7 +469,7 @@ func TestRangeLogs(t *testing.T) { newFilter := func(begin, end int64) { testCase++ event = 0 - filter = sys.NewRangeFilter(begin, end, addresses, nil) + filter = sys.NewRangeFilter(begin, end, addresses, nil, 0) filter.rangeLogsTestHook = make(chan rangeLogsTestEvent) go func(filter *Filter) { filter.Logs(context.Background()) @@ -605,3 +606,41 @@ func TestRangeLogs(t *testing.T) { expEvent(rangeLogsTestReorg, 400, 901) expEvent(rangeLogsTestDone, 0, 0) } + +func TestRangeLimit(t *testing.T) { + db := rawdb.NewMemoryDatabase() + backend, sys := newTestFilterSystem(db, Config{}) + defer db.Close() + + gspec := &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + _, err := gspec.Commit(db, triedb.NewDatabase(db, nil), nil) + if err != nil { + t.Fatal(err) + } + chain, _ := core.GenerateChain(gspec.Config, gspec.ToBlock(), ethash.NewFaker(), db, 10, func(i int, gen *core.BlockGen) {}) + options := core.DefaultConfig().WithStateScheme(rawdb.HashScheme) + options.TxIndexer = &core.TxIndexerConfig{ + Limit: 0, // index all txs + } + bc, err := core.NewBlockChain(db, nil, gspec, ethash.NewFaker(), options) + if err != nil { + t.Fatal(err) + } + _, err = bc.InsertChain(chain) + if err != nil { + t.Fatal(err) + } + backend.startFilterMaps(0, false, filtermaps.DefaultParams) + defer backend.stopFilterMaps() + + // Set rangeLimit to 5, but request a range of 9 (end - begin = 9, from 0 to 9) + filter := sys.NewRangeFilter(0, 9, nil, nil, 5) + _, err = filter.Logs(context.Background()) + if err == nil || !strings.Contains(err.Error(), "exceed maximum block range") { + t.Fatalf("expected range limit error, got %v", err) + } +} diff --git a/eth/gasestimator/gasestimator.go b/eth/gasestimator/gasestimator.go index af988f2f25..df0dbfecd1 100644 --- a/eth/gasestimator/gasestimator.go +++ b/eth/gasestimator/gasestimator.go @@ -20,7 +20,6 @@ import ( "context" "errors" "fmt" - "math" "math/big" "github.com/ethereum/go-ethereum/arbitrum/filter" @@ -298,7 +297,7 @@ func run(ctx context.Context, call *core.Message, opts *Options) (*core.Executio evm.Cancel() }() // Execute the call, returning a wrapped error or the result - result, err := core.ApplyMessage(evm, call, new(core.GasPool).AddGas(math.MaxUint64)) + result, err := core.ApplyMessage(evm, call, nil) if vmerr := dirtyState.Error(); vmerr != nil { return nil, vmerr } diff --git a/eth/gasprice/gasprice.go b/eth/gasprice/gasprice.go index ea30d88ff3..f1b1a67cfc 100644 --- a/eth/gasprice/gasprice.go +++ b/eth/gasprice/gasprice.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" + "github.com/holiman/uint256" ) const sampleNumber = 3 // Number of transactions sampled in a block @@ -258,12 +259,12 @@ func (oracle *Oracle) getBlockValues(ctx context.Context, blockNum uint64, limit sortedTxs := make([]*types.Transaction, len(txs)) copy(sortedTxs, txs) baseFee := block.BaseFee() + baseFee256 := new(uint256.Int) + if baseFee != nil { + baseFee256.SetFromBig(baseFee) + } slices.SortFunc(sortedTxs, func(a, b *types.Transaction) int { - // It's okay to discard the error because a tx would never be - // accepted into a block with an invalid effective tip. - tip1, _ := a.EffectiveGasTip(baseFee) - tip2, _ := b.EffectiveGasTip(baseFee) - return tip1.Cmp(tip2) + return a.EffectiveGasTipCmp(b, baseFee256) }) var prices []*big.Int diff --git a/eth/handler.go b/eth/handler.go index ff970e2ba6..27b5e60697 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -86,12 +86,15 @@ type txPool interface { // Pending should return pending transactions. // The slice should be modifiable by the caller. - Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction + Pending(filter txpool.PendingFilter) (map[common.Address][]*txpool.LazyTransaction, int) // SubscribeTransactions subscribes to new transaction events. The subscriber // can decide whether to receive notifications only for newly seen transactions // or also for reorged out ones. SubscribeTransactions(ch chan<- core.NewTxsEvent, reorgs bool) event.Subscription + + // FilterType returns whether the given tx type is supported by the txPool. + FilterType(kind byte) bool } // handlerConfig is the collection of initialization parameters to create a full @@ -111,9 +114,7 @@ type handlerConfig struct { type handler struct { nodeID enode.ID networkID uint64 - - snapSync atomic.Bool // Flag whether snap sync is enabled (gets disabled if we already have blocks) - synced atomic.Bool // Flag whether we're considered synchronised (enables transaction processing) + synced atomic.Bool // Flag whether we're considered synchronised (enables transaction processing) database ethdb.Database txpool txPool @@ -161,40 +162,13 @@ func newHandler(config *handlerConfig) (*handler, error) { handlerDoneCh: make(chan struct{}), handlerStartCh: make(chan struct{}), } - if config.Sync == ethconfig.FullSync { - // The database seems empty as the current block is the genesis. Yet the snap - // block is ahead, so snap sync was enabled for this node at a certain point. - // The scenarios where this can happen is - // * if the user manually (or via a bad block) rolled back a snap sync node - // below the sync point. - // * the last snap sync is not finished while user specifies a full sync this - // time. But we don't have any recent state for full sync. - // In these cases however it's safe to reenable snap sync. - fullBlock, snapBlock := h.chain.CurrentBlock(), h.chain.CurrentSnapBlock() - if fullBlock.Number.Uint64() == 0 && snapBlock.Number.Uint64() > 0 { - h.snapSync.Store(true) - log.Warn("Switch sync mode from full sync to snap sync", "reason", "snap sync incomplete") - } else if !h.chain.HasState(fullBlock.Root) { - h.snapSync.Store(true) - log.Warn("Switch sync mode from full sync to snap sync", "reason", "head state missing") - } - } else { - head := h.chain.CurrentBlock() - if head.Number.Uint64() > 0 && h.chain.HasState(head.Root) { - log.Info("Switch sync mode from snap sync to full sync", "reason", "snap sync complete") - } else { - // If snap sync was requested and our database is empty, grant it - h.snapSync.Store(true) - log.Info("Enabled snap sync", "head", head.Number, "hash", head.Hash()) - } - } + // Construct the downloader (long sync) + h.downloader = downloader.New(config.Database, config.Sync, h.eventMux, h.chain, h.removePeer, h.enableSyncedFeatures) + // If snap sync is requested but snapshots are disabled, fail loudly - if h.snapSync.Load() && (config.Chain.Snapshots() == nil && config.Chain.TrieDB().Scheme() == rawdb.HashScheme) { + if h.downloader.ConfigSyncMode() == ethconfig.SnapSync && (config.Chain.Snapshots() == nil && config.Chain.TrieDB().Scheme() == rawdb.HashScheme) { return nil, errors.New("snap sync not supported with snapshots disabled") } - // Construct the downloader (long sync) - h.downloader = downloader.New(config.Database, h.eventMux, h.chain, h.removePeer, h.enableSyncedFeatures) - fetchTx := func(peer string, hashes []common.Hash) error { p := h.peers.peer(peer) if p == nil { @@ -205,7 +179,16 @@ func newHandler(config *handlerConfig) (*handler, error) { addTxs := func(txs []*types.Transaction) []error { return h.txpool.Add(txs, false) } - h.txFetcher = fetcher.NewTxFetcher(h.txpool.Has, addTxs, fetchTx, h.removePeer) + validateMeta := func(tx common.Hash, kind byte) error { + if h.txpool.Has(tx) { + return txpool.ErrAlreadyKnown + } + if !h.txpool.FilterType(kind) { + return types.ErrTxTypeNotSupported + } + return nil + } + h.txFetcher = fetcher.NewTxFetcher(h.chain, validateMeta, addTxs, fetchTx, h.removePeer) return h, nil } @@ -267,7 +250,7 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { return err } reject := false // reserved peer slots - if h.snapSync.Load() { + if h.downloader.ConfigSyncMode() == ethconfig.SnapSync { if snap == nil { // If we are running snap-sync, we want to reserve roughly half the peer // slots for peers supporting the snap protocol. @@ -524,7 +507,7 @@ func (h *handler) BroadcastTransactions(txs types.Transactions) { annCount += len(hashes) peer.AsyncSendPooledTransactionHashes(hashes) } - log.Debug("Distributed transactions", "plaintxs", len(txs)-blobTxs-largeTxs, "blobtxs", blobTxs, "largetxs", largeTxs, + log.Trace("Distributed transactions", "plaintxs", len(txs)-blobTxs-largeTxs, "blobtxs", blobTxs, "largetxs", largeTxs, "bcastpeers", len(txset), "bcastcount", directCount, "annpeers", len(annos), "anncount", annCount) } @@ -544,15 +527,7 @@ func (h *handler) txBroadcastLoop() { // enableSyncedFeatures enables the post-sync functionalities when the initial // sync is finished. func (h *handler) enableSyncedFeatures() { - // Mark the local node as synced. h.synced.Store(true) - - // If we were running snap sync and it finished, disable doing another - // round on next sync cycle - if h.snapSync.Load() { - log.Info("Snap sync complete, auto disabling") - h.snapSync.Store(false) - } } // blockRangeState holds the state of the block range update broadcasting mechanism. @@ -590,7 +565,7 @@ func (h *handler) blockRangeLoop(st *blockRangeState) { if ev == nil { continue } - if _, ok := ev.Data.(downloader.StartEvent); ok && h.snapSync.Load() { + if _, ok := ev.Data.(downloader.StartEvent); ok && h.downloader.ConfigSyncMode() == ethconfig.SnapSync { h.blockRangeWhileSnapSyncing(st) } case <-st.headCh: diff --git a/eth/handler_eth.go b/eth/handler_eth.go index 11742b14ad..8704a86af4 100644 --- a/eth/handler_eth.go +++ b/eth/handler_eth.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/protocols/eth" @@ -61,19 +62,42 @@ func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error { return h.txFetcher.Notify(peer.ID(), packet.Types, packet.Sizes, packet.Hashes) case *eth.TransactionsPacket: - for _, tx := range *packet { - if tx.Type() == types.BlobTxType { - return errors.New("disallowed broadcast blob transaction") - } + txs, err := packet.Items() + if err != nil { + return fmt.Errorf("Transactions: %v", err) + } + if err := handleTransactions(peer, txs, true); err != nil { + return fmt.Errorf("Transactions: %v", err) } - return h.txFetcher.Enqueue(peer.ID(), *packet, false) + return h.txFetcher.Enqueue(peer.ID(), txs, false) - case *eth.PooledTransactionsResponse: - // If we receive any blob transactions missing sidecars, or with - // sidecars that don't correspond to the versioned hashes reported - // in the header, disconnect from the sending peer. - for _, tx := range *packet { - if tx.Type() == types.BlobTxType { + case *eth.PooledTransactionsPacket: + txs, err := packet.List.Items() + if err != nil { + return fmt.Errorf("PooledTransactions: %v", err) + } + if err := handleTransactions(peer, txs, false); err != nil { + return fmt.Errorf("PooledTransactions: %v", err) + } + return h.txFetcher.Enqueue(peer.ID(), txs, true) + + default: + return fmt.Errorf("unexpected eth packet type: %T", packet) + } +} + +// handleTransactions marks all given transactions as known to the peer +// and performs basic validations. +func handleTransactions(peer *eth.Peer, list []*types.Transaction, directBroadcast bool) error { + seen := make(map[common.Hash]struct{}) + for _, tx := range list { + if tx.Type() == types.BlobTxType { + if directBroadcast { + return errors.New("disallowed broadcast blob transaction") + } else { + // If we receive any blob transactions missing sidecars, or with + // sidecars that don't correspond to the versioned hashes reported + // in the header, disconnect from the sending peer. if tx.BlobTxSidecar() == nil { return errors.New("received sidecar-less blob transaction") } @@ -82,9 +106,16 @@ func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error { } } } - return h.txFetcher.Enqueue(peer.ID(), *packet, true) - default: - return fmt.Errorf("unexpected eth packet type: %T", packet) + // Check for duplicates. + hash := tx.Hash() + if _, exists := seen[hash]; exists { + return fmt.Errorf("multiple copies of the same hash %v", hash) + } + seen[hash] = struct{}{} + + // Mark as known. + peer.MarkTransaction(hash) } + return nil } diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index 3968ca59d4..8ab0247004 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -38,9 +38,8 @@ import ( // testEthHandler is a mock event handler to listen for inbound network requests // on the `eth` protocol and convert them into a more easily testable form. type testEthHandler struct { - blockBroadcasts event.Feed - txAnnounces event.Feed - txBroadcasts event.Feed + txAnnounces event.Feed + txBroadcasts event.Feed } func (h *testEthHandler) Chain() *core.BlockChain { panic("no backing chain") } @@ -51,20 +50,24 @@ func (h *testEthHandler) PeerInfo(enode.ID) interface{} { panic("not used func (h *testEthHandler) Handle(peer *eth.Peer, packet eth.Packet) error { switch packet := packet.(type) { - case *eth.NewBlockPacket: - h.blockBroadcasts.Send(packet.Block) - return nil - case *eth.NewPooledTransactionHashesPacket: h.txAnnounces.Send(packet.Hashes) return nil case *eth.TransactionsPacket: - h.txBroadcasts.Send(([]*types.Transaction)(*packet)) + txs, err := packet.Items() + if err != nil { + return err + } + h.txBroadcasts.Send(txs) return nil - case *eth.PooledTransactionsResponse: - h.txBroadcasts.Send(([]*types.Transaction)(*packet)) + case *eth.PooledTransactionsPacket: + txs, err := packet.List.Items() + if err != nil { + return err + } + h.txBroadcasts.Send(txs) return nil default: @@ -74,7 +77,7 @@ func (h *testEthHandler) Handle(peer *eth.Peer, packet eth.Packet) error { // Tests that peers are correctly accepted (or rejected) based on the advertised // fork IDs in the protocol handshake. -func TestForkIDSplit68(t *testing.T) { testForkIDSplit(t, eth.ETH68) } +func TestForkIDSplit69(t *testing.T) { testForkIDSplit(t, eth.ETH69) } func testForkIDSplit(t *testing.T, protocol uint) { t.Parallel() @@ -226,13 +229,13 @@ func testForkIDSplit(t *testing.T, protocol uint) { } // Tests that received transactions are added to the local pool. -func TestRecvTransactions68(t *testing.T) { testRecvTransactions(t, eth.ETH68) } +func TestRecvTransactions69(t *testing.T) { testRecvTransactions(t, eth.ETH69) } func testRecvTransactions(t *testing.T, protocol uint) { t.Parallel() // Create a message handler, configure it to accept transactions and watch them - handler := newTestHandler() + handler := newTestHandler(ethconfig.FullSync) defer handler.close() handler.handler.synced.Store(true) // mark synced to accept transactions @@ -255,7 +258,8 @@ func testRecvTransactions(t *testing.T, protocol uint) { return eth.Handle((*ethHandler)(handler.handler), peer) }) // Run the handshake locally to avoid spinning up a source handler - if err := src.Handshake(1, handler.chain, eth.BlockRangeUpdatePacket{}); err != nil { + head := handler.chain.CurrentBlock() + if err := src.Handshake(1, handler.chain, eth.BlockRangeUpdatePacket{EarliestBlock: 0, LatestBlock: head.Number.Uint64(), LatestBlockHash: head.Hash()}); err != nil { t.Fatalf("failed to run protocol handshake") } // Send the transaction to the sink and verify that it's added to the tx pool @@ -278,13 +282,13 @@ func testRecvTransactions(t *testing.T, protocol uint) { } // This test checks that pending transactions are sent. -func TestSendTransactions68(t *testing.T) { testSendTransactions(t, eth.ETH68) } +func TestSendTransactions69(t *testing.T) { testSendTransactions(t, eth.ETH69) } func testSendTransactions(t *testing.T, protocol uint) { t.Parallel() // Create a message handler and fill the pool with big transactions - handler := newTestHandler() + handler := newTestHandler(ethconfig.FullSync) defer handler.close() insert := make([]*types.Transaction, 100) @@ -310,7 +314,8 @@ func testSendTransactions(t *testing.T, protocol uint) { return eth.Handle((*ethHandler)(handler.handler), peer) }) // Run the handshake locally to avoid spinning up a source handler - if err := sink.Handshake(1, handler.chain, eth.BlockRangeUpdatePacket{}); err != nil { + head := handler.chain.CurrentBlock() + if err := sink.Handshake(1, handler.chain, eth.BlockRangeUpdatePacket{EarliestBlock: 0, LatestBlock: head.Number.Uint64(), LatestBlockHash: head.Hash()}); err != nil { t.Fatalf("failed to run protocol handshake") } // After the handshake completes, the source handler should stream the sink @@ -330,22 +335,16 @@ func testSendTransactions(t *testing.T, protocol uint) { // Make sure we get all the transactions on the correct channels seen := make(map[common.Hash]struct{}) for len(seen) < len(insert) { - switch protocol { - case 68: - select { - case hashes := <-anns: - for _, hash := range hashes { - if _, ok := seen[hash]; ok { - t.Errorf("duplicate transaction announced: %x", hash) - } - seen[hash] = struct{}{} + select { + case hashes := <-anns: + for _, hash := range hashes { + if _, ok := seen[hash]; ok { + t.Errorf("duplicate transaction announced: %x", hash) } - case <-bcasts: - t.Errorf("initial tx broadcast received on post eth/66") + seen[hash] = struct{}{} } - - default: - panic("unsupported protocol, please extend test") + case <-bcasts: + t.Errorf("initial tx broadcast received on post eth/66") } } for _, tx := range insert { @@ -357,7 +356,7 @@ func testSendTransactions(t *testing.T, protocol uint) { // Tests that transactions get propagated to all attached peers, either via direct // broadcasts or via announcements/retrievals. -func TestTransactionPropagation68(t *testing.T) { testTransactionPropagation(t, eth.ETH68) } +func TestTransactionPropagation69(t *testing.T) { testTransactionPropagation(t, eth.ETH69) } func testTransactionPropagation(t *testing.T, protocol uint) { t.Parallel() @@ -365,13 +364,12 @@ func testTransactionPropagation(t *testing.T, protocol uint) { // Create a source handler to send transactions from and a number of sinks // to receive them. We need multiple sinks since a one-to-one peering would // broadcast all transactions without announcement. - source := newTestHandler() - source.handler.snapSync.Store(false) // Avoid requiring snap, otherwise some will be dropped below + source := newTestHandler(ethconfig.FullSync) defer source.close() sinks := make([]*testHandler, 10) for i := 0; i < len(sinks); i++ { - sinks[i] = newTestHandler() + sinks[i] = newTestHandler(ethconfig.FullSync) defer sinks[i].close() sinks[i].handler.synced.Store(true) // mark synced to accept transactions diff --git a/eth/handler_test.go b/eth/handler_test.go index 971873e221..6de9137f81 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -128,10 +128,11 @@ func (p *testTxPool) Add(txs []*types.Transaction, sync bool) []error { } // Pending returns all the transactions known to the pool -func (p *testTxPool) Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction { +func (p *testTxPool) Pending(filter txpool.PendingFilter) (map[common.Address][]*txpool.LazyTransaction, int) { p.lock.RLock() defer p.lock.RUnlock() + var count int batches := make(map[common.Address][]*types.Transaction) for _, tx := range p.pool { from, _ := types.Sender(types.HomesteadSigner{}, tx) @@ -152,9 +153,10 @@ func (p *testTxPool) Pending(filter txpool.PendingFilter) map[common.Address][]* Gas: tx.Gas(), BlobGas: tx.BlobGas(), }) + count++ } } - return pending + return pending, count } // SubscribeTransactions should return an event subscription of NewTxsEvent and @@ -163,6 +165,15 @@ func (p *testTxPool) SubscribeTransactions(ch chan<- core.NewTxsEvent, reorgs bo return p.txFeed.Subscribe(ch) } +// FilterType should check whether the pool supports the given type of transactions. +func (p *testTxPool) FilterType(kind byte) bool { + switch kind { + case types.LegacyTxType, types.AccessListTxType, types.DynamicFeeTxType, types.BlobTxType, types.SetCodeTxType: + return true + } + return false +} + // testHandler is a live implementation of the Ethereum protocol handler, just // preinitialized with some sane testing defaults and the transaction pool mocked // out. @@ -174,13 +185,13 @@ type testHandler struct { } // newTestHandler creates a new handler for testing purposes with no blocks. -func newTestHandler() *testHandler { - return newTestHandlerWithBlocks(0) +func newTestHandler(mode ethconfig.SyncMode) *testHandler { + return newTestHandlerWithBlocks(0, mode) } // newTestHandlerWithBlocks creates a new handler for testing purposes, with a // given number of initial blocks. -func newTestHandlerWithBlocks(blocks int) *testHandler { +func newTestHandlerWithBlocks(blocks int, mode ethconfig.SyncMode) *testHandler { // Create a database pre-initialize with a genesis block db := rawdb.NewMemoryDatabase() gspec := &core.Genesis{ @@ -200,7 +211,7 @@ func newTestHandlerWithBlocks(blocks int) *testHandler { Chain: chain, TxPool: txpool, Network: 1, - Sync: ethconfig.SnapSync, + Sync: mode, BloomCache: 1, }) handler.Start(1000) diff --git a/eth/protocols/eth/dispatcher.go b/eth/protocols/eth/dispatcher.go index cba40596fc..3f78fb4646 100644 --- a/eth/protocols/eth/dispatcher.go +++ b/eth/protocols/eth/dispatcher.go @@ -22,6 +22,7 @@ import ( "time" "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/tracker" ) var ( @@ -47,9 +48,10 @@ type Request struct { sink chan *Response // Channel to deliver the response on cancel chan struct{} // Channel to cancel requests ahead of time - code uint64 // Message code of the request packet - want uint64 // Message code of the response packet - data interface{} // Data content of the request packet + code uint64 // Message code of the request packet + want uint64 // Message code of the response packet + numItems int // Number of requested items + data interface{} // Data content of the request packet Peer string // Demultiplexer if cross-peer requests are batched together Sent time.Time // Timestamp when the request was sent @@ -190,20 +192,31 @@ func (p *Peer) dispatchResponse(res *Response, metadata func() interface{}) erro func (p *Peer) dispatcher() { pending := make(map[uint64]*Request) +loop: for { select { case reqOp := <-p.reqDispatch: req := reqOp.req req.Sent = time.Now() - requestTracker.Track(p.id, p.version, req.code, req.want, req.id) - err := p2p.Send(p.rw, req.code, req.data) - reqOp.fail <- err - - if err == nil { - pending[req.id] = req + treq := tracker.Request{ + ID: req.id, + ReqCode: req.code, + RespCode: req.want, + Size: req.numItems, + } + if err := p.tracker.Track(treq); err != nil { + reqOp.fail <- err + continue loop + } + if err := p2p.Send(p.rw, req.code, req.data); err != nil { + reqOp.fail <- err + continue loop } + pending[req.id] = req + reqOp.fail <- nil + case cancelOp := <-p.reqCancel: // Retrieve the pending request to cancel and short circuit if it // has already been serviced and is not available anymore @@ -220,9 +233,6 @@ func (p *Peer) dispatcher() { res := resOp.res res.Req = pending[res.id] - // Independent if the request exists or not, track this packet - requestTracker.Fulfil(p.id, p.version, res.code, res.id) - switch { case res.Req == nil: // Response arrived with an untracked ID. Since even cancelled @@ -249,6 +259,7 @@ func (p *Peer) dispatcher() { } case <-p.term: + p.tracker.Stop() return } } diff --git a/eth/protocols/eth/handler.go b/eth/protocols/eth/handler.go index ea624d0bf4..f1090608c5 100644 --- a/eth/protocols/eth/handler.go +++ b/eth/protocols/eth/handler.go @@ -166,21 +166,6 @@ type Decoder interface { Time() time.Time } -var eth68 = map[uint64]msgHandler{ - NewBlockHashesMsg: handleNewBlockhashes, - NewBlockMsg: handleNewBlock, - TransactionsMsg: handleTransactions, - NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes, - GetBlockHeadersMsg: handleGetBlockHeaders, - BlockHeadersMsg: handleBlockHeaders, - GetBlockBodiesMsg: handleGetBlockBodies, - BlockBodiesMsg: handleBlockBodies, - GetReceiptsMsg: handleGetReceipts68, - ReceiptsMsg: handleReceipts[*ReceiptList68], - GetPooledTransactionsMsg: handleGetPooledTransactions, - PooledTransactionsMsg: handlePooledTransactions, -} - var eth69 = map[uint64]msgHandler{ TransactionsMsg: handleTransactions, NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes, @@ -188,8 +173,8 @@ var eth69 = map[uint64]msgHandler{ BlockHeadersMsg: handleBlockHeaders, GetBlockBodiesMsg: handleGetBlockBodies, BlockBodiesMsg: handleBlockBodies, - GetReceiptsMsg: handleGetReceipts69, - ReceiptsMsg: handleReceipts[*ReceiptList69], + GetReceiptsMsg: handleGetReceipts, + ReceiptsMsg: handleReceipts, GetPooledTransactionsMsg: handleGetPooledTransactions, PooledTransactionsMsg: handlePooledTransactions, BlockRangeUpdateMsg: handleBlockRangeUpdate, @@ -209,9 +194,7 @@ func handleMessage(backend Backend, peer *Peer) error { defer msg.Discard() var handlers map[uint64]msgHandler - if peer.version == ETH68 { - handlers = eth68 - } else if peer.version == ETH69 { + if peer.version == ETH69 { handlers = eth69 } else { return fmt.Errorf("unknown eth protocol version: %v", peer.version) diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index 749b6544ea..d95644e54c 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -23,9 +23,11 @@ import ( "math/big" "math/rand" "os" + "reflect" "testing" "time" + "github.com/davecgh/go-spew/spew" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/beacon" "github.com/ethereum/go-ethereum/consensus/ethash" @@ -42,6 +44,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" "github.com/holiman/uint256" ) @@ -171,7 +174,7 @@ func (b *testBackend) Handle(*Peer, Packet) error { } // Tests that block headers can be retrieved from a remote chain based on user queries. -func TestGetBlockHeaders68(t *testing.T) { testGetBlockHeaders(t, ETH68) } +func TestGetBlockHeaders69(t *testing.T) { testGetBlockHeaders(t, ETH69) } func testGetBlockHeaders(t *testing.T, protocol uint) { t.Parallel() @@ -360,8 +363,8 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { GetBlockHeadersRequest: tt.query, }) if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, &BlockHeadersPacket{ - RequestId: 123, - BlockHeadersRequest: headers, + RequestId: 123, + List: encodeRL(headers), }); err != nil { t.Errorf("test %d: headers mismatch: %v", i, err) } @@ -374,7 +377,7 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { RequestId: 456, GetBlockHeadersRequest: tt.query, }) - expected := &BlockHeadersPacket{RequestId: 456, BlockHeadersRequest: headers} + expected := &BlockHeadersPacket{RequestId: 456, List: encodeRL(headers)} if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, expected); err != nil { t.Errorf("test %d by hash: headers mismatch: %v", i, err) } @@ -384,7 +387,7 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { } // Tests that block contents can be retrieved from a remote chain based on their hashes. -func TestGetBlockBodies68(t *testing.T) { testGetBlockBodies(t, ETH68) } +func TestGetBlockBodies69(t *testing.T) { testGetBlockBodies(t, ETH69) } func testGetBlockBodies(t *testing.T, protocol uint) { t.Parallel() @@ -437,7 +440,7 @@ func testGetBlockBodies(t *testing.T, protocol uint) { // Collect the hashes to request, and the response to expect var ( hashes []common.Hash - bodies []*BlockBody + bodies []BlockBody seen = make(map[int64]bool) ) for j := 0; j < tt.random; j++ { @@ -449,7 +452,7 @@ func testGetBlockBodies(t *testing.T, protocol uint) { block := backend.chain.GetBlockByNumber(uint64(num)) hashes = append(hashes, block.Hash()) if len(bodies) < tt.expected { - bodies = append(bodies, &BlockBody{Transactions: block.Transactions(), Uncles: block.Uncles(), Withdrawals: block.Withdrawals()}) + bodies = append(bodies, encodeBody(block)) } break } @@ -459,7 +462,7 @@ func testGetBlockBodies(t *testing.T, protocol uint) { hashes = append(hashes, hash) if tt.available[j] && len(bodies) < tt.expected { block := backend.chain.GetBlockByHash(hash) - bodies = append(bodies, &BlockBody{Transactions: block.Transactions(), Uncles: block.Uncles(), Withdrawals: block.Withdrawals()}) + bodies = append(bodies, encodeBody(block)) } } @@ -469,16 +472,71 @@ func testGetBlockBodies(t *testing.T, protocol uint) { GetBlockBodiesRequest: hashes, }) if err := p2p.ExpectMsg(peer.app, BlockBodiesMsg, &BlockBodiesPacket{ - RequestId: 123, - BlockBodiesResponse: bodies, + RequestId: 123, + List: encodeRL(bodies), }); err != nil { t.Fatalf("test %d: bodies mismatch: %v", i, err) } } } +func encodeBody(b *types.Block) BlockBody { + body := BlockBody{ + Transactions: encodeRL([]*types.Transaction(b.Transactions())), + Uncles: encodeRL(b.Uncles()), + } + if b.Withdrawals() != nil { + wd := encodeRL([]*types.Withdrawal(b.Withdrawals())) + body.Withdrawals = &wd + } + return body +} + +func TestHashBody(t *testing.T) { + key, _ := crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + signer := types.NewCancunSigner(big.NewInt(1)) + + // create block 1 + header := &types.Header{Number: big.NewInt(11)} + txs := []*types.Transaction{ + types.MustSignNewTx(key, signer, &types.DynamicFeeTx{ + ChainID: big.NewInt(1), + Nonce: 1, + Data: []byte("testing"), + }), + types.MustSignNewTx(key, signer, &types.LegacyTx{ + Nonce: 2, + Data: []byte("testing"), + }), + } + uncles := []*types.Header{{Number: big.NewInt(10)}} + body1 := &types.Body{Transactions: txs, Uncles: uncles} + block1 := types.NewBlock(header, body1, nil, trie.NewStackTrie(nil)) + + // create block 2 (has withdrawals) + header2 := &types.Header{Number: big.NewInt(12)} + body2 := &types.Body{ + Withdrawals: []*types.Withdrawal{{Index: 10}, {Index: 11}}, + } + block2 := types.NewBlock(header2, body2, nil, trie.NewStackTrie(nil)) + + expectedHashes := BlockBodyHashes{ + TransactionRoots: []common.Hash{block1.TxHash(), block2.TxHash()}, + WithdrawalRoots: []common.Hash{common.Hash{}, *block2.Header().WithdrawalsHash}, + UncleHashes: []common.Hash{block1.UncleHash(), block2.UncleHash()}, + } + + // compute hash like protocol handler does + protocolBodies := []BlockBody{encodeBody(block1), encodeBody(block2)} + hashes := hashBodyParts(protocolBodies) + if !reflect.DeepEqual(hashes, expectedHashes) { + t.Errorf("wrong hashes: %s", spew.Sdump(hashes)) + t.Logf("expected: %s", spew.Sdump(expectedHashes)) + } +} + // Tests that the transaction receipts can be retrieved based on hashes. -func TestGetBlockReceipts68(t *testing.T) { testGetBlockReceipts(t, ETH68) } +func TestGetBlockReceipts69(t *testing.T) { testGetBlockReceipts(t, ETH69) } func testGetBlockReceipts(t *testing.T, protocol uint) { t.Parallel() @@ -528,13 +586,13 @@ func testGetBlockReceipts(t *testing.T, protocol uint) { // Collect the hashes to request, and the response to expect var ( hashes []common.Hash - receipts []*ReceiptList68 + receipts rlp.RawList[*ReceiptList] ) for i := uint64(0); i <= backend.chain.CurrentBlock().Number.Uint64(); i++ { block := backend.chain.GetBlockByNumber(i) hashes = append(hashes, block.Hash()) - trs := backend.chain.GetReceiptsByHash(block.Hash()) - receipts = append(receipts, NewReceiptList68(trs)) + br := backend.chain.GetReceiptsByHash(block.Hash()) + receipts.Append(NewReceiptList(br)) } // Send the hash request and verify the response @@ -542,7 +600,7 @@ func testGetBlockReceipts(t *testing.T, protocol uint) { RequestId: 123, GetReceiptsRequest: hashes, }) - if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, &ReceiptsPacket[*ReceiptList68]{ + if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, &ReceiptsPacket{ RequestId: 123, List: receipts, }); err != nil { @@ -598,7 +656,7 @@ func setup() (*testBackend, *testPeer) { } } backend := newTestBackendWithGenerator(maxBodiesServe+15, true, false, gen) - peer, _ := newTestPeer("peer", ETH68, backend) + peer, _ := newTestPeer("peer", ETH69, backend) // Discard all messages go func() { for { @@ -643,7 +701,7 @@ func testGetPooledTransaction(t *testing.T, blobTx bool) { backend := newTestBackendWithGenerator(0, true, true, nil) defer backend.close() - peer, _ := newTestPeer("peer", ETH68, backend) + peer, _ := newTestPeer("peer", ETH69, backend) defer peer.close() var ( @@ -688,10 +746,18 @@ func testGetPooledTransaction(t *testing.T, blobTx bool) { RequestId: 123, GetPooledTransactionsRequest: []common.Hash{tx.Hash()}, }) - if err := p2p.ExpectMsg(peer.app, PooledTransactionsMsg, PooledTransactionsPacket{ - RequestId: 123, - PooledTransactionsResponse: []*types.Transaction{tx}, + if err := p2p.ExpectMsg(peer.app, PooledTransactionsMsg, &PooledTransactionsPacket{ + RequestId: 123, + List: encodeRL([]*types.Transaction{tx}), }); err != nil { t.Errorf("pooled transaction mismatch: %v", err) } } + +func encodeRL[T any](slice []T) rlp.RawList[T] { + rl, err := rlp.EncodeToRawList(slice) + if err != nil { + panic(err) + } + return rl +} diff --git a/eth/protocols/eth/handlers.go b/eth/protocols/eth/handlers.go index aad3353d88..90717472f9 100644 --- a/eth/protocols/eth/handlers.go +++ b/eth/protocols/eth/handlers.go @@ -17,23 +17,21 @@ package eth import ( + "bytes" "encoding/json" - "errors" "fmt" - "time" + "math" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/tracker" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" ) -// requestTracker is a singleton tracker for eth/66 and newer request times. -var requestTracker = tracker.New(ProtocolName, 5*time.Minute) - func handleGetBlockHeaders(backend Backend, msg Decoder, peer *Peer) error { // Decode the complex header query var query GetBlockHeadersPacket @@ -248,73 +246,27 @@ func ServiceGetBlockBodiesQuery(chain *core.BlockChain, query GetBlockBodiesRequ return bodies } -func handleGetReceipts68(backend Backend, msg Decoder, peer *Peer) error { +func handleGetReceipts(backend Backend, msg Decoder, peer *Peer) error { // Decode the block receipts retrieval message var query GetReceiptsPacket if err := msg.Decode(&query); err != nil { return err } - response := ServiceGetReceiptsQuery68(backend.Chain(), query.GetReceiptsRequest) + response := ServiceGetReceiptsQuery(backend.Chain(), query.GetReceiptsRequest) return peer.ReplyReceiptsRLP(query.RequestId, response) } -func handleGetReceipts69(backend Backend, msg Decoder, peer *Peer) error { - // Decode the block receipts retrieval message - var query GetReceiptsPacket - if err := msg.Decode(&query); err != nil { - return err - } - response := serviceGetReceiptsQuery69(backend.Chain(), query.GetReceiptsRequest) - return peer.ReplyReceiptsRLP(query.RequestId, response) -} - -// ServiceGetReceiptsQuery68 assembles the response to a receipt query. It is -// exposed to allow external packages to test protocol behavior. -func ServiceGetReceiptsQuery68(chain *core.BlockChain, query GetReceiptsRequest) []rlp.RawValue { - // Gather state data until the fetch or network limits is reached - var ( - bytes int - receipts []rlp.RawValue - ) - for lookups, hash := range query { - if bytes >= softResponseLimit || len(receipts) >= maxReceiptsServe || - lookups >= 2*maxReceiptsServe { - break - } - // Retrieve the requested block's receipts - results := chain.GetReceiptsRLP(hash) - if results == nil { - if header := chain.GetHeaderByHash(hash); header == nil || header.ReceiptHash != types.EmptyRootHash { - continue - } - } else { - body := chain.GetBodyRLP(hash) - if body == nil { - continue - } - var err error - results, err = blockReceiptsToNetwork68(results, body) - if err != nil { - log.Error("Error in block receipts conversion", "hash", hash, "err", err) - continue - } - } - receipts = append(receipts, results) - bytes += len(results) - } - return receipts -} - -// serviceGetReceiptsQuery69 assembles the response to a receipt query. -// It does not send the bloom filters for the receipts -func serviceGetReceiptsQuery69(chain *core.BlockChain, query GetReceiptsRequest) []rlp.RawValue { +// ServiceGetReceiptsQuery assembles the response to a receipt query. +// It does not send the bloom filters for the receipts. It is exposed +// to allow external packages to test protocol behavior. +func ServiceGetReceiptsQuery(chain *core.BlockChain, query GetReceiptsRequest) rlp.RawList[*ReceiptList] { // Gather state data until the fetch or network limits is reached var ( bytes int - receipts []rlp.RawValue + receipts rlp.RawList[*ReceiptList] ) for lookups, hash := range query { - if bytes >= softResponseLimit || len(receipts) >= maxReceiptsServe || + if bytes >= softResponseLimit || receipts.Len() >= maxReceiptsServe || lookups >= 2*maxReceiptsServe { break } @@ -330,35 +282,36 @@ func serviceGetReceiptsQuery69(chain *core.BlockChain, query GetReceiptsRequest) continue } var err error - results, err = blockReceiptsToNetwork69(results, body) + results, err = blockReceiptsToNetwork(results, body) if err != nil { log.Error("Error in block receipts conversion", "hash", hash, "err", err) continue } } - receipts = append(receipts, results) + receipts.AppendRaw(results) bytes += len(results) } return receipts } -func handleNewBlockhashes(backend Backend, msg Decoder, peer *Peer) error { - return errors.New("block announcements disallowed") // We dropped support for non-merge networks -} - -func handleNewBlock(backend Backend, msg Decoder, peer *Peer) error { - return errors.New("block broadcasts disallowed") // We dropped support for non-merge networks -} - func handleBlockHeaders(backend Backend, msg Decoder, peer *Peer) error { // A batch of headers arrived to one of our previous requests res := new(BlockHeadersPacket) if err := msg.Decode(res); err != nil { return err } + tresp := tracker.Response{ID: res.RequestId, MsgCode: BlockHeadersMsg, Size: res.List.Len()} + if err := peer.tracker.Fulfil(tresp); err != nil { + return fmt.Errorf("BlockHeaders: %w", err) + } + headers, err := res.List.Items() + if err != nil { + return fmt.Errorf("BlockHeaders: %w", err) + } + metadata := func() interface{} { - hashes := make([]common.Hash, len(res.BlockHeadersRequest)) - for i, header := range res.BlockHeadersRequest { + hashes := make([]common.Hash, len(headers)) + for i, header := range headers { hashes[i] = header.Hash() } return hashes @@ -366,7 +319,7 @@ func handleBlockHeaders(backend Backend, msg Decoder, peer *Peer) error { return peer.dispatchResponse(&Response{ id: res.RequestId, code: BlockHeadersMsg, - Res: &res.BlockHeadersRequest, + Res: (*BlockHeadersRequest)(&headers), }, metadata) } @@ -376,53 +329,145 @@ func handleBlockBodies(backend Backend, msg Decoder, peer *Peer) error { if err := msg.Decode(res); err != nil { return err } - metadata := func() interface{} { - var ( - txsHashes = make([]common.Hash, len(res.BlockBodiesResponse)) - uncleHashes = make([]common.Hash, len(res.BlockBodiesResponse)) - withdrawalHashes = make([]common.Hash, len(res.BlockBodiesResponse)) - ) - hasher := trie.NewStackTrie(nil) - for i, body := range res.BlockBodiesResponse { - txsHashes[i] = types.DeriveSha(types.Transactions(body.Transactions), hasher) - uncleHashes[i] = types.CalcUncleHash(body.Uncles) - if body.Withdrawals != nil { - withdrawalHashes[i] = types.DeriveSha(types.Withdrawals(body.Withdrawals), hasher) - } - } - return [][]common.Hash{txsHashes, uncleHashes, withdrawalHashes} + + // Check against the request. + length := res.List.Len() + tresp := tracker.Response{ID: res.RequestId, MsgCode: BlockBodiesMsg, Size: length} + if err := peer.tracker.Fulfil(tresp); err != nil { + return fmt.Errorf("BlockBodies: %w", err) } + + // Collect items and dispatch. + items, err := res.List.Items() + if err != nil { + return fmt.Errorf("BlockBodies: %w", err) + } + metadata := func() any { return hashBodyParts(items) } return peer.dispatchResponse(&Response{ id: res.RequestId, code: BlockBodiesMsg, - Res: &res.BlockBodiesResponse, + Res: (*BlockBodiesResponse)(&items), }, metadata) } -func handleReceipts[L ReceiptsList](backend Backend, msg Decoder, peer *Peer) error { +// BlockBodyHashes contains the lists of block body part roots for a list of block bodies. +type BlockBodyHashes struct { + TransactionRoots []common.Hash + WithdrawalRoots []common.Hash + UncleHashes []common.Hash +} + +func hashBodyParts(items []BlockBody) BlockBodyHashes { + h := BlockBodyHashes{ + TransactionRoots: make([]common.Hash, len(items)), + WithdrawalRoots: make([]common.Hash, len(items)), + UncleHashes: make([]common.Hash, len(items)), + } + hasher := trie.NewStackTrie(nil) + for i, body := range items { + // txs + txsList := newDerivableRawList(&body.Transactions, writeTxForHash) + h.TransactionRoots[i] = types.DeriveSha(txsList, hasher) + // uncles + if body.Uncles.Len() == 0 { + h.UncleHashes[i] = types.EmptyUncleHash + } else { + h.UncleHashes[i] = crypto.Keccak256Hash(body.Uncles.Bytes()) + } + // withdrawals + if body.Withdrawals != nil { + wdlist := newDerivableRawList(body.Withdrawals, nil) + h.WithdrawalRoots[i] = types.DeriveSha(wdlist, hasher) + } + } + return h +} + +// derivableRawList implements types.DerivableList for a serialized RLP list. +type derivableRawList struct { + data []byte + offsets []uint32 + write func([]byte, *bytes.Buffer) +} + +func newDerivableRawList[T any](list *rlp.RawList[T], write func([]byte, *bytes.Buffer)) *derivableRawList { + dl := derivableRawList{data: list.Content(), write: write} + if dl.write == nil { + // default transform is identity + dl.write = func(b []byte, buf *bytes.Buffer) { buf.Write(b) } + } + // Assert to ensure 32-bit offsets are valid. This can never trigger + // unless a block body component or p2p receipt list is larger than 4GB. + if uint(len(dl.data)) > math.MaxUint32 { + panic("list data too big for derivableRawList") + } + it := list.ContentIterator() + dl.offsets = make([]uint32, list.Len()) + for i := 0; it.Next(); i++ { + dl.offsets[i] = uint32(it.Offset()) + } + return &dl +} + +// Len returns the number of items in the list. +func (dl *derivableRawList) Len() int { + return len(dl.offsets) +} + +// EncodeIndex writes the i'th item to the buffer. +func (dl *derivableRawList) EncodeIndex(i int, buf *bytes.Buffer) { + start := dl.offsets[i] + end := uint32(len(dl.data)) + if i != len(dl.offsets)-1 { + end = dl.offsets[i+1] + } + dl.write(dl.data[start:end], buf) +} + +// writeTxForHash changes a transaction in 'network encoding' into the format used for +// the transactions MPT. +func writeTxForHash(tx []byte, buf *bytes.Buffer) { + k, content, _, _ := rlp.Split(tx) + if k == rlp.List { + buf.Write(tx) // legacy tx + } else { + buf.Write(content) // typed tx + } +} + +func handleReceipts(backend Backend, msg Decoder, peer *Peer) error { // A batch of receipts arrived to one of our previous requests - res := new(ReceiptsPacket[L]) + res := new(ReceiptsPacket) if err := msg.Decode(res); err != nil { return err } - // Assign temporary hashing buffer to each list item, the same buffer is shared - // between all receipt list instances. - buffers := new(receiptListBuffers) - for i := range res.List { - res.List[i].setBuffers(buffers) + + tresp := tracker.Response{ID: res.RequestId, MsgCode: ReceiptsMsg, Size: res.List.Len()} + if err := peer.tracker.Fulfil(tresp); err != nil { + return fmt.Errorf("Receipts: %w", err) + } + + receiptLists, err := res.List.Items() + if err != nil { + return fmt.Errorf("Receipts: %w", err) } metadata := func() interface{} { hasher := trie.NewStackTrie(nil) - hashes := make([]common.Hash, len(res.List)) - for i := range res.List { - hashes[i] = types.DeriveSha(res.List[i], hasher) + hashes := make([]common.Hash, len(receiptLists)) + for i := range receiptLists { + hashes[i] = types.DeriveSha(receiptLists[i].Derivable(), hasher) } return hashes } + var enc ReceiptsRLPResponse - for i := range res.List { - enc = append(enc, res.List[i].EncodeForStorage()) + for i := range receiptLists { + encReceipts, err := receiptLists[i].EncodeForStorage() + if err != nil { + return fmt.Errorf("Receipts: invalid list %d: %v", i, err) + } + enc = append(enc, encReceipts) } return peer.dispatchResponse(&Response{ id: res.RequestId, @@ -446,7 +491,7 @@ func handleNewPooledTransactionHashes(backend Backend, msg Decoder, peer *Peer) } // Schedule all the unknown hashes for retrieval for _, hash := range ann.Hashes { - peer.markTransaction(hash) + peer.MarkTransaction(hash) } return backend.Handle(peer, ann) } @@ -494,19 +539,8 @@ func handleTransactions(backend Backend, msg Decoder, peer *Peer) error { if err := msg.Decode(&txs); err != nil { return err } - // Duplicate transactions are not allowed - seen := make(map[common.Hash]struct{}) - for i, tx := range txs { - // Validate and mark the remote transaction - if tx == nil { - return fmt.Errorf("Transactions: transaction %d is nil", i) - } - hash := tx.Hash() - if _, exists := seen[hash]; exists { - return fmt.Errorf("Transactions: multiple copies of the same hash %v", hash) - } - seen[hash] = struct{}{} - peer.markTransaction(hash) + if txs.Len() > maxTransactionAnnouncements { + return fmt.Errorf("too many transactions") } return backend.Handle(peer, &txs) } @@ -516,28 +550,22 @@ func handlePooledTransactions(backend Backend, msg Decoder, peer *Peer) error { if !backend.AcceptTxs() { return nil } - // Transactions can be processed, parse all of them and deliver to the pool - var txs PooledTransactionsPacket - if err := msg.Decode(&txs); err != nil { + + // Check against request and decode. + var resp PooledTransactionsPacket + if err := msg.Decode(&resp); err != nil { return err } - // Duplicate transactions are not allowed - seen := make(map[common.Hash]struct{}) - for i, tx := range txs.PooledTransactionsResponse { - // Validate and mark the remote transaction - if tx == nil { - return fmt.Errorf("PooledTransactions: transaction %d is nil", i) - } - hash := tx.Hash() - if _, exists := seen[hash]; exists { - return fmt.Errorf("PooledTransactions: multiple copies of the same hash %v", hash) - } - seen[hash] = struct{}{} - peer.markTransaction(hash) + tresp := tracker.Response{ + ID: resp.RequestId, + MsgCode: PooledTransactionsMsg, + Size: resp.List.Len(), + } + if err := peer.tracker.Fulfil(tresp); err != nil { + return fmt.Errorf("PooledTransactions: %w", err) } - requestTracker.Fulfil(peer.id, peer.version, PooledTransactionsMsg, txs.RequestId) - return backend.Handle(peer, &txs.PooledTransactionsResponse) + return backend.Handle(peer, &resp) } func handleBlockRangeUpdate(backend Backend, msg Decoder, peer *Peer) error { diff --git a/eth/protocols/eth/handshake.go b/eth/protocols/eth/handshake.go index bb3d1b8eb4..359e4e36bb 100644 --- a/eth/protocols/eth/handshake.go +++ b/eth/protocols/eth/handshake.go @@ -36,62 +36,6 @@ const ( // Handshake executes the eth protocol handshake, negotiating version number, // network IDs, difficulties, head and genesis blocks. func (p *Peer) Handshake(networkID uint64, chain forkid.Blockchain, rangeMsg BlockRangeUpdatePacket) error { - switch p.version { - case ETH69: - return p.handshake69(networkID, chain, rangeMsg) - case ETH68: - return p.handshake68(networkID, chain) - default: - return errors.New("unsupported protocol version") - } -} - -func (p *Peer) handshake68(networkID uint64, chain forkid.Blockchain) error { - var ( - genesis = chain.Genesis() - latest = chain.CurrentHeader() - forkID = forkid.NewID(chain.Config(), genesis, latest.Number.Uint64(), latest.Time) - forkFilter = forkid.NewFilter(chain) - ) - errc := make(chan error, 2) - go func() { - pkt := &StatusPacket68{ - ProtocolVersion: uint32(p.version), - NetworkID: networkID, - Head: latest.Hash(), - Genesis: genesis.Hash(), - ForkID: forkID, - } - errc <- p2p.Send(p.rw, StatusMsg, pkt) - }() - var status StatusPacket68 // safe to read after two values have been received from errc - go func() { - errc <- p.readStatus68(networkID, &status, genesis.Hash(), forkFilter) - }() - - return waitForHandshake(errc, p) -} - -func (p *Peer) readStatus68(networkID uint64, status *StatusPacket68, genesis common.Hash, forkFilter forkid.Filter) error { - if err := p.readStatusMsg(status); err != nil { - return err - } - if status.NetworkID != networkID { - return fmt.Errorf("%w: %d (!= %d)", errNetworkIDMismatch, status.NetworkID, networkID) - } - if uint(status.ProtocolVersion) != p.version { - return fmt.Errorf("%w: %d (!= %d)", errProtocolVersionMismatch, status.ProtocolVersion, p.version) - } - if status.Genesis != genesis { - return fmt.Errorf("%w: %x (!= %x)", errGenesisMismatch, status.Genesis, genesis) - } - if err := forkFilter(status.ForkID); err != nil { - return fmt.Errorf("%w: %v", errForkIDRejected, err) - } - return nil -} - -func (p *Peer) handshake69(networkID uint64, chain forkid.Blockchain, rangeMsg BlockRangeUpdatePacket) error { var ( genesis = chain.Genesis() latest = chain.CurrentHeader() @@ -101,7 +45,7 @@ func (p *Peer) handshake69(networkID uint64, chain forkid.Blockchain, rangeMsg B errc := make(chan error, 2) go func() { - pkt := &StatusPacket69{ + pkt := &StatusPacket{ ProtocolVersion: uint32(p.version), NetworkID: networkID, Genesis: genesis.Hash(), @@ -112,15 +56,15 @@ func (p *Peer) handshake69(networkID uint64, chain forkid.Blockchain, rangeMsg B } errc <- p2p.Send(p.rw, StatusMsg, pkt) }() - var status StatusPacket69 // safe to read after two values have been received from errc + var status StatusPacket // safe to read after two values have been received from errc go func() { - errc <- p.readStatus69(networkID, &status, genesis.Hash(), forkFilter) + errc <- p.readStatus(networkID, &status, genesis.Hash(), forkFilter) }() return waitForHandshake(errc, p) } -func (p *Peer) readStatus69(networkID uint64, status *StatusPacket69, genesis common.Hash, forkFilter forkid.Filter) error { +func (p *Peer) readStatus(networkID uint64, status *StatusPacket, genesis common.Hash, forkFilter forkid.Filter) error { if err := p.readStatusMsg(status); err != nil { return err } @@ -191,16 +135,16 @@ func markError(p *Peer, err error) { return } m := meters.get(p.Inbound()) - switch errors.Unwrap(err) { - case errNetworkIDMismatch: + switch { + case errors.Is(err, errNetworkIDMismatch): m.networkIDMismatch.Mark(1) - case errProtocolVersionMismatch: + case errors.Is(err, errProtocolVersionMismatch): m.protocolVersionMismatch.Mark(1) - case errGenesisMismatch: + case errors.Is(err, errGenesisMismatch): m.genesisMismatch.Mark(1) - case errForkIDRejected: + case errors.Is(err, errForkIDRejected): m.forkidRejected.Mark(1) - case p2p.DiscReadTimeout: + case errors.Is(err, p2p.DiscReadTimeout): m.timeoutError.Mark(1) default: m.peerError.Mark(1) diff --git a/eth/protocols/eth/handshake_test.go b/eth/protocols/eth/handshake_test.go index 2fab3ea5a8..e2f1e7592a 100644 --- a/eth/protocols/eth/handshake_test.go +++ b/eth/protocols/eth/handshake_test.go @@ -18,7 +18,6 @@ package eth import ( "errors" - "math/big" "testing" "github.com/ethereum/go-ethereum/common" @@ -28,7 +27,7 @@ import ( ) // Tests that handshake failures are detected and reported correctly. -func TestHandshake68(t *testing.T) { testHandshake(t, ETH68) } +func TestHandshake69(t *testing.T) { testHandshake(t, ETH69) } func testHandshake(t *testing.T, protocol uint) { t.Parallel() @@ -52,21 +51,25 @@ func testHandshake(t *testing.T, protocol uint) { want: errNoStatusMsg, }, { - code: StatusMsg, data: StatusPacket68{10, 1, new(big.Int), head.Hash(), genesis.Hash(), forkID}, + code: StatusMsg, data: StatusPacket{10, 1, genesis.Hash(), forkID, 0, head.Number.Uint64(), head.Hash()}, want: errProtocolVersionMismatch, }, { - code: StatusMsg, data: StatusPacket68{uint32(protocol), 999, new(big.Int), head.Hash(), genesis.Hash(), forkID}, + code: StatusMsg, data: StatusPacket{uint32(protocol), 999, genesis.Hash(), forkID, 0, head.Number.Uint64(), head.Hash()}, want: errNetworkIDMismatch, }, { - code: StatusMsg, data: StatusPacket68{uint32(protocol), 1, new(big.Int), head.Hash(), common.Hash{3}, forkID}, + code: StatusMsg, data: StatusPacket{uint32(protocol), 1, common.Hash{3}, forkID, 0, head.Number.Uint64(), head.Hash()}, want: errGenesisMismatch, }, { - code: StatusMsg, data: StatusPacket68{uint32(protocol), 1, new(big.Int), head.Hash(), genesis.Hash(), forkid.ID{Hash: [4]byte{0x00, 0x01, 0x02, 0x03}}}, + code: StatusMsg, data: StatusPacket{uint32(protocol), 1, genesis.Hash(), forkid.ID{Hash: [4]byte{0x00, 0x01, 0x02, 0x03}}, 0, head.Number.Uint64(), head.Hash()}, want: errForkIDRejected, }, + { + code: StatusMsg, data: StatusPacket{uint32(protocol), 1, genesis.Hash(), forkID, head.Number.Uint64() + 1, head.Number.Uint64(), head.Hash()}, + want: errInvalidBlockRange, + }, } for i, test := range tests { // Create the two peers to shake with each other diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index 40c54a3570..3c6d58d670 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -19,11 +19,13 @@ package eth import ( "math/rand" "sync/atomic" + "time" mapset "github.com/deckarep/golang-set/v2" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/tracker" "github.com/ethereum/go-ethereum/rlp" ) @@ -43,9 +45,10 @@ const ( // Peer is a collection of relevant information we have about a `eth` peer. type Peer struct { + *p2p.Peer // The embedded P2P package peer + id string // Unique ID for the peer, cached - *p2p.Peer // The embedded P2P package peer rw p2p.MsgReadWriter // Input/output streams for snap version uint // Protocol version negotiated lastRange atomic.Pointer[BlockRangeUpdatePacket] @@ -55,6 +58,7 @@ type Peer struct { txBroadcast chan []common.Hash // Channel used to queue transaction propagation requests txAnnounce chan []common.Hash // Channel used to queue transaction announcement requests + tracker *tracker.Tracker reqDispatch chan *request // Dispatch channel to send requests and track then until fulfillment reqCancel chan *cancel // Dispatch channel to cancel pending requests and untrack them resDispatch chan *response // Dispatch channel to fulfil pending requests and untrack them @@ -65,14 +69,17 @@ type Peer struct { // NewPeer creates a wrapper for a network connection and negotiated protocol // version. func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool) *Peer { + cap := p2p.Cap{Name: ProtocolName, Version: version} + id := p.ID().String() peer := &Peer{ - id: p.ID().String(), + id: id, Peer: p, rw: rw, version: version, knownTxs: newKnownCache(maxKnownTxs), txBroadcast: make(chan []common.Hash), txAnnounce: make(chan []common.Hash), + tracker: tracker.New(cap, id, 5*time.Minute), reqDispatch: make(chan *request), reqCancel: make(chan *cancel), resDispatch: make(chan *response), @@ -115,9 +122,9 @@ func (p *Peer) KnownTransaction(hash common.Hash) bool { return p.knownTxs.Contains(hash) } -// markTransaction marks a transaction as known for the peer, ensuring that it +// MarkTransaction marks a transaction as known for the peer, ensuring that it // will never be propagated to this particular peer. -func (p *Peer) markTransaction(hash common.Hash) { +func (p *Peer) MarkTransaction(hash common.Hash) { // If we reached the memory allowance, drop a previously known transaction hash p.knownTxs.Add(hash) } @@ -208,10 +215,10 @@ func (p *Peer) ReplyBlockBodiesRLP(id uint64, bodies []rlp.RawValue) error { } // ReplyReceiptsRLP is the response to GetReceipts. -func (p *Peer) ReplyReceiptsRLP(id uint64, receipts []rlp.RawValue) error { - return p2p.Send(p.rw, ReceiptsMsg, &ReceiptsRLPPacket{ - RequestId: id, - ReceiptsRLPResponse: receipts, +func (p *Peer) ReplyReceiptsRLP(id uint64, receipts rlp.RawList[*ReceiptList]) error { + return p2p.Send(p.rw, ReceiptsMsg, &ReceiptsPacket{ + RequestId: id, + List: receipts, }) } @@ -222,10 +229,11 @@ func (p *Peer) RequestOneHeader(hash common.Hash, sink chan *Response) (*Request id := rand.Uint64() req := &Request{ - id: id, - sink: sink, - code: GetBlockHeadersMsg, - want: BlockHeadersMsg, + id: id, + sink: sink, + code: GetBlockHeadersMsg, + want: BlockHeadersMsg, + numItems: 1, data: &GetBlockHeadersPacket{ RequestId: id, GetBlockHeadersRequest: &GetBlockHeadersRequest{ @@ -249,10 +257,11 @@ func (p *Peer) RequestHeadersByHash(origin common.Hash, amount int, skip int, re id := rand.Uint64() req := &Request{ - id: id, - sink: sink, - code: GetBlockHeadersMsg, - want: BlockHeadersMsg, + id: id, + sink: sink, + code: GetBlockHeadersMsg, + want: BlockHeadersMsg, + numItems: amount, data: &GetBlockHeadersPacket{ RequestId: id, GetBlockHeadersRequest: &GetBlockHeadersRequest{ @@ -276,10 +285,11 @@ func (p *Peer) RequestHeadersByNumber(origin uint64, amount int, skip int, rever id := rand.Uint64() req := &Request{ - id: id, - sink: sink, - code: GetBlockHeadersMsg, - want: BlockHeadersMsg, + id: id, + sink: sink, + code: GetBlockHeadersMsg, + want: BlockHeadersMsg, + numItems: amount, data: &GetBlockHeadersPacket{ RequestId: id, GetBlockHeadersRequest: &GetBlockHeadersRequest{ @@ -303,10 +313,11 @@ func (p *Peer) RequestBodies(hashes []common.Hash, sink chan *Response) (*Reques id := rand.Uint64() req := &Request{ - id: id, - sink: sink, - code: GetBlockBodiesMsg, - want: BlockBodiesMsg, + id: id, + sink: sink, + code: GetBlockBodiesMsg, + want: BlockBodiesMsg, + numItems: len(hashes), data: &GetBlockBodiesPacket{ RequestId: id, GetBlockBodiesRequest: hashes, @@ -324,10 +335,11 @@ func (p *Peer) RequestReceipts(hashes []common.Hash, sink chan *Response) (*Requ id := rand.Uint64() req := &Request{ - id: id, - sink: sink, - code: GetReceiptsMsg, - want: ReceiptsMsg, + id: id, + sink: sink, + code: GetReceiptsMsg, + want: ReceiptsMsg, + numItems: len(hashes), data: &GetReceiptsPacket{ RequestId: id, GetReceiptsRequest: hashes, @@ -341,10 +353,18 @@ func (p *Peer) RequestReceipts(hashes []common.Hash, sink chan *Response) (*Requ // RequestTxs fetches a batch of transactions from a remote node. func (p *Peer) RequestTxs(hashes []common.Hash) error { - p.Log().Debug("Fetching batch of transactions", "count", len(hashes)) + p.Log().Trace("Fetching batch of transactions", "count", len(hashes)) id := rand.Uint64() - requestTracker.Track(p.id, p.version, GetPooledTransactionsMsg, PooledTransactionsMsg, id) + err := p.tracker.Track(tracker.Request{ + ID: id, + ReqCode: GetPooledTransactionsMsg, + RespCode: PooledTransactionsMsg, + Size: len(hashes), + }) + if err != nil { + return err + } return p2p.Send(p.rw, GetPooledTransactionsMsg, &GetPooledTransactionsPacket{ RequestId: id, GetPooledTransactionsRequest: hashes, diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go index 7c41e7a996..ef65a7d034 100644 --- a/eth/protocols/eth/protocol.go +++ b/eth/protocols/eth/protocol.go @@ -20,7 +20,6 @@ import ( "errors" "fmt" "io" - "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/forkid" @@ -30,7 +29,6 @@ import ( // Constants to match up protocol versions and messages const ( - ETH68 = 68 ETH69 = 69 ) @@ -40,15 +38,18 @@ const ProtocolName = "eth" // ProtocolVersions are the supported versions of the `eth` protocol (first // is primary). -var ProtocolVersions = []uint{ETH69, ETH68} +var ProtocolVersions = []uint{ETH69} // protocolLengths are the number of implemented message corresponding to // different protocol versions. -var protocolLengths = map[uint]uint64{ETH68: 17, ETH69: 18} +var protocolLengths = map[uint]uint64{ETH69: 18} // maxMessageSize is the maximum cap on the size of a protocol message. const maxMessageSize = 10 * 1024 * 1024 +// This is the maximum number of transactions in a Transactions message. +const maxTransactionAnnouncements = 5000 + const ( StatusMsg = 0x00 NewBlockHashesMsg = 0x01 @@ -85,17 +86,7 @@ type Packet interface { } // StatusPacket is the network packet for the status message. -type StatusPacket68 struct { - ProtocolVersion uint32 - NetworkID uint64 - TD *big.Int - Head common.Hash - Genesis common.Hash - ForkID forkid.ID -} - -// StatusPacket69 is the network packet for the status message. -type StatusPacket69 struct { +type StatusPacket struct { ProtocolVersion uint32 NetworkID uint64 Genesis common.Hash @@ -106,28 +97,10 @@ type StatusPacket69 struct { LatestBlockHash common.Hash } -// NewBlockHashesPacket is the network packet for the block announcements. -type NewBlockHashesPacket []struct { - Hash common.Hash // Hash of one particular block being announced - Number uint64 // Number of one particular block being announced -} - -// Unpack retrieves the block hashes and numbers from the announcement packet -// and returns them in a split flat format that's more consistent with the -// internal data structures. -func (p *NewBlockHashesPacket) Unpack() ([]common.Hash, []uint64) { - var ( - hashes = make([]common.Hash, len(*p)) - numbers = make([]uint64, len(*p)) - ) - for i, body := range *p { - hashes[i], numbers[i] = body.Hash, body.Number - } - return hashes, numbers -} - // TransactionsPacket is the network packet for broadcasting new transactions. -type TransactionsPacket []*types.Transaction +type TransactionsPacket struct { + rlp.RawList[*types.Transaction] +} // GetBlockHeadersRequest represents a block header query. type GetBlockHeadersRequest struct { @@ -185,7 +158,7 @@ type BlockHeadersRequest []*types.Header // BlockHeadersPacket represents a block header response over with request ID wrapping. type BlockHeadersPacket struct { RequestId uint64 - BlockHeadersRequest + List rlp.RawList[*types.Header] } // BlockHeadersRLPResponse represents a block header response, to use when we already @@ -198,12 +171,6 @@ type BlockHeadersRLPPacket struct { BlockHeadersRLPResponse } -// NewBlockPacket is the network packet for the block propagation message. -type NewBlockPacket struct { - Block *types.Block - TD *big.Int -} - // GetBlockBodiesRequest represents a block body query. type GetBlockBodiesRequest []common.Hash @@ -213,14 +180,11 @@ type GetBlockBodiesPacket struct { GetBlockBodiesRequest } -// BlockBodiesResponse is the network packet for block content distribution. -type BlockBodiesResponse []*BlockBody - // BlockBodiesPacket is the network packet for block content distribution with // request ID wrapping. type BlockBodiesPacket struct { RequestId uint64 - BlockBodiesResponse + List rlp.RawList[BlockBody] } // BlockBodiesRLPResponse is used for replying to block body requests, in cases @@ -234,25 +198,14 @@ type BlockBodiesRLPPacket struct { BlockBodiesRLPResponse } +// BlockBodiesResponse is the network packet for block content distribution. +type BlockBodiesResponse []BlockBody + // BlockBody represents the data content of a single block. type BlockBody struct { - Transactions []*types.Transaction // Transactions contained within a block - Uncles []*types.Header // Uncles contained within a block - Withdrawals []*types.Withdrawal `rlp:"optional"` // Withdrawals contained within a block -} - -// Unpack retrieves the transactions and uncles from the range packet and returns -// them in a split flat format that's more consistent with the internal data structures. -func (p *BlockBodiesResponse) Unpack() ([][]*types.Transaction, [][]*types.Header, [][]*types.Withdrawal) { - var ( - txset = make([][]*types.Transaction, len(*p)) - uncleset = make([][]*types.Header, len(*p)) - withdrawalset = make([][]*types.Withdrawal, len(*p)) - ) - for i, body := range *p { - txset[i], uncleset[i], withdrawalset[i] = body.Transactions, body.Uncles, body.Withdrawals - } - return txset, uncleset, withdrawalset + Transactions rlp.RawList[*types.Transaction] + Uncles rlp.RawList[*types.Header] + Withdrawals *rlp.RawList[*types.Withdrawal] `rlp:"optional"` } // GetReceiptsRequest represents a block receipts query. @@ -267,30 +220,16 @@ type GetReceiptsPacket struct { // ReceiptsResponse is the network packet for block receipts distribution. type ReceiptsResponse []types.Receipts -// ReceiptsList is a type constraint for block receceipt list types. -type ReceiptsList interface { - *ReceiptList68 | *ReceiptList69 - setBuffers(*receiptListBuffers) - EncodeForStorage() rlp.RawValue - types.DerivableList -} - // ReceiptsPacket is the network packet for block receipts distribution with // request ID wrapping. -type ReceiptsPacket[L ReceiptsList] struct { +type ReceiptsPacket struct { RequestId uint64 - List []L + List rlp.RawList[*ReceiptList] } // ReceiptsRLPResponse is used for receipts, when we already have it encoded type ReceiptsRLPResponse []rlp.RawValue -// ReceiptsRLPPacket is ReceiptsRLPResponse with request ID wrapping. -type ReceiptsRLPPacket struct { - RequestId uint64 - ReceiptsRLPResponse -} - // NewPooledTransactionHashesPacket represents a transaction announcement packet on eth/68 and newer. type NewPooledTransactionHashesPacket struct { Types []byte @@ -314,7 +253,7 @@ type PooledTransactionsResponse []*types.Transaction // with request ID wrapping. type PooledTransactionsPacket struct { RequestId uint64 - PooledTransactionsResponse + List rlp.RawList[*types.Transaction] } // PooledTransactionsRLPResponse is the network packet for transaction distribution, used @@ -334,14 +273,8 @@ type BlockRangeUpdatePacket struct { LatestBlockHash common.Hash } -func (*StatusPacket68) Name() string { return "Status" } -func (*StatusPacket68) Kind() byte { return StatusMsg } - -func (*StatusPacket69) Name() string { return "Status" } -func (*StatusPacket69) Kind() byte { return StatusMsg } - -func (*NewBlockHashesPacket) Name() string { return "NewBlockHashes" } -func (*NewBlockHashesPacket) Kind() byte { return NewBlockHashesMsg } +func (*StatusPacket) Name() string { return "Status" } +func (*StatusPacket) Kind() byte { return StatusMsg } func (*TransactionsPacket) Name() string { return "Transactions" } func (*TransactionsPacket) Kind() byte { return TransactionsMsg } @@ -358,17 +291,14 @@ func (*GetBlockBodiesRequest) Kind() byte { return GetBlockBodiesMsg } func (*BlockBodiesResponse) Name() string { return "BlockBodies" } func (*BlockBodiesResponse) Kind() byte { return BlockBodiesMsg } -func (*NewBlockPacket) Name() string { return "NewBlock" } -func (*NewBlockPacket) Kind() byte { return NewBlockMsg } - func (*NewPooledTransactionHashesPacket) Name() string { return "NewPooledTransactionHashes" } func (*NewPooledTransactionHashesPacket) Kind() byte { return NewPooledTransactionHashesMsg } func (*GetPooledTransactionsRequest) Name() string { return "GetPooledTransactions" } func (*GetPooledTransactionsRequest) Kind() byte { return GetPooledTransactionsMsg } -func (*PooledTransactionsResponse) Name() string { return "PooledTransactions" } -func (*PooledTransactionsResponse) Kind() byte { return PooledTransactionsMsg } +func (*PooledTransactionsPacket) Name() string { return "PooledTransactions" } +func (*PooledTransactionsPacket) Kind() byte { return PooledTransactionsMsg } func (*GetReceiptsRequest) Name() string { return "GetReceipts" } func (*GetReceiptsRequest) Kind() byte { return GetReceiptsMsg } diff --git a/eth/protocols/eth/protocol_test.go b/eth/protocols/eth/protocol_test.go index b402cba549..f35d9f7d9c 100644 --- a/eth/protocols/eth/protocol_test.go +++ b/eth/protocols/eth/protocol_test.go @@ -78,34 +78,33 @@ func TestEmptyMessages(t *testing.T) { for i, msg := range []any{ // Headers GetBlockHeadersPacket{1111, nil}, - BlockHeadersPacket{1111, nil}, // Bodies GetBlockBodiesPacket{1111, nil}, - BlockBodiesPacket{1111, nil}, BlockBodiesRLPPacket{1111, nil}, // Receipts GetReceiptsPacket{1111, nil}, // Transactions GetPooledTransactionsPacket{1111, nil}, - PooledTransactionsPacket{1111, nil}, PooledTransactionsRLPPacket{1111, nil}, // Headers - BlockHeadersPacket{1111, BlockHeadersRequest([]*types.Header{})}, + BlockHeadersPacket{1111, encodeRL([]*types.Header{})}, // Bodies GetBlockBodiesPacket{1111, GetBlockBodiesRequest([]common.Hash{})}, - BlockBodiesPacket{1111, BlockBodiesResponse([]*BlockBody{})}, + BlockBodiesPacket{1111, encodeRL([]BlockBody{})}, BlockBodiesRLPPacket{1111, BlockBodiesRLPResponse([]rlp.RawValue{})}, // Receipts GetReceiptsPacket{1111, GetReceiptsRequest([]common.Hash{})}, - ReceiptsPacket[*ReceiptList68]{1111, []*ReceiptList68{}}, - ReceiptsPacket[*ReceiptList69]{1111, []*ReceiptList69{}}, + ReceiptsPacket{1111, encodeRL([]*ReceiptList{})}, // Transactions GetPooledTransactionsPacket{1111, GetPooledTransactionsRequest([]common.Hash{})}, - PooledTransactionsPacket{1111, PooledTransactionsResponse([]*types.Transaction{})}, + PooledTransactionsPacket{1111, encodeRL([]*types.Transaction{})}, PooledTransactionsRLPPacket{1111, PooledTransactionsRLPResponse([]rlp.RawValue{})}, } { - if have, _ := rlp.EncodeToBytes(msg); !bytes.Equal(have, want) { + have, err := rlp.EncodeToBytes(msg) + if err != nil { + t.Errorf("test %d, type %T, error: %v", i, msg, err) + } else if !bytes.Equal(have, want) { t.Errorf("test %d, type %T, have\n\t%x\nwant\n\t%x", i, msg, have, want) } } @@ -116,13 +115,12 @@ func TestMessages(t *testing.T) { // Some basic structs used during testing var ( header *types.Header - blockBody *BlockBody + blockBody BlockBody blockBodyRlp rlp.RawValue txs []*types.Transaction txRlps []rlp.RawValue hashes []common.Hash receipts []*types.Receipt - receiptsRlp rlp.RawValue err error ) @@ -150,9 +148,9 @@ func TestMessages(t *testing.T) { } } // init the block body data, both object and rlp form - blockBody = &BlockBody{ - Transactions: txs, - Uncles: []*types.Header{header}, + blockBody = BlockBody{ + Transactions: encodeRL(txs), + Uncles: encodeRL([]*types.Header{header}), } blockBodyRlp, err = rlp.EncodeToBytes(blockBody) if err != nil { @@ -191,11 +189,6 @@ func TestMessages(t *testing.T) { } miniDeriveFields(receipts[0], 0) miniDeriveFields(receipts[1], 1) - rlpData, err := rlp.EncodeToBytes(receipts) - if err != nil { - t.Fatal(err) - } - receiptsRlp = rlpData } for i, tc := range []struct { @@ -211,7 +204,7 @@ func TestMessages(t *testing.T) { common.FromHex("ca820457c682270f050580"), }, { - BlockHeadersPacket{1111, BlockHeadersRequest{header}}, + BlockHeadersPacket{1111, encodeRL([]*types.Header{header})}, common.FromHex("f90202820457f901fcf901f9a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000940000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008208ae820d0582115c8215b3821a0a827788a00000000000000000000000000000000000000000000000000000000000000000880000000000000000"), }, { @@ -219,7 +212,7 @@ func TestMessages(t *testing.T) { common.FromHex("f847820457f842a000000000000000000000000000000000000000000000000000000000deadc0dea000000000000000000000000000000000000000000000000000000000feedbeef"), }, { - BlockBodiesPacket{1111, BlockBodiesResponse([]*BlockBody{blockBody})}, + BlockBodiesPacket{1111, encodeRL([]BlockBody{blockBody})}, common.FromHex("f902dc820457f902d6f902d3f8d2f867088504a817c8088302e2489435353535353535353535353535353535353535358202008025a064b1702d9298fee62dfeccc57d322a463ad55ca201256d01f62b45b2e1c21c12a064b1702d9298fee62dfeccc57d322a463ad55ca201256d01f62b45b2e1c21c10f867098504a817c809830334509435353535353535353535353535353535353535358202d98025a052f8f61201b2b11a78d6e866abc9c3db2ae8631fa656bfe5cb53668255367afba052f8f61201b2b11a78d6e866abc9c3db2ae8631fa656bfe5cb53668255367afbf901fcf901f9a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000940000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008208ae820d0582115c8215b3821a0a827788a00000000000000000000000000000000000000000000000000000000000000000880000000000000000"), }, { // Identical to non-rlp-shortcut version @@ -231,16 +224,7 @@ func TestMessages(t *testing.T) { common.FromHex("f847820457f842a000000000000000000000000000000000000000000000000000000000deadc0dea000000000000000000000000000000000000000000000000000000000feedbeef"), }, { - ReceiptsPacket[*ReceiptList68]{1111, []*ReceiptList68{NewReceiptList68(receipts)}}, - common.FromHex("f902e8820457f902e2f902dff901698082014d80b9010000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000014000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000004000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000f85ff85d940000000000000000000000000000000000000011f842a0000000000000000000000000000000000000000000000000000000000000deada0000000000000000000000000000000000000000000000000000000000000beef830100ffb9017001f9016c018201bc80b9010000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000001000000000000000000000000000000000000000000000000040000000000000000000000000004000000000000000000000000000000000000000000000000000000008000400000000000000000000000000000000000000000000000000000000000000000000000000000040f862f860940000000000000000000000000000000000000022f842a00000000000000000000000000000000000000000000000000000000000005668a0000000000000000000000000000000000000000000000000000000000000977386020f0f0f0608"), - }, - { - // Identical to the eth/68 encoding above. - ReceiptsRLPPacket{1111, ReceiptsRLPResponse([]rlp.RawValue{receiptsRlp})}, - common.FromHex("f902e6820457f902e0f902ddf901688082014db9010000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000014000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000004000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000f85ff85d940000000000000000000000000000000000000011f842a0000000000000000000000000000000000000000000000000000000000000deada0000000000000000000000000000000000000000000000000000000000000beef830100ffb9016f01f9016b018201bcb9010000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000001000000000000000000000000000000000000000000000000040000000000000000000000000004000000000000000000000000000000000000000000000000000000008000400000000000000000000000000000000000000000000000000000000000000000000000000000040f862f860940000000000000000000000000000000000000022f842a00000000000000000000000000000000000000000000000000000000000005668a0000000000000000000000000000000000000000000000000000000000000977386020f0f0f0608"), - }, - { - ReceiptsPacket[*ReceiptList69]{1111, []*ReceiptList69{NewReceiptList69(receipts)}}, + ReceiptsPacket{1111, encodeRL([]*ReceiptList{NewReceiptList(receipts)})}, common.FromHex("f8dc820457f8d7f8d5f867808082014d80f85ff85d940000000000000000000000000000000000000011f842a0000000000000000000000000000000000000000000000000000000000000deada0000000000000000000000000000000000000000000000000000000000000beef830100fff86a01018201bc80f862f860940000000000000000000000000000000000000022f842a00000000000000000000000000000000000000000000000000000000000005668a0000000000000000000000000000000000000000000000000000000000000977386020f0f0f0608"), }, { @@ -248,7 +232,7 @@ func TestMessages(t *testing.T) { common.FromHex("f847820457f842a000000000000000000000000000000000000000000000000000000000deadc0dea000000000000000000000000000000000000000000000000000000000feedbeef"), }, { - PooledTransactionsPacket{1111, PooledTransactionsResponse(txs)}, + PooledTransactionsPacket{1111, encodeRL(txs)}, common.FromHex("f8d7820457f8d2f867088504a817c8088302e2489435353535353535353535353535353535353535358202008025a064b1702d9298fee62dfeccc57d322a463ad55ca201256d01f62b45b2e1c21c12a064b1702d9298fee62dfeccc57d322a463ad55ca201256d01f62b45b2e1c21c10f867098504a817c809830334509435353535353535353535353535353535353535358202d98025a052f8f61201b2b11a78d6e866abc9c3db2ae8631fa656bfe5cb53668255367afba052f8f61201b2b11a78d6e866abc9c3db2ae8631fa656bfe5cb53668255367afb"), }, { diff --git a/eth/protocols/eth/receipt.go b/eth/protocols/eth/receipt.go index b59d7ff4e1..e248d3fc52 100644 --- a/eth/protocols/eth/receipt.go +++ b/eth/protocols/eth/receipt.go @@ -27,9 +27,6 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) -// This is just a sanity limit for the size of a single receipt. -const maxReceiptSize = 16 * 1024 * 1024 - // Receipt is the representation of receipts for networking purposes. type Receipt struct { TxType byte @@ -50,142 +47,18 @@ func newReceipt(tr *types.Receipt) Receipt { return r } -// decode68 parses a receipt in the eth/68 network encoding. -func (r *Receipt) decode68(buf *receiptListBuffers, s *rlp.Stream) error { - k, size, err := s.Kind() - if err != nil { - return err - } - - *r = Receipt{} - if k == rlp.List { - // Legacy receipt. - return r.decodeInnerList(s, false, true) - } - // Typed receipt. - if size < 2 || size > maxReceiptSize { - return fmt.Errorf("invalid receipt size %d", size) - } - buf.tmp.Reset() - buf.tmp.Grow(int(size)) - payload := buf.tmp.Bytes()[:int(size)] - if err := s.ReadBytes(payload); err != nil { - return err - } - r.TxType = payload[0] - s2 := rlp.NewStream(bytes.NewReader(payload[1:]), 0) - return r.decodeInnerList(s2, false, true) -} - -// decode69 parses a receipt in the eth/69 network encoding. -func (r *Receipt) decode69(s *rlp.Stream) error { - *r = Receipt{} - return r.decodeInnerList(s, true, false) -} - -// decodeDatabase parses a receipt in the basic database encoding. -func (r *Receipt) decodeDatabase(txType byte, s *rlp.Stream) error { - *r = Receipt{TxType: txType} - return r.decodeInnerList(s, false, false) -} - -func (r *Receipt) decodeInnerList(s *rlp.Stream, readTxType, readBloom bool) error { - _, err := s.List() - if err != nil { - return err - } - if readTxType { - r.TxType, err = s.Uint8() - if err != nil { - return fmt.Errorf("invalid txType: %w", err) - } - } - r.PostStateOrStatus, err = s.Bytes() - if err != nil { - return fmt.Errorf("invalid postStateOrStatus: %w", err) - } - r.GasUsed, err = s.Uint64() - if err != nil { - return fmt.Errorf("invalid gasUsed: %w", err) - } - r.GasUsedForL1, err = s.Uint64() - if err != nil { - return fmt.Errorf("invalid gasUsedForL1: %w", err) - } - if readBloom { - var b types.Bloom - if err := s.ReadBytes(b[:]); err != nil { - return fmt.Errorf("invalid bloom: %v", err) - } - } - r.Logs, err = s.Raw() - if err != nil { - return fmt.Errorf("invalid logs: %w", err) - } - return s.ListEnd() -} - -// encodeForStorage produces the the storage encoding, i.e. the result matches -// the RLP encoding of types.ReceiptForStorage. -func (r *Receipt) encodeForStorage(w *rlp.EncoderBuffer) { - list := w.List() - w.WriteBytes(r.PostStateOrStatus) - w.WriteUint64(r.GasUsed) - w.WriteUint64(r.GasUsedForL1) - w.Write(r.Logs) - w.ListEnd(list) -} - -// encodeForNetwork68 produces the eth/68 network protocol encoding of a receipt. -// Note this recomputes the bloom filter of the receipt. -func (r *Receipt) encodeForNetwork68(buf *receiptListBuffers, w *rlp.EncoderBuffer) { - writeInner := func(w *rlp.EncoderBuffer) { - list := w.List() - w.WriteBytes(r.PostStateOrStatus) - w.WriteUint64(r.GasUsed) - w.WriteUint64(r.GasUsedForL1) - bloom := r.bloom(&buf.bloom) - w.WriteBytes(bloom[:]) - w.Write(r.Logs) - w.ListEnd(list) - } - - if r.TxType == 0 { - writeInner(w) - } else { - buf.tmp.Reset() - buf.tmp.WriteByte(r.TxType) - buf.enc.Reset(&buf.tmp) - writeInner(&buf.enc) - buf.enc.Flush() - w.WriteBytes(buf.tmp.Bytes()) - } -} - -// encodeForNetwork69 produces the eth/69 network protocol encoding of a receipt. -func (r *Receipt) encodeForNetwork69(w *rlp.EncoderBuffer) { - list := w.List() - w.WriteUint64(uint64(r.TxType)) - w.WriteBytes(r.PostStateOrStatus) - w.WriteUint64(r.GasUsed) - w.WriteUint64(r.GasUsedForL1) - w.Write(r.Logs) - w.ListEnd(list) -} - // encodeForHash encodes a receipt for the block receiptsRoot derivation. -func (r *Receipt) encodeForHash(buf *receiptListBuffers, out *bytes.Buffer) { +func (r *Receipt) encodeForHash(bloomBuf *[6]byte, out *bytes.Buffer) { // For typed receipts, add the tx type. if r.TxType != 0 { out.WriteByte(r.TxType) } // Encode list = [postStateOrStatus, gasUsed, bloom, logs]. - w := &buf.enc - w.Reset(out) + w := rlp.NewEncoderBuffer(out) l := w.List() w.WriteBytes(r.PostStateOrStatus) w.WriteUint64(r.GasUsed) - bloom := r.bloom(&buf.bloom) + bloom := r.bloom(bloomBuf) w.WriteBytes(bloom[:]) w.Write(r.Logs) w.ListEnd(l) @@ -218,203 +91,125 @@ func (r *Receipt) bloom(buffer *[6]byte) types.Bloom { return b } -type receiptListBuffers struct { - enc rlp.EncoderBuffer - bloom [6]byte - tmp bytes.Buffer -} - -func initBuffers(buf **receiptListBuffers) { - if *buf == nil { - *buf = new(receiptListBuffers) +// decode assigns the fields of r by decoding the network format. +func (r *Receipt) decode(input []byte) error { + input, _, err := rlp.SplitList(input) + if err != nil { + return fmt.Errorf("inner list: %v", err) } -} -// encodeForStorage encodes a list of receipts for the database. -func (buf *receiptListBuffers) encodeForStorage(rs []Receipt) rlp.RawValue { - var out bytes.Buffer - w := &buf.enc - w.Reset(&out) - outer := w.List() - for _, receipts := range rs { - receipts.encodeForStorage(w) + // txType + var txType uint64 + txType, input, err = rlp.SplitUint64(input) + if err != nil { + return fmt.Errorf("invalid txType: %w", err) } - w.ListEnd(outer) - w.Flush() - return out.Bytes() -} - -// ReceiptList68 is a block receipt list as downloaded by eth/68. -// This also implements types.DerivableList for validation purposes. -type ReceiptList68 struct { - buf *receiptListBuffers - items []Receipt -} - -// NewReceiptList68 creates a receipt list. -// This is slow, and exists for testing purposes. -func NewReceiptList68(trs []*types.Receipt) *ReceiptList68 { - rl := &ReceiptList68{items: make([]Receipt, len(trs))} - for i, tr := range trs { - rl.items[i] = newReceipt(tr) + if txType > 0x7f { + return fmt.Errorf("invalid txType: too large") } - return rl -} + r.TxType = byte(txType) -func blockReceiptsToNetwork68(blockReceipts, blockBody rlp.RawValue) ([]byte, error) { - txTypesIter, err := txTypesInBody(blockBody) + // status + r.PostStateOrStatus, input, err = rlp.SplitString(input) if err != nil { - return nil, fmt.Errorf("invalid block body: %v", err) + return fmt.Errorf("invalid postStateOrStatus: %w", err) } - nextTxType, stopTxTypes := iter.Pull(txTypesIter) - defer stopTxTypes() - - var ( - out bytes.Buffer - buf receiptListBuffers - ) - blockReceiptIter, _ := rlp.NewListIterator(blockReceipts) - innerReader := bytes.NewReader(nil) - innerStream := rlp.NewStream(innerReader, 0) - w := rlp.NewEncoderBuffer(&out) - outer := w.List() - for i := 0; blockReceiptIter.Next(); i++ { - content := blockReceiptIter.Value() - innerReader.Reset(content) - innerStream.Reset(innerReader, uint64(len(content))) - var r Receipt - txType, _ := nextTxType() - if err := r.decodeDatabase(txType, innerStream); err != nil { - return nil, fmt.Errorf("invalid database receipt %d: %v", i, err) - } - r.encodeForNetwork68(&buf, &w) + if len(r.PostStateOrStatus) > 1 && len(r.PostStateOrStatus) != 32 { + return fmt.Errorf("invalid postStateOrStatus length %d", len(r.PostStateOrStatus)) } - w.ListEnd(outer) - w.Flush() - return out.Bytes(), nil -} - -// setBuffers implements ReceiptsList. -func (rl *ReceiptList68) setBuffers(buf *receiptListBuffers) { - rl.buf = buf -} - -// EncodeForStorage encodes the receipts for storage into the database. -func (rl *ReceiptList68) EncodeForStorage() rlp.RawValue { - initBuffers(&rl.buf) - return rl.buf.encodeForStorage(rl.items) -} -// Len implements types.DerivableList. -func (rl *ReceiptList68) Len() int { - return len(rl.items) -} - -// EncodeIndex implements types.DerivableList. -func (rl *ReceiptList68) EncodeIndex(i int, out *bytes.Buffer) { - initBuffers(&rl.buf) - rl.items[i].encodeForHash(rl.buf, out) -} - -// DecodeRLP decodes a list of receipts from the network format. -func (rl *ReceiptList68) DecodeRLP(s *rlp.Stream) error { - initBuffers(&rl.buf) - if _, err := s.List(); err != nil { - return err + // gas + r.GasUsed, input, err = rlp.SplitUint64(input) + if err != nil { + return fmt.Errorf("invalid gasUsed: %w", err) } - for i := 0; s.MoreDataInList(); i++ { - var item Receipt - err := item.decode68(rl.buf, s) - if err != nil { - return fmt.Errorf("receipt %d: %v", i, err) - } - rl.items = append(rl.items, item) + r.GasUsedForL1, input, err = rlp.SplitUint64(input) + if err != nil { + return fmt.Errorf("invalid gasUsedForL1: %w", err) } - return s.ListEnd() -} -// EncodeRLP encodes the list into the network format of eth/68. -func (rl *ReceiptList68) EncodeRLP(_w io.Writer) error { - initBuffers(&rl.buf) - w := rlp.NewEncoderBuffer(_w) - outer := w.List() - for i := range rl.items { - rl.items[i].encodeForNetwork68(rl.buf, &w) + // logs + _, rest, err := rlp.SplitList(input) + if err != nil { + return fmt.Errorf("invalid logs: %w", err) } - w.ListEnd(outer) - return w.Flush() + if len(rest) != 0 { + return fmt.Errorf("junk at end of receipt") + } + r.Logs = input + return nil } -// ReceiptList69 is the block receipt list as downloaded by eth/69. -// This implements types.DerivableList for validation purposes. -type ReceiptList69 struct { - buf *receiptListBuffers - items []Receipt +// ReceiptList is the block receipt list as downloaded by eth/69. +type ReceiptList struct { + items rlp.RawList[Receipt] } -// NewReceiptList69 creates a receipt list. +// NewReceiptList creates a receipt list. // This is slow, and exists for testing purposes. -func NewReceiptList69(trs []*types.Receipt) *ReceiptList69 { - rl := &ReceiptList69{items: make([]Receipt, len(trs))} - for i, tr := range trs { - rl.items[i] = newReceipt(tr) +func NewReceiptList(trs []*types.Receipt) *ReceiptList { + rl := new(ReceiptList) + for _, tr := range trs { + r := newReceipt(tr) + encoded, _ := rlp.EncodeToBytes(&r) + rl.items.AppendRaw(encoded) } return rl } -// setBuffers implements ReceiptsList. -func (rl *ReceiptList69) setBuffers(buf *receiptListBuffers) { - rl.buf = buf -} - -// EncodeForStorage encodes the receipts for storage into the database. -func (rl *ReceiptList69) EncodeForStorage() rlp.RawValue { - initBuffers(&rl.buf) - return rl.buf.encodeForStorage(rl.items) -} - -// Len implements types.DerivableList. -func (rl *ReceiptList69) Len() int { - return len(rl.items) +// DecodeRLP decodes a list receipts from the network format. +func (rl *ReceiptList) DecodeRLP(s *rlp.Stream) error { + return rl.items.DecodeRLP(s) } -// EncodeIndex implements types.DerivableList. -func (rl *ReceiptList69) EncodeIndex(i int, out *bytes.Buffer) { - initBuffers(&rl.buf) - rl.items[i].encodeForHash(rl.buf, out) +// EncodeRLP encodes the list into the network format of eth/69. +func (rl *ReceiptList) EncodeRLP(w io.Writer) error { + return rl.items.EncodeRLP(w) } -// DecodeRLP decodes a list receipts from the network format. -func (rl *ReceiptList69) DecodeRLP(s *rlp.Stream) error { - if _, err := s.List(); err != nil { - return err - } - for i := 0; s.MoreDataInList(); i++ { - var item Receipt - err := item.decode69(s) +// EncodeForStorage encodes a list of receipts for the database. +// It only strips the first element (TxType) from each receipt's +// raw RLP without the actual decoding and re-encoding. +func (rl *ReceiptList) EncodeForStorage() (rlp.RawValue, error) { + var out bytes.Buffer + w := rlp.NewEncoderBuffer(&out) + outer := w.List() + it := rl.items.ContentIterator() + for it.Next() { + content, _, err := rlp.SplitList(it.Value()) + if err != nil { + return nil, fmt.Errorf("bad receipt: %v", err) + } + _, _, rest, err := rlp.Split(content) if err != nil { - return fmt.Errorf("receipt %d: %v", i, err) + return nil, fmt.Errorf("bad receipt: %v", err) } - rl.items = append(rl.items, item) + inner := w.List() + w.Write(rest) + w.ListEnd(inner) } - return s.ListEnd() -} - -// EncodeRLP encodes the list into the network format of eth/69. -func (rl *ReceiptList69) EncodeRLP(_w io.Writer) error { - w := rlp.NewEncoderBuffer(_w) - outer := w.List() - for i := range rl.items { - rl.items[i].encodeForNetwork69(&w) + if it.Err() != nil { + return nil, fmt.Errorf("bad list: %v", it.Err()) } w.ListEnd(outer) - return w.Flush() + w.Flush() + return out.Bytes(), nil +} + +// Derivable returns a DerivableList, which can be used to decode +func (rl *ReceiptList) Derivable() types.DerivableList { + var bloomBuf [6]byte + return newDerivableRawList(&rl.items, func(data []byte, outbuf *bytes.Buffer) { + var r Receipt + if r.decode(data) == nil { + r.encodeForHash(&bloomBuf, outbuf) + } + }) } -// blockReceiptsToNetwork69 takes a slice of rlp-encoded receipts, and transactions, -// and applies the type-encoding on the receipts (for non-legacy receipts). -// e.g. for non-legacy receipts: receipt-data -> {tx-type || receipt-data} -func blockReceiptsToNetwork69(blockReceipts, blockBody rlp.RawValue) ([]byte, error) { +// blockReceiptsToNetwork takes a slice of rlp-encoded receipts, and transactions, +// and re-encodes them for the network protocol. +func blockReceiptsToNetwork(blockReceipts, blockBody rlp.RawValue) ([]byte, error) { txTypesIter, err := txTypesInBody(blockBody) if err != nil { return nil, fmt.Errorf("invalid block body: %v", err) diff --git a/eth/protocols/eth/receipt_test.go b/eth/protocols/eth/receipt_test.go index ea0754ec05..4cf81789d5 100644 --- a/eth/protocols/eth/receipt_test.go +++ b/eth/protocols/eth/receipt_test.go @@ -63,6 +63,18 @@ var receiptsTests = []struct { input: []types.ReceiptForStorage{{CumulativeGasUsed: 555, GasUsedForL1: 1, Status: 1, Logs: receiptsTestLogs2}}, txs: []*types.Transaction{types.NewTx(&types.AccessListTx{})}, }, + { + input: []types.ReceiptForStorage{ + {CumulativeGasUsed: 111, PostState: common.HexToHash("0x1111").Bytes(), Logs: receiptsTestLogs1}, + {CumulativeGasUsed: 222, Status: 0, Logs: receiptsTestLogs2}, + {CumulativeGasUsed: 333, Status: 1, Logs: nil}, + }, + txs: []*types.Transaction{ + types.NewTx(&types.LegacyTx{}), + types.NewTx(&types.AccessListTx{}), + types.NewTx(&types.DynamicFeeTx{}), + }, + }, } func init() { @@ -83,7 +95,7 @@ func init() { } } -func TestReceiptList69(t *testing.T) { +func TestReceiptList(t *testing.T) { for i, test := range receiptsTests { // encode receipts from types.ReceiptForStorage object. canonDB, _ := rlp.EncodeToBytes(test.input) @@ -93,54 +105,20 @@ func TestReceiptList69(t *testing.T) { canonBody, _ := rlp.EncodeToBytes(blockBody) // convert from storage encoding to network encoding - network, err := blockReceiptsToNetwork69(canonDB, canonBody) + network, err := blockReceiptsToNetwork(canonDB, canonBody) if err != nil { - t.Fatalf("test[%d]: blockReceiptsToNetwork69 error: %v", i, err) + t.Fatalf("test[%d]: blockReceiptsToNetwork error: %v", i, err) } // parse as Receipts response list from network encoding - var rl ReceiptList69 + var rl ReceiptList if err := rlp.DecodeBytes(network, &rl); err != nil { t.Fatalf("test[%d]: can't decode network receipts: %v", i, err) } - rlStorageEnc := rl.EncodeForStorage() - if !bytes.Equal(rlStorageEnc, canonDB) { - t.Fatalf("test[%d]: re-encoded receipts not equal\nhave: %x\nwant: %x", i, rlStorageEnc, canonDB) - } - rlNetworkEnc, _ := rlp.EncodeToBytes(&rl) - if !bytes.Equal(rlNetworkEnc, network) { - t.Fatalf("test[%d]: re-encoded network receipt list not equal\nhave: %x\nwant: %x", i, rlNetworkEnc, network) - } - - // compute root hash from ReceiptList69 and compare. - responseHash := types.DeriveSha(&rl, trie.NewStackTrie(nil)) - if responseHash != test.root { - t.Fatalf("test[%d]: wrong root hash from ReceiptList69\nhave: %v\nwant: %v", i, responseHash, test.root) - } - } -} - -func TestReceiptList68(t *testing.T) { - for i, test := range receiptsTests { - // encode receipts from types.ReceiptForStorage object. - canonDB, _ := rlp.EncodeToBytes(test.input) - - // encode block body from types object. - blockBody := types.Body{Transactions: test.txs} - canonBody, _ := rlp.EncodeToBytes(blockBody) - - // convert from storage encoding to network encoding - network, err := blockReceiptsToNetwork68(canonDB, canonBody) + rlStorageEnc, err := rl.EncodeForStorage() if err != nil { - t.Fatalf("test[%d]: blockReceiptsToNetwork68 error: %v", i, err) - } - - // parse as Receipts response list from network encoding - var rl ReceiptList68 - if err := rlp.DecodeBytes(network, &rl); err != nil { - t.Fatalf("test[%d]: can't decode network receipts: %v", i, err) + t.Fatalf("test[%d]: error from EncodeForStorage: %v", i, err) } - rlStorageEnc := rl.EncodeForStorage() if !bytes.Equal(rlStorageEnc, canonDB) { t.Fatalf("test[%d]: re-encoded receipts not equal\nhave: %x\nwant: %x", i, rlStorageEnc, canonDB) } @@ -149,10 +127,10 @@ func TestReceiptList68(t *testing.T) { t.Fatalf("test[%d]: re-encoded network receipt list not equal\nhave: %x\nwant: %x", i, rlNetworkEnc, network) } - // compute root hash from ReceiptList68 and compare. - responseHash := types.DeriveSha(&rl, trie.NewStackTrie(nil)) + // compute root hash from ReceiptList and compare. + responseHash := types.DeriveSha(rl.Derivable(), trie.NewStackTrie(nil)) if responseHash != test.root { - t.Fatalf("test[%d]: wrong root hash from ReceiptList68\nhave: %v\nwant: %v", i, responseHash, test.root) + t.Fatalf("test[%d]: wrong root hash from ReceiptList\nhave: %v\nwant: %v", i, responseHash, test.root) } } } diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index d69b6951be..f6cbfcf43b 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -31,6 +31,8 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" + "github.com/ethereum/go-ethereum/p2p/tracker" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/triedb/database" @@ -94,6 +96,7 @@ func MakeProtocols(backend Backend) []p2p.Protocol { Length: protocolLengths[version], Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error { return backend.RunPeer(NewPeer(version, p, rw), func(peer *Peer) error { + defer peer.Close() return Handle(backend, peer) }) }, @@ -147,7 +150,6 @@ func HandleMessage(backend Backend, peer *Peer) error { // Handle the message depending on its contents switch { case msg.Code == GetAccountRangeMsg: - // Decode the account retrieval request var req GetAccountRangePacket if err := msg.Decode(&req); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) @@ -163,23 +165,40 @@ func HandleMessage(backend Backend, peer *Peer) error { }) case msg.Code == AccountRangeMsg: - // A range of accounts arrived to one of our previous requests - res := new(AccountRangePacket) + res := new(accountRangeInput) if err := msg.Decode(res); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) } + + // Check response validity. + if len := res.Proof.Len(); len > 128 { + return fmt.Errorf("AccountRange: invalid proof (length %d)", len) + } + tresp := tracker.Response{ID: res.ID, MsgCode: AccountRangeMsg, Size: len(res.Accounts.Content())} + if err := peer.tracker.Fulfil(tresp); err != nil { + return err + } + + // Decode. + accounts, err := res.Accounts.Items() + if err != nil { + return fmt.Errorf("AccountRange: invalid accounts list: %v", err) + } + proof, err := res.Proof.Items() + if err != nil { + return fmt.Errorf("AccountRange: invalid proof: %v", err) + } + // Ensure the range is monotonically increasing - for i := 1; i < len(res.Accounts); i++ { - if bytes.Compare(res.Accounts[i-1].Hash[:], res.Accounts[i].Hash[:]) >= 0 { - return fmt.Errorf("accounts not monotonically increasing: #%d [%x] vs #%d [%x]", i-1, res.Accounts[i-1].Hash[:], i, res.Accounts[i].Hash[:]) + for i := 1; i < len(accounts); i++ { + if bytes.Compare(accounts[i-1].Hash[:], accounts[i].Hash[:]) >= 0 { + return fmt.Errorf("accounts not monotonically increasing: #%d [%x] vs #%d [%x]", i-1, accounts[i-1].Hash[:], i, accounts[i].Hash[:]) } } - requestTracker.Fulfil(peer.id, peer.version, AccountRangeMsg, res.ID) - return backend.Handle(peer, res) + return backend.Handle(peer, &AccountRangePacket{res.ID, accounts, proof}) case msg.Code == GetStorageRangesMsg: - // Decode the storage retrieval request var req GetStorageRangesPacket if err := msg.Decode(&req); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) @@ -195,25 +214,42 @@ func HandleMessage(backend Backend, peer *Peer) error { }) case msg.Code == StorageRangesMsg: - // A range of storage slots arrived to one of our previous requests - res := new(StorageRangesPacket) + res := new(storageRangesInput) if err := msg.Decode(res); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) } + + // Check response validity. + if len := res.Proof.Len(); len > 128 { + return fmt.Errorf("StorageRangesMsg: invalid proof (length %d)", len) + } + tresp := tracker.Response{ID: res.ID, MsgCode: StorageRangesMsg, Size: len(res.Slots.Content())} + if err := peer.tracker.Fulfil(tresp); err != nil { + return fmt.Errorf("StorageRangesMsg: %w", err) + } + + // Decode. + slotLists, err := res.Slots.Items() + if err != nil { + return fmt.Errorf("AccountRange: invalid accounts list: %v", err) + } + proof, err := res.Proof.Items() + if err != nil { + return fmt.Errorf("AccountRange: invalid proof: %v", err) + } + // Ensure the ranges are monotonically increasing - for i, slots := range res.Slots { + for i, slots := range slotLists { for j := 1; j < len(slots); j++ { if bytes.Compare(slots[j-1].Hash[:], slots[j].Hash[:]) >= 0 { return fmt.Errorf("storage slots not monotonically increasing for account #%d: #%d [%x] vs #%d [%x]", i, j-1, slots[j-1].Hash[:], j, slots[j].Hash[:]) } } } - requestTracker.Fulfil(peer.id, peer.version, StorageRangesMsg, res.ID) - return backend.Handle(peer, res) + return backend.Handle(peer, &StorageRangesPacket{res.ID, slotLists, proof}) case msg.Code == GetByteCodesMsg: - // Decode bytecode retrieval request var req GetByteCodesPacket if err := msg.Decode(&req); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) @@ -228,17 +264,25 @@ func HandleMessage(backend Backend, peer *Peer) error { }) case msg.Code == ByteCodesMsg: - // A batch of byte codes arrived to one of our previous requests - res := new(ByteCodesPacket) + res := new(byteCodesInput) if err := msg.Decode(res); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) } - requestTracker.Fulfil(peer.id, peer.version, ByteCodesMsg, res.ID) - return backend.Handle(peer, res) + length := res.Codes.Len() + tresp := tracker.Response{ID: res.ID, MsgCode: ByteCodesMsg, Size: length} + if err := peer.tracker.Fulfil(tresp); err != nil { + return fmt.Errorf("ByteCodes: %w", err) + } + + codes, err := res.Codes.Items() + if err != nil { + return fmt.Errorf("ByteCodes: %w", err) + } + + return backend.Handle(peer, &ByteCodesPacket{res.ID, codes}) case msg.Code == GetTrieNodesMsg: - // Decode trie node retrieval request var req GetTrieNodesPacket if err := msg.Decode(&req); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) @@ -255,14 +299,21 @@ func HandleMessage(backend Backend, peer *Peer) error { }) case msg.Code == TrieNodesMsg: - // A batch of trie nodes arrived to one of our previous requests - res := new(TrieNodesPacket) + res := new(trieNodesInput) if err := msg.Decode(res); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) } - requestTracker.Fulfil(peer.id, peer.version, TrieNodesMsg, res.ID) - return backend.Handle(peer, res) + tresp := tracker.Response{ID: res.ID, MsgCode: TrieNodesMsg, Size: res.Nodes.Len()} + if err := peer.tracker.Fulfil(tresp); err != nil { + return fmt.Errorf("TrieNodes: %w", err) + } + nodes, err := res.Nodes.Items() + if err != nil { + return fmt.Errorf("TrieNodes: %w", err) + } + + return backend.Handle(peer, &TrieNodesPacket{res.ID, nodes}) default: return fmt.Errorf("%w: %v", errInvalidMsgCode, msg.Code) @@ -510,21 +561,32 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s if reader == nil { reader, _ = triedb.StateReader(req.Root) } + // Retrieve trie nodes until the packet size limit is reached var ( - nodes [][]byte - bytes uint64 - loads int // Trie hash expansions to count database reads + outerIt = req.Paths.ContentIterator() + nodes [][]byte + bytes uint64 + loads int // Trie hash expansions to count database reads ) - for _, pathset := range req.Paths { - switch len(pathset) { + for outerIt.Next() { + innerIt, err := rlp.NewListIterator(outerIt.Value()) + if err != nil { + return nodes, err + } + + switch innerIt.Count() { case 0: // Ensure we penalize invalid requests return nil, fmt.Errorf("%w: zero-item pathset requested", errBadRequest) case 1: // If we're only retrieving an account trie node, fetch it directly - blob, resolved, err := accTrie.GetNode(pathset[0]) + accKey := nextBytes(&innerIt) + if accKey == nil { + return nodes, fmt.Errorf("%w: invalid account node request", errBadRequest) + } + blob, resolved, err := accTrie.GetNode(accKey) loads += resolved // always account database reads, even for failures if err != nil { break @@ -533,33 +595,41 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s bytes += uint64(len(blob)) default: - var stRoot common.Hash - // Storage slots requested, open the storage trie and retrieve from there + accKey := nextBytes(&innerIt) + if accKey == nil { + return nodes, fmt.Errorf("%w: invalid account storage request", errBadRequest) + } + var stRoot common.Hash if reader == nil { // We don't have the requested state snapshotted yet (or it is stale), // but can look up the account via the trie instead. - account, err := accTrie.GetAccountByHash(common.BytesToHash(pathset[0])) + account, err := accTrie.GetAccountByHash(common.BytesToHash(accKey)) loads += 8 // We don't know the exact cost of lookup, this is an estimate if err != nil || account == nil { break } stRoot = account.Root } else { - account, err := reader.Account(common.BytesToHash(pathset[0])) + account, err := reader.Account(common.BytesToHash(accKey)) loads++ // always account database reads, even for failures if err != nil || account == nil { break } stRoot = common.BytesToHash(account.Root) } - id := trie.StorageTrieID(req.Root, common.BytesToHash(pathset[0]), stRoot) + + id := trie.StorageTrieID(req.Root, common.BytesToHash(accKey), stRoot) stTrie, err := trie.NewStateTrie(id, triedb) loads++ // always account database reads, even for failures if err != nil { break } - for _, path := range pathset[1:] { + for innerIt.Next() { + path, _, err := rlp.SplitString(innerIt.Value()) + if err != nil { + return nil, fmt.Errorf("%w: invalid storage key: %v", errBadRequest, err) + } blob, resolved, err := stTrie.GetNode(path) loads += resolved // always account database reads, even for failures if err != nil { @@ -582,6 +652,17 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s return nodes, nil } +func nextBytes(it *rlp.Iterator) []byte { + if !it.Next() { + return nil + } + content, _, err := rlp.SplitString(it.Value()) + if err != nil { + return nil + } + return content +} + // NodeInfo represents a short summary of the `snap` sub-protocol metadata // known about the host peer. type NodeInfo struct{} diff --git a/eth/protocols/snap/peer.go b/eth/protocols/snap/peer.go index c57931678c..0b96de4158 100644 --- a/eth/protocols/snap/peer.go +++ b/eth/protocols/snap/peer.go @@ -17,9 +17,13 @@ package snap import ( + "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/tracker" + "github.com/ethereum/go-ethereum/rlp" ) // Peer is a collection of relevant information we have about a `snap` peer. @@ -29,6 +33,7 @@ type Peer struct { *p2p.Peer // The embedded P2P package peer rw p2p.MsgReadWriter // Input/output streams for snap version uint // Protocol version negotiated + tracker *tracker.Tracker logger log.Logger // Contextual logger with the peer id injected } @@ -36,22 +41,26 @@ type Peer struct { // NewPeer creates a wrapper for a network connection and negotiated protocol // version. func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter) *Peer { + cap := p2p.Cap{Name: ProtocolName, Version: version} id := p.ID().String() return &Peer{ id: id, Peer: p, rw: rw, version: version, + tracker: tracker.New(cap, id, 1*time.Minute), logger: log.New("peer", id[:8]), } } // NewFakePeer creates a fake snap peer without a backing p2p peer, for testing purposes. func NewFakePeer(version uint, id string, rw p2p.MsgReadWriter) *Peer { + cap := p2p.Cap{Name: ProtocolName, Version: version} return &Peer{ id: id, rw: rw, version: version, + tracker: tracker.New(cap, id, 1*time.Minute), logger: log.New("peer", id[:8]), } } @@ -71,63 +80,99 @@ func (p *Peer) Log() log.Logger { return p.logger } +// Close releases resources associated with the peer. +func (p *Peer) Close() { + p.tracker.Stop() +} + // RequestAccountRange fetches a batch of accounts rooted in a specific account // trie, starting with the origin. -func (p *Peer) RequestAccountRange(id uint64, root common.Hash, origin, limit common.Hash, bytes uint64) error { +func (p *Peer) RequestAccountRange(id uint64, root common.Hash, origin, limit common.Hash, bytes int) error { p.logger.Trace("Fetching range of accounts", "reqid", id, "root", root, "origin", origin, "limit", limit, "bytes", common.StorageSize(bytes)) - requestTracker.Track(p.id, p.version, GetAccountRangeMsg, AccountRangeMsg, id) + err := p.tracker.Track(tracker.Request{ + ReqCode: GetAccountRangeMsg, + RespCode: AccountRangeMsg, + ID: id, + Size: 2 * bytes, + }) + if err != nil { + return err + } return p2p.Send(p.rw, GetAccountRangeMsg, &GetAccountRangePacket{ ID: id, Root: root, Origin: origin, Limit: limit, - Bytes: bytes, + Bytes: uint64(bytes), }) } // RequestStorageRanges fetches a batch of storage slots belonging to one or more // accounts. If slots from only one account is requested, an origin marker may also // be used to retrieve from there. -func (p *Peer) RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes uint64) error { +func (p *Peer) RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes int) error { if len(accounts) == 1 && origin != nil { p.logger.Trace("Fetching range of large storage slots", "reqid", id, "root", root, "account", accounts[0], "origin", common.BytesToHash(origin), "limit", common.BytesToHash(limit), "bytes", common.StorageSize(bytes)) } else { p.logger.Trace("Fetching ranges of small storage slots", "reqid", id, "root", root, "accounts", len(accounts), "first", accounts[0], "bytes", common.StorageSize(bytes)) } - requestTracker.Track(p.id, p.version, GetStorageRangesMsg, StorageRangesMsg, id) + + p.tracker.Track(tracker.Request{ + ReqCode: GetStorageRangesMsg, + RespCode: StorageRangesMsg, + ID: id, + Size: 2 * bytes, + }) return p2p.Send(p.rw, GetStorageRangesMsg, &GetStorageRangesPacket{ ID: id, Root: root, Accounts: accounts, Origin: origin, Limit: limit, - Bytes: bytes, + Bytes: uint64(bytes), }) } // RequestByteCodes fetches a batch of bytecodes by hash. -func (p *Peer) RequestByteCodes(id uint64, hashes []common.Hash, bytes uint64) error { +func (p *Peer) RequestByteCodes(id uint64, hashes []common.Hash, bytes int) error { p.logger.Trace("Fetching set of byte codes", "reqid", id, "hashes", len(hashes), "bytes", common.StorageSize(bytes)) - requestTracker.Track(p.id, p.version, GetByteCodesMsg, ByteCodesMsg, id) + err := p.tracker.Track(tracker.Request{ + ReqCode: GetByteCodesMsg, + RespCode: ByteCodesMsg, + ID: id, + Size: len(hashes), // ByteCodes is limited by the length of the hash list. + }) + if err != nil { + return err + } return p2p.Send(p.rw, GetByteCodesMsg, &GetByteCodesPacket{ ID: id, Hashes: hashes, - Bytes: bytes, + Bytes: uint64(bytes), }) } // RequestTrieNodes fetches a batch of account or storage trie nodes rooted in -// a specific state trie. -func (p *Peer) RequestTrieNodes(id uint64, root common.Hash, paths []TrieNodePathSet, bytes uint64) error { +// a specific state trie. The `count` is the total count of paths being requested. +func (p *Peer) RequestTrieNodes(id uint64, root common.Hash, count int, paths []TrieNodePathSet, bytes int) error { p.logger.Trace("Fetching set of trie nodes", "reqid", id, "root", root, "pathsets", len(paths), "bytes", common.StorageSize(bytes)) - requestTracker.Track(p.id, p.version, GetTrieNodesMsg, TrieNodesMsg, id) + err := p.tracker.Track(tracker.Request{ + ReqCode: GetTrieNodesMsg, + RespCode: TrieNodesMsg, + ID: id, + Size: count, // TrieNodes is limited by number of items. + }) + if err != nil { + return err + } + encPaths, _ := rlp.EncodeToRawList(paths) return p2p.Send(p.rw, GetTrieNodesMsg, &GetTrieNodesPacket{ ID: id, Root: root, - Paths: paths, - Bytes: bytes, + Paths: encPaths, + Bytes: uint64(bytes), }) } diff --git a/eth/protocols/snap/protocol.go b/eth/protocols/snap/protocol.go index 0db206b081..25fe25822b 100644 --- a/eth/protocols/snap/protocol.go +++ b/eth/protocols/snap/protocol.go @@ -78,6 +78,12 @@ type GetAccountRangePacket struct { Bytes uint64 // Soft limit at which to stop returning data } +type accountRangeInput struct { + ID uint64 // ID of the request this is a response for + Accounts rlp.RawList[*AccountData] // List of consecutive accounts from the trie + Proof rlp.RawList[[]byte] // List of trie nodes proving the account range +} + // AccountRangePacket represents an account query response. type AccountRangePacket struct { ID uint64 // ID of the request this is a response for @@ -123,6 +129,12 @@ type GetStorageRangesPacket struct { Bytes uint64 // Soft limit at which to stop returning data } +type storageRangesInput struct { + ID uint64 // ID of the request this is a response for + Slots rlp.RawList[[]*StorageData] // Lists of consecutive storage slots for the requested accounts + Proof rlp.RawList[[]byte] // Merkle proofs for the *last* slot range, if it's incomplete +} + // StorageRangesPacket represents a storage slot query response. type StorageRangesPacket struct { ID uint64 // ID of the request this is a response for @@ -161,6 +173,11 @@ type GetByteCodesPacket struct { Bytes uint64 // Soft limit at which to stop returning data } +type byteCodesInput struct { + ID uint64 // ID of the request this is a response for + Codes rlp.RawList[[]byte] // Requested contract bytecodes +} + // ByteCodesPacket represents a contract bytecode query response. type ByteCodesPacket struct { ID uint64 // ID of the request this is a response for @@ -169,10 +186,10 @@ type ByteCodesPacket struct { // GetTrieNodesPacket represents a state trie node query. type GetTrieNodesPacket struct { - ID uint64 // Request ID to match up responses with - Root common.Hash // Root hash of the account trie to serve - Paths []TrieNodePathSet // Trie node hashes to retrieve the nodes for - Bytes uint64 // Soft limit at which to stop returning data + ID uint64 // Request ID to match up responses with + Root common.Hash // Root hash of the account trie to serve + Paths rlp.RawList[TrieNodePathSet] // Trie node hashes to retrieve the nodes for + Bytes uint64 // Soft limit at which to stop returning data } // TrieNodePathSet is a list of trie node paths to retrieve. A naive way to @@ -187,6 +204,11 @@ type GetTrieNodesPacket struct { // that a slot is accessed before the account path is fully expanded. type TrieNodePathSet [][]byte +type trieNodesInput struct { + ID uint64 // ID of the request this is a response for + Nodes rlp.RawList[[]byte] // Requested state trie nodes +} + // TrieNodesPacket represents a state trie node query response. type TrieNodesPacket struct { ID uint64 // ID of the request this is a response for diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index cf4e494645..841bfb446e 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -412,19 +412,19 @@ type SyncPeer interface { // RequestAccountRange fetches a batch of accounts rooted in a specific account // trie, starting with the origin. - RequestAccountRange(id uint64, root, origin, limit common.Hash, bytes uint64) error + RequestAccountRange(id uint64, root, origin, limit common.Hash, bytes int) error // RequestStorageRanges fetches a batch of storage slots belonging to one or // more accounts. If slots from only one account is requested, an origin marker // may also be used to retrieve from there. - RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes uint64) error + RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes int) error // RequestByteCodes fetches a batch of bytecodes by hash. - RequestByteCodes(id uint64, hashes []common.Hash, bytes uint64) error + RequestByteCodes(id uint64, hashes []common.Hash, bytes int) error // RequestTrieNodes fetches a batch of account or storage trie nodes rooted in // a specific state trie. - RequestTrieNodes(id uint64, root common.Hash, paths []TrieNodePathSet, bytes uint64) error + RequestTrieNodes(id uint64, root common.Hash, count int, paths []TrieNodePathSet, bytes int) error // Log retrieves the peer's own contextual logger. Log() log.Logger @@ -1102,7 +1102,7 @@ func (s *Syncer) assignAccountTasks(success chan *accountResponse, fail chan *ac if cap < minRequestSize { // Don't bother with peers below a bare minimum performance cap = minRequestSize } - if err := peer.RequestAccountRange(reqid, root, req.origin, req.limit, uint64(cap)); err != nil { + if err := peer.RequestAccountRange(reqid, root, req.origin, req.limit, cap); err != nil { peer.Log().Debug("Failed to request account range", "err", err) s.scheduleRevertAccountRequest(req) } @@ -1359,7 +1359,7 @@ func (s *Syncer) assignStorageTasks(success chan *storageResponse, fail chan *st if subtask != nil { origin, limit = req.origin[:], req.limit[:] } - if err := peer.RequestStorageRanges(reqid, root, accounts, origin, limit, uint64(cap)); err != nil { + if err := peer.RequestStorageRanges(reqid, root, accounts, origin, limit, cap); err != nil { log.Debug("Failed to request storage", "err", err) s.scheduleRevertStorageRequest(req) } @@ -1492,7 +1492,7 @@ func (s *Syncer) assignTrienodeHealTasks(success chan *trienodeHealResponse, fai defer s.pend.Done() // Attempt to send the remote request and revert if it fails - if err := peer.RequestTrieNodes(reqid, root, pathsets, maxRequestSize); err != nil { + if err := peer.RequestTrieNodes(reqid, root, len(paths), pathsets, maxRequestSize); err != nil { log.Debug("Failed to request trienode healers", "err", err) s.scheduleRevertTrienodeHealRequest(req) } @@ -1699,9 +1699,13 @@ func (s *Syncer) revertAccountRequest(req *accountRequest) { } close(req.stale) - // Remove the request from the tracked set + // Remove the request from the tracked set and restore the peer to the + // idle pool so it can be reassigned work (skip if peer already left). s.lock.Lock() delete(s.accountReqs, req.id) + if _, ok := s.peers[req.peer]; ok { + s.accountIdlers[req.peer] = struct{}{} + } s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the account @@ -1740,9 +1744,13 @@ func (s *Syncer) revertBytecodeRequest(req *bytecodeRequest) { } close(req.stale) - // Remove the request from the tracked set + // Remove the request from the tracked set and restore the peer to the + // idle pool so it can be reassigned work (skip if peer already left). s.lock.Lock() delete(s.bytecodeReqs, req.id) + if _, ok := s.peers[req.peer]; ok { + s.bytecodeIdlers[req.peer] = struct{}{} + } s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the code @@ -1781,9 +1789,13 @@ func (s *Syncer) revertStorageRequest(req *storageRequest) { } close(req.stale) - // Remove the request from the tracked set + // Remove the request from the tracked set and restore the peer to the + // idle pool so it can be reassigned work (skip if peer already left). s.lock.Lock() delete(s.storageReqs, req.id) + if _, ok := s.peers[req.peer]; ok { + s.storageIdlers[req.peer] = struct{}{} + } s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the storage @@ -1826,9 +1838,13 @@ func (s *Syncer) revertTrienodeHealRequest(req *trienodeHealRequest) { } close(req.stale) - // Remove the request from the tracked set + // Remove the request from the tracked set and restore the peer to the + // idle pool so it can be reassigned work (skip if peer already left). s.lock.Lock() delete(s.trienodeHealReqs, req.id) + if _, ok := s.peers[req.peer]; ok { + s.trienodeHealIdlers[req.peer] = struct{}{} + } s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the trie node @@ -1867,9 +1883,13 @@ func (s *Syncer) revertBytecodeHealRequest(req *bytecodeHealRequest) { } close(req.stale) - // Remove the request from the tracked set + // Remove the request from the tracked set and restore the peer to the + // idle pool so it can be reassigned work (skip if peer already left). s.lock.Lock() delete(s.bytecodeHealReqs, req.id) + if _, ok := s.peers[req.peer]; ok { + s.bytecodeHealIdlers[req.peer] = struct{}{} + } s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the code diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index 97f0349154..0e52e609fe 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -119,10 +119,10 @@ func BenchmarkHashing(b *testing.B) { } type ( - accountHandlerFunc func(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap uint64) error - storageHandlerFunc func(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error - trieHandlerFunc func(t *testPeer, requestId uint64, root common.Hash, paths []TrieNodePathSet, cap uint64) error - codeHandlerFunc func(t *testPeer, id uint64, hashes []common.Hash, max uint64) error + accountHandlerFunc func(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap int) error + storageHandlerFunc func(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max int) error + trieHandlerFunc func(t *testPeer, requestId uint64, root common.Hash, paths []TrieNodePathSet, cap int) error + codeHandlerFunc func(t *testPeer, id uint64, hashes []common.Hash, max int) error ) type testPeer struct { @@ -182,21 +182,21 @@ Trienode requests: %d `, t.nAccountRequests, t.nStorageRequests, t.nBytecodeRequests, t.nTrienodeRequests) } -func (t *testPeer) RequestAccountRange(id uint64, root, origin, limit common.Hash, bytes uint64) error { +func (t *testPeer) RequestAccountRange(id uint64, root, origin, limit common.Hash, bytes int) error { t.logger.Trace("Fetching range of accounts", "reqid", id, "root", root, "origin", origin, "limit", limit, "bytes", common.StorageSize(bytes)) t.nAccountRequests++ go t.accountRequestHandler(t, id, root, origin, limit, bytes) return nil } -func (t *testPeer) RequestTrieNodes(id uint64, root common.Hash, paths []TrieNodePathSet, bytes uint64) error { +func (t *testPeer) RequestTrieNodes(id uint64, root common.Hash, count int, paths []TrieNodePathSet, bytes int) error { t.logger.Trace("Fetching set of trie nodes", "reqid", id, "root", root, "pathsets", len(paths), "bytes", common.StorageSize(bytes)) t.nTrienodeRequests++ go t.trieRequestHandler(t, id, root, paths, bytes) return nil } -func (t *testPeer) RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes uint64) error { +func (t *testPeer) RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes int) error { t.nStorageRequests++ if len(accounts) == 1 && origin != nil { t.logger.Trace("Fetching range of large storage slots", "reqid", id, "root", root, "account", accounts[0], "origin", common.BytesToHash(origin), "limit", common.BytesToHash(limit), "bytes", common.StorageSize(bytes)) @@ -207,7 +207,7 @@ func (t *testPeer) RequestStorageRanges(id uint64, root common.Hash, accounts [] return nil } -func (t *testPeer) RequestByteCodes(id uint64, hashes []common.Hash, bytes uint64) error { +func (t *testPeer) RequestByteCodes(id uint64, hashes []common.Hash, bytes int) error { t.nBytecodeRequests++ t.logger.Trace("Fetching set of byte codes", "reqid", id, "hashes", len(hashes), "bytes", common.StorageSize(bytes)) go t.codeRequestHandler(t, id, hashes, bytes) @@ -215,7 +215,7 @@ func (t *testPeer) RequestByteCodes(id uint64, hashes []common.Hash, bytes uint6 } // defaultTrieRequestHandler is a well-behaving handler for trie healing requests -func defaultTrieRequestHandler(t *testPeer, requestId uint64, root common.Hash, paths []TrieNodePathSet, cap uint64) error { +func defaultTrieRequestHandler(t *testPeer, requestId uint64, root common.Hash, paths []TrieNodePathSet, cap int) error { // Pass the response var nodes [][]byte for _, pathset := range paths { @@ -244,7 +244,7 @@ func defaultTrieRequestHandler(t *testPeer, requestId uint64, root common.Hash, } // defaultAccountRequestHandler is a well-behaving handler for AccountRangeRequests -func defaultAccountRequestHandler(t *testPeer, id uint64, root common.Hash, origin common.Hash, limit common.Hash, cap uint64) error { +func defaultAccountRequestHandler(t *testPeer, id uint64, root common.Hash, origin common.Hash, limit common.Hash, cap int) error { keys, vals, proofs := createAccountRequestResponse(t, root, origin, limit, cap) if err := t.remote.OnAccounts(t, id, keys, vals, proofs); err != nil { t.test.Errorf("Remote side rejected our delivery: %v", err) @@ -254,8 +254,8 @@ func defaultAccountRequestHandler(t *testPeer, id uint64, root common.Hash, orig return nil } -func createAccountRequestResponse(t *testPeer, root common.Hash, origin common.Hash, limit common.Hash, cap uint64) (keys []common.Hash, vals [][]byte, proofs [][]byte) { - var size uint64 +func createAccountRequestResponse(t *testPeer, root common.Hash, origin common.Hash, limit common.Hash, cap int) (keys []common.Hash, vals [][]byte, proofs [][]byte) { + var size int if limit == (common.Hash{}) { limit = common.MaxHash } @@ -266,7 +266,7 @@ func createAccountRequestResponse(t *testPeer, root common.Hash, origin common.H if bytes.Compare(origin[:], entry.k) <= 0 { keys = append(keys, common.BytesToHash(entry.k)) vals = append(vals, entry.v) - size += uint64(32 + len(entry.v)) + size += 32 + len(entry.v) } // If we've exceeded the request threshold, abort if bytes.Compare(entry.k, limit[:]) >= 0 { @@ -290,7 +290,7 @@ func createAccountRequestResponse(t *testPeer, root common.Hash, origin common.H } // defaultStorageRequestHandler is a well-behaving storage request handler -func defaultStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, bOrigin, bLimit []byte, max uint64) error { +func defaultStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, bOrigin, bLimit []byte, max int) error { hashes, slots, proofs := createStorageRequestResponse(t, root, accounts, bOrigin, bLimit, max) if err := t.remote.OnStorage(t, requestId, hashes, slots, proofs); err != nil { t.test.Errorf("Remote side rejected our delivery: %v", err) @@ -299,7 +299,7 @@ func defaultStorageRequestHandler(t *testPeer, requestId uint64, root common.Has return nil } -func defaultCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max uint64) error { +func defaultCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max int) error { var bytecodes [][]byte for _, h := range hashes { bytecodes = append(bytecodes, getCodeByHash(h)) @@ -311,8 +311,8 @@ func defaultCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max return nil } -func createStorageRequestResponse(t *testPeer, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) (hashes [][]common.Hash, slots [][][]byte, proofs [][]byte) { - var size uint64 +func createStorageRequestResponse(t *testPeer, root common.Hash, accounts []common.Hash, origin, limit []byte, max int) (hashes [][]common.Hash, slots [][][]byte, proofs [][]byte) { + var size int for _, account := range accounts { // The first account might start from a different origin and end sooner var originHash common.Hash @@ -338,7 +338,7 @@ func createStorageRequestResponse(t *testPeer, root common.Hash, accounts []comm } keys = append(keys, common.BytesToHash(entry.k)) vals = append(vals, entry.v) - size += uint64(32 + len(entry.v)) + size += 32 + len(entry.v) if bytes.Compare(entry.k, limitHash[:]) >= 0 { break } @@ -377,8 +377,8 @@ func createStorageRequestResponse(t *testPeer, root common.Hash, accounts []comm // createStorageRequestResponseAlwaysProve tests a cornercase, where the peer always // supplies the proof for the last account, even if it is 'complete'. -func createStorageRequestResponseAlwaysProve(t *testPeer, root common.Hash, accounts []common.Hash, bOrigin, bLimit []byte, max uint64) (hashes [][]common.Hash, slots [][][]byte, proofs [][]byte) { - var size uint64 +func createStorageRequestResponseAlwaysProve(t *testPeer, root common.Hash, accounts []common.Hash, bOrigin, bLimit []byte, max int) (hashes [][]common.Hash, slots [][][]byte, proofs [][]byte) { + var size int max = max * 3 / 4 var origin common.Hash @@ -395,7 +395,7 @@ func createStorageRequestResponseAlwaysProve(t *testPeer, root common.Hash, acco } keys = append(keys, common.BytesToHash(entry.k)) vals = append(vals, entry.v) - size += uint64(32 + len(entry.v)) + size += 32 + len(entry.v) if size > max { exit = true } @@ -433,34 +433,34 @@ func createStorageRequestResponseAlwaysProve(t *testPeer, root common.Hash, acco } // emptyRequestAccountRangeFn is a rejects AccountRangeRequests -func emptyRequestAccountRangeFn(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap uint64) error { +func emptyRequestAccountRangeFn(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap int) error { t.remote.OnAccounts(t, requestId, nil, nil, nil) return nil } -func nonResponsiveRequestAccountRangeFn(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap uint64) error { +func nonResponsiveRequestAccountRangeFn(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap int) error { return nil } -func emptyTrieRequestHandler(t *testPeer, requestId uint64, root common.Hash, paths []TrieNodePathSet, cap uint64) error { +func emptyTrieRequestHandler(t *testPeer, requestId uint64, root common.Hash, paths []TrieNodePathSet, cap int) error { t.remote.OnTrieNodes(t, requestId, nil) return nil } -func nonResponsiveTrieRequestHandler(t *testPeer, requestId uint64, root common.Hash, paths []TrieNodePathSet, cap uint64) error { +func nonResponsiveTrieRequestHandler(t *testPeer, requestId uint64, root common.Hash, paths []TrieNodePathSet, cap int) error { return nil } -func emptyStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error { +func emptyStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max int) error { t.remote.OnStorage(t, requestId, nil, nil, nil) return nil } -func nonResponsiveStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error { +func nonResponsiveStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max int) error { return nil } -func proofHappyStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error { +func proofHappyStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max int) error { hashes, slots, proofs := createStorageRequestResponseAlwaysProve(t, root, accounts, origin, limit, max) if err := t.remote.OnStorage(t, requestId, hashes, slots, proofs); err != nil { t.test.Errorf("Remote side rejected our delivery: %v", err) @@ -475,7 +475,7 @@ func proofHappyStorageRequestHandler(t *testPeer, requestId uint64, root common. // return nil //} -func corruptCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max uint64) error { +func corruptCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max int) error { var bytecodes [][]byte for _, h := range hashes { // Send back the hashes @@ -489,7 +489,7 @@ func corruptCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max return nil } -func cappedCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max uint64) error { +func cappedCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max int) error { var bytecodes [][]byte for _, h := range hashes[:1] { bytecodes = append(bytecodes, getCodeByHash(h)) @@ -503,11 +503,11 @@ func cappedCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max } // starvingStorageRequestHandler is somewhat well-behaving storage handler, but it caps the returned results to be very small -func starvingStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error { +func starvingStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max int) error { return defaultStorageRequestHandler(t, requestId, root, accounts, origin, limit, 500) } -func starvingAccountRequestHandler(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap uint64) error { +func starvingAccountRequestHandler(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap int) error { return defaultAccountRequestHandler(t, requestId, root, origin, limit, 500) } @@ -515,7 +515,7 @@ func starvingAccountRequestHandler(t *testPeer, requestId uint64, root common.Ha // return defaultAccountRequestHandler(t, requestId-1, root, origin, 500) //} -func corruptAccountRequestHandler(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap uint64) error { +func corruptAccountRequestHandler(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap int) error { hashes, accounts, proofs := createAccountRequestResponse(t, root, origin, limit, cap) if len(proofs) > 0 { proofs = proofs[1:] @@ -529,7 +529,7 @@ func corruptAccountRequestHandler(t *testPeer, requestId uint64, root common.Has } // corruptStorageRequestHandler doesn't provide good proofs -func corruptStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error { +func corruptStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max int) error { hashes, slots, proofs := createStorageRequestResponse(t, root, accounts, origin, limit, max) if len(proofs) > 0 { proofs = proofs[1:] @@ -542,7 +542,7 @@ func corruptStorageRequestHandler(t *testPeer, requestId uint64, root common.Has return nil } -func noProofStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error { +func noProofStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max int) error { hashes, slots, _ := createStorageRequestResponse(t, root, accounts, origin, limit, max) if err := t.remote.OnStorage(t, requestId, hashes, slots, nil); err != nil { t.logger.Info("remote error on delivery (as expected)", "error", err) @@ -577,7 +577,7 @@ func testSyncBloatedProof(t *testing.T, scheme string) { source.accountTrie = sourceAccountTrie.Copy() source.accountValues = elems - source.accountRequestHandler = func(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap uint64) error { + source.accountRequestHandler = func(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap int) error { var ( keys []common.Hash vals [][]byte @@ -1165,7 +1165,7 @@ func testSyncNoStorageAndOneCodeCappedPeer(t *testing.T, scheme string) { var counter int syncer := setupSyncer( nodeScheme, - mkSource("capped", func(t *testPeer, id uint64, hashes []common.Hash, max uint64) error { + mkSource("capped", func(t *testPeer, id uint64, hashes []common.Hash, max int) error { counter++ return cappedCodeRequestHandler(t, id, hashes, max) }), @@ -1432,7 +1432,7 @@ func testSyncWithUnevenStorage(t *testing.T, scheme string) { source.accountValues = accounts source.setStorageTries(storageTries) source.storageValues = storageElems - source.storageRequestHandler = func(t *testPeer, reqId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error { + source.storageRequestHandler = func(t *testPeer, reqId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max int) error { return defaultStorageRequestHandler(t, reqId, root, accounts, origin, limit, 128) // retrieve storage in large mode } return source diff --git a/eth/state_accessor.go b/eth/state_accessor.go index cf02cbe254..47f5dec20e 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -157,7 +157,7 @@ func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec u if current = eth.blockchain.GetBlockByNumber(next); current == nil { return nil, nil, fmt.Errorf("block #%d not found", next) } - _, err := eth.blockchain.Processor().Process(current, statedb, vm.Config{}) + _, err := eth.blockchain.Processor().Process(ctx, current, statedb, vm.Config{}) if err != nil { return nil, nil, fmt.Errorf("processing block %d failed: %v", current.NumberU64(), err) } @@ -286,7 +286,7 @@ func (eth *Ethereum) stateAtTransaction(ctx context.Context, block *types.Block, // Not yet the searched for transaction, execute on top of the current state statedb.SetTxContext(tx.Hash(), idx) - if _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { + if _, err := core.ApplyMessage(evm, msg, nil); err != nil { return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) } // Ensure any modifications are committed to the state diff --git a/eth/sync.go b/eth/sync.go index ddae8443a3..8b4bd90abf 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -25,7 +25,8 @@ import ( // syncTransactions starts sending all currently pending transactions to the given peer. func (h *handler) syncTransactions(p *eth.Peer) { var hashes []common.Hash - for _, batch := range h.txpool.Pending(txpool.PendingFilter{BlobTxs: false}) { + pending, _ := h.txpool.Pending(txpool.PendingFilter{BlobTxs: false}) + for _, batch := range pending { for _, tx := range batch { hashes = append(hashes, tx.Hash) } diff --git a/eth/sync_test.go b/eth/sync_test.go index dc295f2790..77a50bf6d3 100644 --- a/eth/sync_test.go +++ b/eth/sync_test.go @@ -28,7 +28,7 @@ import ( ) // Tests that snap sync is disabled after a successful sync cycle. -func TestSnapSyncDisabling68(t *testing.T) { testSnapSyncDisabling(t, eth.ETH68, snap.SNAP1) } +func TestSnapSyncDisabling69(t *testing.T) { testSnapSyncDisabling(t, eth.ETH69, snap.SNAP1) } // Tests that snap sync gets disabled as soon as a real block is successfully // imported into the blockchain. @@ -36,17 +36,11 @@ func testSnapSyncDisabling(t *testing.T, ethVer uint, snapVer uint) { t.Parallel() // Create an empty handler and ensure it's in snap sync mode - empty := newTestHandler() - if !empty.handler.snapSync.Load() { - t.Fatalf("snap sync disabled on pristine blockchain") - } + empty := newTestHandler(ethconfig.SnapSync) defer empty.close() // Create a full handler and ensure snap sync ends up disabled - full := newTestHandlerWithBlocks(1024) - if full.handler.snapSync.Load() { - t.Fatalf("snap sync not disabled on non-empty blockchain") - } + full := newTestHandlerWithBlocks(1024, ethconfig.SnapSync) defer full.close() // Sync up the two handlers via both `eth` and `snap` @@ -85,18 +79,21 @@ func testSnapSyncDisabling(t *testing.T, ethVer uint, snapVer uint) { time.Sleep(250 * time.Millisecond) // Check that snap sync was disabled - if err := empty.handler.downloader.BeaconSync(ethconfig.SnapSync, full.chain.CurrentBlock(), nil); err != nil { + if err := empty.handler.downloader.BeaconSync(full.chain.CurrentBlock(), nil); err != nil { t.Fatal("sync failed:", err) } - // Downloader internally has to wait for a timer (3s) to be expired before - // exiting. Poll after to determine if sync is disabled. - time.Sleep(time.Second * 3) - for timeout := time.After(time.Second); ; { + // Snap sync and mode switching happen asynchronously, poll for completion. + timeout := time.NewTimer(15 * time.Second) + tick := time.NewTicker(100 * time.Millisecond) + defer timeout.Stop() + defer tick.Stop() + + for { select { - case <-timeout: + case <-timeout.C: t.Fatalf("snap sync not disabled after successful synchronisation") - case <-time.After(100 * time.Millisecond): - if !empty.handler.snapSync.Load() { + case <-tick.C: + if empty.handler.synced.Load() && empty.handler.downloader.ConfigSyncMode() == ethconfig.FullSync { return } } diff --git a/eth/syncer/syncer.go b/eth/syncer/syncer.go index 6b33ec54ba..c0d54b953b 100644 --- a/eth/syncer/syncer.go +++ b/eth/syncer/syncer.go @@ -22,6 +22,7 @@ import ( "sync" "time" + "github.com/ethereum/go-ethereum/beacon/params" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" @@ -129,17 +130,43 @@ func (s *Syncer) run() { break } if resync { - req.errc <- s.backend.Downloader().BeaconDevSync(ethconfig.FullSync, target) + if mode := s.backend.Downloader().ConfigSyncMode(); mode != ethconfig.FullSync { + req.errc <- fmt.Errorf("unsupported syncmode %v, please relaunch geth with --syncmode full", mode) + } else { + req.errc <- s.backend.Downloader().BeaconDevSync(target) + } } case <-ticker.C: - if target == nil || !s.exitWhenSynced { + if target == nil { continue } - if block := s.backend.BlockChain().GetBlockByHash(target.Hash()); block != nil { - log.Info("Sync target reached", "number", block.NumberU64(), "hash", block.Hash()) - go s.stack.Close() // async since we need to close ourselves - return + + // Terminate the node if the target has been reached + if s.exitWhenSynced { + if block := s.backend.BlockChain().GetBlockByHash(target.Hash()); block != nil { + log.Info("Sync target reached", "number", block.NumberU64(), "hash", block.Hash()) + go s.stack.Close() // async since we need to close ourselves + return + } + } + + // Set the finalized and safe markers relative to the current head. + // The finalized marker is set two epochs behind the target, + // and the safe marker is set one epoch behind the target. + head := s.backend.BlockChain().CurrentHeader() + if head == nil { + continue + } + if header := s.backend.BlockChain().GetHeaderByNumber(head.Number.Uint64() - params.EpochLength*2); header != nil { + if final := s.backend.BlockChain().CurrentFinalBlock(); final == nil || final.Number.Cmp(header.Number) < 0 { + s.backend.BlockChain().SetFinalized(header) + } + } + if header := s.backend.BlockChain().GetHeaderByNumber(head.Number.Uint64() - params.EpochLength); header != nil { + if safe := s.backend.BlockChain().CurrentSafeBlock(); safe == nil || safe.Number.Cmp(header.Number) < 0 { + s.backend.BlockChain().SetSafe(header) + } } case <-s.closed: diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 814e13a200..9c14d6a002 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -559,7 +559,7 @@ func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config } msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee(), runCtx) statedb.SetTxContext(tx.Hash(), i) - if _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(msg.GasLimit)); err != nil { + if _, err := core.ApplyMessage(evm, msg, nil); err != nil { log.Warn("Tracing intermediate roots did not complete", "txindex", i, "txhash", tx.Hash(), "err", err) // We intentionally don't return the error here: if we do, then the RPC server will not // return the roots. Most likely, the caller already knows that a certain transaction fails to @@ -719,7 +719,7 @@ txloop: // Generate the next state snapshot fast without tracing msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee(), runCtx) statedb.SetTxContext(tx.Hash(), i) - if _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(msg.GasLimit)); err != nil { + if _, err := core.ApplyMessage(evm, msg, nil); err != nil { failed = err break txloop } @@ -806,7 +806,7 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee(), runCtx) if txHash != (common.Hash{}) && tx.Hash() != txHash { // Process the tx to update state, but don't trace it. - _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(msg.GasLimit)) + _, err := core.ApplyMessage(evm, msg, nil) if err != nil { return dumps, err } @@ -841,7 +841,7 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block if tracer.OnTxStart != nil { tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) } - _, err = core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(msg.GasLimit)) + _, err = core.ApplyMessage(evm, msg, nil) if writer != nil { writer.Flush() } @@ -1000,7 +1000,7 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, bloc } var ( msg = args.ToMessage(blockContext.BaseFee, api.backend.RPCGasCap(), block.Header(), statedb, core.NewMessageEthcallContext(), true) - tx = args.ToTransaction(types.LegacyTxType) + tx = args.ToTransaction(types.DynamicFeeTxType) traceConfig *TraceConfig ) // Lower the basefee to 0 to avoid breaking EVM @@ -1026,7 +1026,6 @@ func (api *API) traceTx(ctx context.Context, tx *types.Transaction, message *cor tracer *Tracer err error timeout = defaultTraceTimeout - usedGas uint64 ) if config == nil { config = &TraceConfig{} @@ -1070,7 +1069,7 @@ func (api *API) traceTx(ctx context.Context, tx *types.Transaction, message *cor // Call Prepare to clear out the statedb access list statedb.SetTxContext(txctx.TxHash, txctx.TxIndex) - _, _, err = core.ApplyTransactionWithEVM(message, new(core.GasPool).AddGas(message.GasLimit), statedb, vmctx.BlockNumber, txctx.BlockHash, vmctx.Time, tx, &usedGas, evm, nil) + _, _, err = core.ApplyTransactionWithEVM(message, core.NewGasPool(message.GasLimit), statedb, vmctx.BlockNumber, txctx.BlockHash, vmctx.Time, tx, evm, nil) if err != nil { return nil, fmt.Errorf("tracing failed: %w", err) } diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index 80653b4fab..5a9caca2ac 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -194,7 +194,7 @@ func (b *testBackend) StateAtTransaction(ctx context.Context, block *types.Block return tx, context, statedb, release, nil } msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee(), runCtx) - if _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { + if _, err := core.ApplyMessage(evm, msg, nil); err != nil { return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) } statedb.Finalise(evm.ChainConfig().IsEIP158(block.Number())) @@ -1379,3 +1379,350 @@ func TestStandardTraceBlockToFile(t *testing.T) { } } } + +func TestTraceBadBlock(t *testing.T) { + t.Parallel() + + var ( + accounts = newAccounts(2) + storageContract = common.HexToAddress("0x00000000000000000000000000000000deadbeef") + signer = types.HomesteadSigner{} + txHashs = make([]common.Hash, 0, 2) + genesis = &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{ + accounts[0].addr: {Balance: big.NewInt(params.Ether)}, + accounts[1].addr: {Balance: big.NewInt(params.Ether)}, + storageContract: { + Nonce: 1, + Balance: big.NewInt(0), + Code: []byte{ + byte(vm.PUSH1), 0x2a, // push 42 + byte(vm.PUSH1), 0x00, // push slot 0 + byte(vm.SSTORE), // sstore(0, 42) + byte(vm.STOP), + }, + }, + }, + } + ) + backend := newTestBackend(t, 1, genesis, func(i int, b *core.BlockGen) { + // tx 0: plain transfer + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 0, + To: &accounts[1].addr, + Value: big.NewInt(1000), + Gas: params.TxGas, + GasPrice: b.BaseFee(), + Data: nil}), + signer, accounts[0].key) + b.AddTx(tx) + txHashs = append(txHashs, tx.Hash()) + + // tx 1: call storage contract (executes PUSH1, PUSH1, SSTORE, STOP) + tx, _ = types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 1, + To: &storageContract, + Value: big.NewInt(0), + Gas: 50000, + GasPrice: b.BaseFee(), + Data: nil}), + signer, accounts[0].key) + b.AddTx(tx) + txHashs = append(txHashs, tx.Hash()) + }) + defer backend.teardown() + + // Write the block as a bad block so parent state is available + block := backend.chain.GetBlockByNumber(1) + rawdb.WriteBadBlock(backend.chaindb, block) + + api := NewAPI(backend) + result, err := api.TraceBadBlock(context.Background(), block.Hash(), nil) + if err != nil { + t.Fatalf("want no error, have %v", err) + } + if len(result) != 2 { + t.Fatalf("expected 2 tx traces, got %d", len(result)) + } + + // First tx: plain transfer + have, _ := json.Marshal(result) + var traces []struct { + TxHash common.Hash `json:"txHash"` + Result struct { + Gas uint64 `json:"gas"` + Failed bool `json:"failed"` + StructLogs []json.RawMessage `json:"structLogs"` + } `json:"result"` + } + if err := json.Unmarshal(have, &traces); err != nil { + t.Fatalf("failed to unmarshal traces: %v", err) + } + if traces[0].TxHash != txHashs[0] { + t.Errorf("tx 0: hash mismatch, have %v, want %v", traces[0].TxHash, txHashs[0]) + } + if traces[0].Result.Gas != params.TxGas { + t.Errorf("tx 0: gas mismatch, have %d, want %d", traces[0].Result.Gas, params.TxGas) + } + if len(traces[0].Result.StructLogs) != 0 { + t.Errorf("tx 0: expected empty structLogs for plain transfer, got %d entries", len(traces[0].Result.StructLogs)) + } + + // Second tx: contract call + if traces[1].TxHash != txHashs[1] { + t.Errorf("tx 1: hash mismatch, have %v, want %v", traces[1].TxHash, txHashs[1]) + } + if traces[1].Result.Failed { + t.Error("tx 1: expected success, got failed") + } + // Contract has 4 opcodes: PUSH1, PUSH1, SSTORE, STOP + if len(traces[1].Result.StructLogs) != 4 { + t.Errorf("tx 1: expected 4 structLog entries for contract call, got %d", len(traces[1].Result.StructLogs)) + } + + // Non-existent bad block + _, err = api.TraceBadBlock(context.Background(), common.Hash{42}, nil) + if err == nil { + t.Fatal("want error for non-existent bad block, have none") + } + wantErr := fmt.Sprintf("bad block %#x not found", common.Hash{42}) + if err.Error() != wantErr { + t.Errorf("error mismatch, want '%s', have '%v'", wantErr, err) + } +} + +func TestIntermediateRoots(t *testing.T) { + t.Parallel() + + // Initialize test accounts and a contract that writes to storage. + var ( + accounts = newAccounts(2) + storageContract = common.HexToAddress("0x00000000000000000000000000000000deadbeef") + signer = types.HomesteadSigner{} + genesis = &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{ + accounts[0].addr: {Balance: big.NewInt(params.Ether)}, + accounts[1].addr: {Balance: big.NewInt(params.Ether)}, + // Contract: SSTORE(CALLVALUE, CALLVALUE) + storageContract: { + Nonce: 1, + Balance: big.NewInt(0), + Code: []byte{ + byte(vm.CALLVALUE), + byte(vm.CALLVALUE), + byte(vm.SSTORE), + byte(vm.STOP), + }, + }, + }, + } + ) + backend := newTestBackend(t, 1, genesis, func(i int, b *core.BlockGen) { + // tx 0: plain transfer + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 0, + To: &accounts[1].addr, + Value: big.NewInt(1000), + Gas: params.TxGas, + GasPrice: b.BaseFee(), + Data: nil}), + signer, accounts[0].key) + b.AddTx(tx) + + // tx 1: sstore(1, 1) + tx, _ = types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 1, + To: &storageContract, + Value: big.NewInt(1), + Gas: 50000, + GasPrice: b.BaseFee(), + Data: nil}), + signer, accounts[0].key) + b.AddTx(tx) + + // tx 2: sstore(2, 2) + tx, _ = types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 2, + To: &storageContract, + Value: big.NewInt(2), + Gas: 50000, + GasPrice: b.BaseFee(), + Data: nil}), + signer, accounts[0].key) + b.AddTx(tx) + }) + defer backend.teardown() + + api := NewAPI(backend) + block := backend.chain.GetBlockByNumber(1) + + // Should return one root per tx + roots, err := api.IntermediateRoots(context.Background(), block.Hash(), nil) + if err != nil { + t.Fatalf("want no error, have %v", err) + } + if len(roots) != 3 { + t.Fatalf("root count mismatch, have %d, want 3", len(roots)) + } + for i, root := range roots { + if root == (common.Hash{}) { + t.Errorf("root[%d] should not be zero", i) + } + } + if roots[0] == roots[1] { + t.Error("root[0] and root[1] should differ (transfer vs sstore)") + } + if roots[1] == roots[2] { + t.Error("root[1] and root[2] should differ (sstore to different slots)") + } + + // Intermediate roots of a bad block + rawdb.WriteBadBlock(backend.chaindb, block) + badRoots, err := api.IntermediateRoots(context.Background(), block.Hash(), nil) + if err != nil { + t.Fatalf("want no error for bad block fallback, have %v", err) + } + if !reflect.DeepEqual(roots, badRoots) { + t.Errorf("bad block roots mismatch, have %v, want %v", badRoots, roots) + } + + // Genesis block: should return error + genesisBlock := backend.chain.GetBlockByNumber(0) + _, err = api.IntermediateRoots(context.Background(), genesisBlock.Hash(), nil) + if err == nil || err.Error() != "genesis is not traceable" { + t.Fatalf("want 'genesis is not traceable' error, have %v", err) + } + + // Non-existent block: should return error + _, err = api.IntermediateRoots(context.Background(), common.Hash{42}, nil) + if err == nil { + t.Fatal("want error for non-existent block, have none") + } + wantErr := fmt.Sprintf("block %#x not found", common.Hash{42}) + if err.Error() != wantErr { + t.Errorf("error mismatch, want '%s', have '%v'", wantErr, err) + } +} + +func TestStandardTraceBadBlockToFile(t *testing.T) { + var ( + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + funds = big.NewInt(1000000000000000) + + aa = common.HexToAddress("0x7217d81b76bdd8707601e959454e3d776aee5f43") + aaCode = []byte{byte(vm.PUSH1), 0x00, byte(vm.POP)} + + bb = common.HexToAddress("0x7217d81b76bdd8707601e959454e3d776aee5f44") + bbCode = []byte{byte(vm.PUSH2), 0x00, 0x01, byte(vm.POP)} + ) + + genesis := &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{ + address: {Balance: funds}, + aa: { + Code: aaCode, + Nonce: 1, + Balance: big.NewInt(0), + }, + bb: { + Code: bbCode, + Nonce: 1, + Balance: big.NewInt(0), + }, + }, + } + txHashs := make([]common.Hash, 0, 2) + backend := newTestBackend(t, 1, genesis, func(i int, b *core.BlockGen) { + b.SetCoinbase(common.Address{1}) + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 0, + To: &aa, + Value: big.NewInt(0), + Gas: 50000, + GasPrice: b.BaseFee(), + Data: nil, + }), types.HomesteadSigner{}, key) + b.AddTx(tx) + txHashs = append(txHashs, tx.Hash()) + + tx, _ = types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 1, + To: &bb, + Value: big.NewInt(1), + Gas: 100000, + GasPrice: b.BaseFee(), + Data: nil, + }), types.HomesteadSigner{}, key) + b.AddTx(tx) + txHashs = append(txHashs, tx.Hash()) + }) + defer backend.teardown() + + // Write the block as a bad block + block := backend.chain.GetBlockByNumber(1) + rawdb.WriteBadBlock(backend.chaindb, block) + + var testSuite = []struct { + config *StdTraceConfig + want []string + }{ + { + // All txs traced + config: nil, + want: []string{ + `{"pc":0,"op":96,"gas":"0x7148","gasCost":"0x3","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"PUSH1"} +{"pc":2,"op":80,"gas":"0x7145","gasCost":"0x2","memSize":0,"stack":["0x0"],"depth":1,"refund":0,"opName":"POP"} +{"pc":3,"op":0,"gas":"0x7143","gasCost":"0x0","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"STOP"} +{"output":"","gasUsed":"0x5"} +`, + `{"pc":0,"op":97,"gas":"0x13498","gasCost":"0x3","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"PUSH2"} +{"pc":3,"op":80,"gas":"0x13495","gasCost":"0x2","memSize":0,"stack":["0x1"],"depth":1,"refund":0,"opName":"POP"} +{"pc":4,"op":0,"gas":"0x13493","gasCost":"0x0","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"STOP"} +{"output":"","gasUsed":"0x5"} +`, + }, + }, + { + // Specific tx traced + config: &StdTraceConfig{TxHash: txHashs[1]}, + want: []string{ + `{"pc":0,"op":97,"gas":"0x13498","gasCost":"0x3","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"PUSH2"} +{"pc":3,"op":80,"gas":"0x13495","gasCost":"0x2","memSize":0,"stack":["0x1"],"depth":1,"refund":0,"opName":"POP"} +{"pc":4,"op":0,"gas":"0x13493","gasCost":"0x0","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"STOP"} +{"output":"","gasUsed":"0x5"} +`, + }, + }, + } + + api := NewAPI(backend) + for i, tc := range testSuite { + txTraces, err := api.StandardTraceBadBlockToFile(context.Background(), block.Hash(), tc.config) + if err != nil { + t.Fatalf("test %d: unexpected error %v", i, err) + } + if len(txTraces) != len(tc.want) { + t.Fatalf("test %d: file count mismatch, have %d, want %d", i, len(txTraces), len(tc.want)) + } + for j, traceFileName := range txTraces { + defer os.Remove(traceFileName) + traceReceived, err := os.ReadFile(traceFileName) + if err != nil { + t.Fatalf("test %d: could not read trace file: %v", i, err) + } + if tc.want[j] != string(traceReceived) { + t.Fatalf("test %d, trace %d: result mismatch\nhave:\n%s\nwant:\n%s", i, j, string(traceReceived), tc.want[j]) + } + } + } + + // Non-existent bad block + _, err := api.StandardTraceBadBlockToFile(context.Background(), common.Hash{42}, nil) + if err == nil { + t.Fatal("want error for non-existent bad block, have none") + } +} diff --git a/eth/tracers/internal/tracetest/calltrace_test.go b/eth/tracers/internal/tracetest/calltrace_test.go index 1bca0ea3eb..bed9977da9 100644 --- a/eth/tracers/internal/tracetest/calltrace_test.go +++ b/eth/tracers/internal/tracetest/calltrace_test.go @@ -43,6 +43,7 @@ type callLog struct { Address common.Address `json:"address"` Topics []common.Hash `json:"topics"` Data hexutil.Bytes `json:"data"` + Index hexutil.Uint `json:"index"` Position hexutil.Uint `json:"position"` } @@ -142,7 +143,7 @@ func testCallTracer(tracerName string, dirPath string, t *testing.T) { } evm := vm.NewEVM(context, logState, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) - vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) + vmRet, err := core.ApplyMessage(evm, msg, nil) if err != nil { t.Fatalf("failed to execute transaction: %v", err) } @@ -234,7 +235,7 @@ func benchTracer(tracerName string, test *callTracerTest, b *testing.B) { if tracer.OnTxStart != nil { tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) } - _, err = core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) + _, err = core.ApplyMessage(evm, msg, nil) if err != nil { b.Fatalf("failed to execute transaction: %v", err) } @@ -311,7 +312,7 @@ func TestInternals(t *testing.T) { byte(vm.LOG0), }, tracer: mkTracer("callTracer", json.RawMessage(`{ "withLog": true }`)), - want: fmt.Sprintf(`{"beforeEVMTransfers":[{"purpose":"feePayment","from":"%s","to":null,"value":"0x13880"}],"afterEVMTransfers":[{"purpose":"gasRefund","from":null,"to":"%s","value":"0xdce2"},{"purpose":"tip","from":null,"to":"0x0000000000000000000000000000000000000000","value":"0x5b9e"}],"from":"%s","gas":"0x13880","gasUsed":"0x5b9e","to":"0x00000000000000000000000000000000deadbeef","input":"0x","logs":[{"address":"0x00000000000000000000000000000000deadbeef","topics":[],"data":"0x000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","position":"0x0"}],"value":"0x0","type":"CALL"}`, origin.Hex(), origin.Hex(), originHex), + want: fmt.Sprintf(`{"beforeEVMTransfers":[{"purpose":"feePayment","from":"%s","to":null,"value":"0x13880"}],"afterEVMTransfers":[{"purpose":"gasRefund","from":null,"to":"%s","value":"0xdce2"},{"purpose":"tip","from":null,"to":"0x0000000000000000000000000000000000000000","value":"0x5b9e"}],"from":"%s","gas":"0x13880","gasUsed":"0x5b9e","to":"0x00000000000000000000000000000000deadbeef","input":"0x","logs":[{"address":"0x00000000000000000000000000000000deadbeef","topics":[],"data":"0x000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","index":"0x0","position":"0x0"}],"value":"0x0","type":"CALL"}`, origin.Hex(), origin.Hex(), originHex), }, { // Leads to OOM on the prestate tracer @@ -384,7 +385,7 @@ func TestInternals(t *testing.T) { t.Fatalf("test %v: failed to create message: %v", tc.name, err) } tc.tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) - vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) + vmRet, err := core.ApplyMessage(evm, msg, nil) if err != nil { t.Fatalf("test %v: failed to execute transaction: %v", tc.name, err) } diff --git a/eth/tracers/internal/tracetest/erc7562_tracer_test.go b/eth/tracers/internal/tracetest/erc7562_tracer_test.go index 9cb19bf6fa..8161d8fafb 100644 --- a/eth/tracers/internal/tracetest/erc7562_tracer_test.go +++ b/eth/tracers/internal/tracetest/erc7562_tracer_test.go @@ -125,7 +125,7 @@ func TestErc7562Tracer(t *testing.T) { } evm := vm.NewEVM(context, logState, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) - vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) + vmRet, err := core.ApplyMessage(evm, msg, nil) if err != nil { t.Fatalf("failed to execute transaction: %v", err) } diff --git a/eth/tracers/internal/tracetest/flat_calltrace_test.go b/eth/tracers/internal/tracetest/flat_calltrace_test.go index 6f5a2f0ca2..c08441a132 100644 --- a/eth/tracers/internal/tracetest/flat_calltrace_test.go +++ b/eth/tracers/internal/tracetest/flat_calltrace_test.go @@ -113,7 +113,7 @@ func flatCallTracerTestRunner(tracerName string, filename string, dirPath string } evm := vm.NewEVM(context, state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) - vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) + vmRet, err := core.ApplyMessage(evm, msg, nil) if err != nil { return fmt.Errorf("failed to execute transaction: %v", err) } diff --git a/eth/tracers/internal/tracetest/prestate_test.go b/eth/tracers/internal/tracetest/prestate_test.go index 6a09f28d61..038207c736 100644 --- a/eth/tracers/internal/tracetest/prestate_test.go +++ b/eth/tracers/internal/tracetest/prestate_test.go @@ -105,7 +105,7 @@ func testPrestateTracer(tracerName string, dirPath string, t *testing.T) { } evm := vm.NewEVM(context, state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) - vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) + vmRet, err := core.ApplyMessage(evm, msg, nil) if err != nil { t.Fatalf("failed to execute transaction: %v", err) } diff --git a/eth/tracers/internal/tracetest/selfdestruct_state_test.go b/eth/tracers/internal/tracetest/selfdestruct_state_test.go new file mode 100644 index 0000000000..c1491b0d93 --- /dev/null +++ b/eth/tracers/internal/tracetest/selfdestruct_state_test.go @@ -0,0 +1,652 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package tracetest + +import ( + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/consensus/beacon" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/params" +) + +// accountState represents the expected final state of an account +type accountState struct { + Balance *big.Int + Nonce uint64 + Code []byte + Exists bool +} + +// selfdestructStateTracer tracks state changes during selfdestruct operations +type selfdestructStateTracer struct { + env *tracing.VMContext + accounts map[common.Address]*accountState +} + +func newSelfdestructStateTracer() *selfdestructStateTracer { + return &selfdestructStateTracer{ + accounts: make(map[common.Address]*accountState), + } +} + +func (t *selfdestructStateTracer) OnTxStart(env *tracing.VMContext, tx *types.Transaction, from common.Address) { + t.env = env +} + +func (t *selfdestructStateTracer) OnTxEnd(receipt *types.Receipt, err error) { + // Nothing to do +} + +func (t *selfdestructStateTracer) getOrCreateAccount(addr common.Address) *accountState { + if acc, ok := t.accounts[addr]; ok { + return acc + } + + // Initialize with current state from statedb + acc := &accountState{ + Balance: t.env.StateDB.GetBalance(addr).ToBig(), + Nonce: t.env.StateDB.GetNonce(addr), + Code: t.env.StateDB.GetCode(addr), + Exists: t.env.StateDB.Exist(addr), + } + t.accounts[addr] = acc + return acc +} + +func (t *selfdestructStateTracer) OnBalanceChange(addr common.Address, prev, new *big.Int, reason tracing.BalanceChangeReason) { + acc := t.getOrCreateAccount(addr) + acc.Balance = new +} + +func (t *selfdestructStateTracer) OnNonceChangeV2(addr common.Address, prev, new uint64, reason tracing.NonceChangeReason) { + acc := t.getOrCreateAccount(addr) + acc.Nonce = new + + // If this is a selfdestruct nonce change, mark account as not existing + if reason == tracing.NonceChangeSelfdestruct { + acc.Exists = false + } +} + +func (t *selfdestructStateTracer) OnCodeChangeV2(addr common.Address, prevCodeHash common.Hash, prevCode []byte, codeHash common.Hash, code []byte, reason tracing.CodeChangeReason) { + acc := t.getOrCreateAccount(addr) + acc.Code = code + + // If this is a selfdestruct code change, mark account as not existing + if reason == tracing.CodeChangeSelfDestruct { + acc.Exists = false + } +} + +func (t *selfdestructStateTracer) Hooks() *tracing.Hooks { + return &tracing.Hooks{ + OnTxStart: t.OnTxStart, + OnTxEnd: t.OnTxEnd, + OnBalanceChange: t.OnBalanceChange, + OnNonceChangeV2: t.OnNonceChangeV2, + OnCodeChangeV2: t.OnCodeChangeV2, + } +} + +func (t *selfdestructStateTracer) Accounts() map[common.Address]*accountState { + return t.accounts +} + +// verifyAccountState compares actual and expected account state and reports any mismatches +func verifyAccountState(t *testing.T, addr common.Address, actual, expected *accountState) { + if actual.Balance.Cmp(expected.Balance) != 0 { + t.Errorf("address %s: balance mismatch: have %s, want %s", + addr.Hex(), actual.Balance, expected.Balance) + } + if actual.Nonce != expected.Nonce { + t.Errorf("address %s: nonce mismatch: have %d, want %d", + addr.Hex(), actual.Nonce, expected.Nonce) + } + if len(actual.Code) != len(expected.Code) { + t.Errorf("address %s: code length mismatch: have %d, want %d", + addr.Hex(), len(actual.Code), len(expected.Code)) + } + if actual.Exists != expected.Exists { + t.Errorf("address %s: exists mismatch: have %v, want %v", + addr.Hex(), actual.Exists, expected.Exists) + } +} + +// setupTestBlockchain creates a blockchain with the given genesis and transaction, +// returns the blockchain, the first block, and a statedb at genesis for testing +func setupTestBlockchain(t *testing.T, genesis *core.Genesis, tx *types.Transaction, useBeacon bool) (*core.BlockChain, *types.Block, *state.StateDB) { + var engine consensus.Engine + if useBeacon { + engine = beacon.New(ethash.NewFaker()) + } else { + engine = ethash.NewFaker() + } + + _, blocks, _ := core.GenerateChainWithGenesis(genesis, engine, 1, func(i int, b *core.BlockGen) { + b.AddTx(tx) + }) + db := rawdb.NewMemoryDatabase() + blockchain, err := core.NewBlockChain(db, nil, genesis, engine, nil) + if err != nil { + t.Fatalf("failed to create blockchain: %v", err) + } + if _, err := blockchain.InsertChain(blocks); err != nil { + t.Fatalf("failed to insert chain: %v", err) + } + genesisBlock := blockchain.GetBlockByNumber(0) + if genesisBlock == nil { + t.Fatalf("failed to get genesis block") + } + statedb, err := blockchain.StateAt(genesisBlock.Root()) + if err != nil { + t.Fatalf("failed to get state: %v", err) + } + + return blockchain, blocks[0], statedb +} + +func TestSelfdestructStateTracer(t *testing.T) { + t.Parallel() + + const ( + // Gas limit high enough for all test scenarios (factory creation + multiple calls) + testGasLimit = 500000 + + // Common balance amounts used across tests + testBalanceInitial = 100 // Initial balance for contracts being tested + testBalanceSent = 50 // Amount sent back in sendback tests + testBalanceFactory = 200 // Factory needs extra balance for contract creation + ) + + // Helper to create *big.Int for wei amounts + wei := func(amount int64) *big.Int { + return big.NewInt(amount) + } + + // Test account (transaction sender) + var ( + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + caller = crypto.PubkeyToAddress(key.PublicKey) + ) + + // Simple selfdestruct test contracts + var ( + contract = common.HexToAddress("0x00000000000000000000000000000000000000bb") + recipient = common.HexToAddress("0x00000000000000000000000000000000000000cc") + ) + // Build selfdestruct code: PUSH20 SELFDESTRUCT + selfdestructCode := []byte{byte(vm.PUSH20)} + selfdestructCode = append(selfdestructCode, recipient.Bytes()...) + selfdestructCode = append(selfdestructCode, byte(vm.SELFDESTRUCT)) + + // Factory test contracts (create-and-destroy pattern) + var ( + factory = common.HexToAddress("0x00000000000000000000000000000000000000ff") + ) + // Factory code: creates a contract with 100 wei and calls it to trigger selfdestruct back to factory + // See selfdestruct_test_contracts/factory.yul for source + // Runtime bytecode compiled with: solc --strict-assembly --evm-version paris factory.yul --bin + // (Using paris to avoid PUSH0 opcode which is not available pre-Shanghai) + var ( + factoryCode = common.Hex2Bytes("6a6133ff6000526002601ef360a81b600052600080808080600b816064f05af100") + createdContractAddr = crypto.CreateAddress(factory, 0) // Address where factory creates the contract + ) + + // Sendback test contracts (A→B→A pattern) + // For the refund test: Coordinator calls A, then B + // A selfdestructs to B, B sends funds back to A + var ( + contractA = common.HexToAddress("0x00000000000000000000000000000000000000aa") + contractB = common.HexToAddress("0x00000000000000000000000000000000000000bb") + coordinator = common.HexToAddress("0x00000000000000000000000000000000000000cc") + ) + // Contract A: if msg.value > 0, accept funds; else selfdestruct to B + // See selfdestruct_test_contracts/contractA.yul for source + // Runtime bytecode compiled with: solc --strict-assembly --evm-version paris contractA.yul --bin + contractACode := common.Hex2Bytes("60003411600a5760bbff5b00") + + // Contract B: sends 50 wei back to contract A + // See selfdestruct_test_contracts/contractB.yul for source + // Runtime bytecode compiled with: solc --strict-assembly --evm-version paris contractB.yul --bin + contractBCode := common.Hex2Bytes("6000808080603260aa5af100") + + // Coordinator: calls A (A selfdestructs to B), then calls B (B sends funds to A) + // See selfdestruct_test_contracts/coordinator.yul for source + // Runtime bytecode compiled with: solc --strict-assembly --evm-version paris coordinator.yul --bin + coordinatorCode := common.Hex2Bytes("60008080808060aa818080808060bb955af1505af100") + + // Factory for create-and-refund test: creates A with 100 wei, calls A, calls B + // See selfdestruct_test_contracts/factoryRefund.yul for source + // Runtime bytecode compiled with: solc --strict-assembly --evm-version paris factoryRefund.yul --bin + var ( + factoryRefund = common.HexToAddress("0x00000000000000000000000000000000000000dd") + factoryRefundCode = common.Hex2Bytes("60008080808060bb78600c600d600039600c6000f3fe60003411600a5760bbff5b0082528180808080601960076064f05af1505af100") + createdContractAddrA = crypto.CreateAddress(factoryRefund, 0) // Address where factory creates contract A + ) + + // Self-destruct-to-self test contracts + var ( + contractSelfDestruct = common.HexToAddress("0x00000000000000000000000000000000000000aa") + coordinatorSendAfter = common.HexToAddress("0x00000000000000000000000000000000000000ee") + ) + // Contract that selfdestructs to self + // See selfdestruct_test_contracts/contractSelfDestruct.yul + contractSelfDestructCode := common.Hex2Bytes("30ff") + + // Coordinator: calls contract (triggers selfdestruct to self), stores balance, sends 50 wei, stores balance again + // See selfdestruct_test_contracts/coordinatorSendAfter.yul + coordinatorSendAfterCode := common.Hex2Bytes("60aa600080808080855af150803160005560008080806032855af1503160015500") + + // Factory with balance checking: creates contract, calls it, checks balances + // See selfdestruct_test_contracts/factorySelfDestructBalanceCheck.yul + var ( + factorySelfDestructBalanceCheck = common.HexToAddress("0x00000000000000000000000000000000000000fd") + factorySelfDestructBalanceCheckCode = common.Hex2Bytes("6e6002600d60003960026000f3fe30ff600052600f60116064f0600080808080855af150803160005560008080806032855af1503160015500") + createdContractAddrSelfBalanceCheck = crypto.CreateAddress(factorySelfDestructBalanceCheck, 0) + ) + + tests := []struct { + name string + description string + targetContract common.Address + genesis *core.Genesis + useBeacon bool + expectedResults map[common.Address]accountState + expectedStorage map[common.Address]map[uint64]*big.Int + }{ + { + name: "pre_6780_existing", + description: "Pre-EIP-6780: Existing contract selfdestructs to recipient. Contract should be destroyed and balance transferred.", + targetContract: contract, + genesis: &core.Genesis{ + Config: params.AllEthashProtocolChanges, + Alloc: types.GenesisAlloc{ + caller: {Balance: big.NewInt(params.Ether)}, + contract: { + Balance: wei(testBalanceInitial), + Code: selfdestructCode, + }, + }, + }, + useBeacon: false, + expectedResults: map[common.Address]accountState{ + contract: { + Balance: wei(0), + Nonce: 0, + Code: []byte{}, + Exists: false, + }, + recipient: { + Balance: wei(testBalanceInitial), // Received contract's balance + Nonce: 0, + Code: []byte{}, + Exists: true, + }, + }, + }, + { + name: "post_6780_existing", + description: "Post-EIP-6780: Existing contract selfdestructs to recipient. Balance transferred but contract NOT destroyed (code/storage remain).", + targetContract: contract, + genesis: &core.Genesis{ + Config: params.AllDevChainProtocolChanges, + Alloc: types.GenesisAlloc{ + caller: {Balance: big.NewInt(params.Ether)}, + contract: { + Balance: wei(testBalanceInitial), + Code: selfdestructCode, + }, + }, + }, + useBeacon: true, + expectedResults: map[common.Address]accountState{ + contract: { + Balance: wei(0), + Nonce: 0, + Code: selfdestructCode, + Exists: true, + }, + recipient: { + Balance: wei(testBalanceInitial), + Nonce: 0, + Code: []byte{}, + Exists: true, + }, + }, + }, + { + name: "pre_6780_create_destroy", + description: "Pre-EIP-6780: Factory creates contract with 100 wei, contract selfdestructs back to factory. Contract destroyed, factory gets refund.", + targetContract: factory, + genesis: &core.Genesis{ + Config: params.AllEthashProtocolChanges, + Alloc: types.GenesisAlloc{ + caller: {Balance: big.NewInt(params.Ether)}, + factory: { + Balance: wei(testBalanceFactory), + Code: factoryCode, + }, + }, + }, + useBeacon: false, + expectedResults: map[common.Address]accountState{ + factory: { + Balance: wei(testBalanceFactory), + Nonce: 1, + Code: factoryCode, + Exists: true, + }, + createdContractAddr: { + Balance: wei(0), + Nonce: 0, + Code: []byte{}, + Exists: false, + }, + }, + }, + { + name: "post_6780_create_destroy", + description: "Post-EIP-6780: Factory creates contract with 100 wei, contract selfdestructs back to factory. Contract destroyed (EIP-6780 exception for same-tx creation).", + targetContract: factory, + genesis: &core.Genesis{ + Config: params.AllDevChainProtocolChanges, + Alloc: types.GenesisAlloc{ + caller: {Balance: big.NewInt(params.Ether)}, + factory: { + Balance: wei(testBalanceFactory), + Code: factoryCode, + }, + }, + }, + useBeacon: true, + expectedResults: map[common.Address]accountState{ + factory: { + Balance: wei(testBalanceFactory), + Nonce: 1, + Code: factoryCode, + Exists: true, + }, + createdContractAddr: { + Balance: wei(0), + Nonce: 0, + Code: []byte{}, + Exists: false, + }, + }, + }, + { + name: "pre_6780_sendback", + description: "Pre-EIP-6780: Contract A selfdestructs sending funds to B, then B sends funds back to A's address. Funds sent to destroyed address are burnt.", + targetContract: coordinator, + genesis: &core.Genesis{ + Config: params.AllEthashProtocolChanges, + Alloc: types.GenesisAlloc{ + caller: {Balance: big.NewInt(params.Ether)}, + contractA: { + Balance: wei(testBalanceInitial), + Code: contractACode, + }, + contractB: { + Balance: wei(0), + Code: contractBCode, + }, + coordinator: { + Code: coordinatorCode, + }, + }, + }, + useBeacon: false, + expectedResults: map[common.Address]accountState{ + contractA: { + Balance: wei(0), + Nonce: 0, + Code: []byte{}, + Exists: false, + }, + contractB: { + // 100 received - 50 sent back + Balance: wei(testBalanceSent), + Nonce: 0, + Code: contractBCode, + Exists: true, + }, + }, + }, + { + name: "post_6780_existing_sendback", + description: "Post-EIP-6780: Existing contract A selfdestructs to B, then B sends funds back to A. Funds are NOT burnt (A still exists post-6780).", + targetContract: coordinator, + genesis: &core.Genesis{ + Config: params.AllDevChainProtocolChanges, + Alloc: types.GenesisAlloc{ + caller: {Balance: big.NewInt(params.Ether)}, + contractA: { + Balance: wei(testBalanceInitial), + Code: contractACode, + }, + contractB: { + Balance: wei(0), + Code: contractBCode, + }, + coordinator: { + Code: coordinatorCode, + }, + }, + }, + useBeacon: true, + expectedResults: map[common.Address]accountState{ + contractA: { + Balance: wei(testBalanceSent), + Nonce: 0, + Code: contractACode, + Exists: true, + }, + contractB: { + Balance: wei(testBalanceSent), + Nonce: 0, + Code: contractBCode, + Exists: true, + }, + }, + }, + { + name: "post_6780_create_destroy_sendback", + description: "Post-EIP-6780: Factory creates A, A selfdestructs to B, B sends funds back to A. Funds are burnt (A was destroyed via EIP-6780 exception).", + targetContract: factoryRefund, + genesis: &core.Genesis{ + Config: params.AllDevChainProtocolChanges, + Alloc: types.GenesisAlloc{ + caller: {Balance: big.NewInt(params.Ether)}, + contractB: { + Balance: wei(0), + Code: contractBCode, + }, + factoryRefund: { + Balance: wei(testBalanceFactory), + Code: factoryRefundCode, + }, + }, + }, + useBeacon: true, + expectedResults: map[common.Address]accountState{ + createdContractAddrA: { + // Funds sent back are burnt! + Balance: wei(0), + Nonce: 0, + Code: []byte{}, + Exists: false, + }, + contractB: { + Balance: wei(testBalanceSent), + Nonce: 0, + Code: contractBCode, + Exists: true, + }, + }, + }, + { + name: "post_6780_existing_to_self", + description: "Post-EIP-6780: Pre-existing contract selfdestructs to itself. Balance NOT burnt (selfdestruct-to-self is no-op for existing contracts).", + targetContract: coordinatorSendAfter, + genesis: &core.Genesis{ + Config: params.AllDevChainProtocolChanges, + Alloc: types.GenesisAlloc{ + caller: {Balance: big.NewInt(params.Ether)}, + contractSelfDestruct: { + Balance: wei(testBalanceInitial), + Code: contractSelfDestructCode, + }, + coordinatorSendAfter: { + Balance: wei(testBalanceInitial), + Code: coordinatorSendAfterCode, + }, + }, + }, + useBeacon: true, + expectedResults: map[common.Address]accountState{ + contractSelfDestruct: { + Balance: wei(150), + Nonce: 0, + Code: contractSelfDestructCode, + Exists: true, + }, + coordinatorSendAfter: { + Balance: wei(testBalanceSent), + Nonce: 0, + Code: coordinatorSendAfterCode, + Exists: true, + }, + }, + expectedStorage: map[common.Address]map[uint64]*big.Int{ + coordinatorSendAfter: { + 0: wei(testBalanceInitial), + 1: wei(150), + }, + }, + }, + { + name: "post_6780_create_destroy_to_self", + description: "Post-EIP-6780: Factory creates contract, contract selfdestructs to itself. Balance IS burnt and contract destroyed (EIP-6780 exception for same-tx creation).", + targetContract: factorySelfDestructBalanceCheck, + genesis: &core.Genesis{ + Config: params.AllDevChainProtocolChanges, + Alloc: types.GenesisAlloc{ + caller: {Balance: big.NewInt(params.Ether)}, + factorySelfDestructBalanceCheck: { + Balance: wei(testBalanceFactory), + Code: factorySelfDestructBalanceCheckCode, + }, + }, + }, + useBeacon: true, + expectedResults: map[common.Address]accountState{ + createdContractAddrSelfBalanceCheck: { + Balance: wei(0), + Nonce: 0, + Code: []byte{}, + Exists: false, + }, + factorySelfDestructBalanceCheck: { + Balance: wei(testBalanceSent), + Nonce: 1, + Code: factorySelfDestructBalanceCheckCode, + Exists: true, + }, + }, + expectedStorage: map[common.Address]map[uint64]*big.Int{ + factorySelfDestructBalanceCheck: { + 0: wei(0), + 1: wei(0), + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + var ( + signer = types.HomesteadSigner{} + tx *types.Transaction + err error + ) + + tx, err = types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 0, + To: &tt.targetContract, + Value: big.NewInt(0), + Gas: testGasLimit, + GasPrice: big.NewInt(params.InitialBaseFee * 2), + Data: nil, + }), signer, key) + if err != nil { + t.Fatalf("failed to sign transaction: %v", err) + } + + blockchain, block, statedb := setupTestBlockchain(t, tt.genesis, tx, tt.useBeacon) + defer blockchain.Stop() + + tracer := newSelfdestructStateTracer() + hookedState := state.NewHookedState(statedb, tracer.Hooks()) + msg, err := core.TransactionToMessage(tx, signer, nil, core.NewMessageReplayContext()) + if err != nil { + t.Fatalf("failed to prepare transaction for tracing: %v", err) + } + context := core.NewEVMBlockContext(block.Header(), blockchain, nil) + evm := vm.NewEVM(context, hookedState, tt.genesis.Config, vm.Config{Tracer: tracer.Hooks()}) + _, _, err = core.ApplyTransactionWithEVM(msg, core.NewGasPool(tx.Gas()), statedb, block.Number(), block.Hash(), block.Time(), tx, evm, nil) + if err != nil { + t.Fatalf("failed to execute transaction: %v", err) + } + + results := tracer.Accounts() + + // Verify storage + for addr, expectedSlots := range tt.expectedStorage { + for slot, expectedValue := range expectedSlots { + actualValue := statedb.GetState(addr, common.BigToHash(big.NewInt(int64(slot)))) + if actualValue.Big().Cmp(expectedValue) != 0 { + t.Errorf("address %s slot %d: storage mismatch: have %s, want %s", + addr.Hex(), slot, actualValue.Big(), expectedValue) + } + } + } + + // Verify results + for addr, expected := range tt.expectedResults { + actual, ok := results[addr] + if !ok { + t.Errorf("address %s missing from results", addr.Hex()) + continue + } + verifyAccountState(t, addr, actual, &expected) + } + }) + } +} diff --git a/eth/tracers/internal/tracetest/selfdestruct_test_contracts/contractA.yul b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/contractA.yul new file mode 100644 index 0000000000..109551f26e --- /dev/null +++ b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/contractA.yul @@ -0,0 +1,18 @@ +object "ContractA" { + code { + datacopy(0, dataoffset("Runtime"), datasize("Runtime")) + return(0, datasize("Runtime")) + } + object "Runtime" { + code { + // If receiving funds (msg.value > 0), just accept them and return + if gt(callvalue(), 0) { + stop() + } + + // Otherwise, selfdestruct to B (transfers balance immediately, then stops execution) + let contractB := 0x00000000000000000000000000000000000000bb + selfdestruct(contractB) + } + } +} diff --git a/eth/tracers/internal/tracetest/selfdestruct_test_contracts/contractB.yul b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/contractB.yul new file mode 100644 index 0000000000..c737355fb6 --- /dev/null +++ b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/contractB.yul @@ -0,0 +1,14 @@ +object "ContractB" { + code { + datacopy(0, dataoffset("Runtime"), datasize("Runtime")) + return(0, datasize("Runtime")) + } + object "Runtime" { + code { + // Send 50 wei back to contract A + let contractA := 0x00000000000000000000000000000000000000aa + let success := call(gas(), contractA, 50, 0, 0, 0, 0) + stop() + } + } +} diff --git a/eth/tracers/internal/tracetest/selfdestruct_test_contracts/contractSelfDestruct.yul b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/contractSelfDestruct.yul new file mode 100644 index 0000000000..73884c5dd4 --- /dev/null +++ b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/contractSelfDestruct.yul @@ -0,0 +1,12 @@ +object "ContractSelfDestruct" { + code { + datacopy(0, dataoffset("Runtime"), datasize("Runtime")) + return(0, datasize("Runtime")) + } + object "Runtime" { + code { + // Simply selfdestruct to self + selfdestruct(address()) + } + } +} diff --git a/eth/tracers/internal/tracetest/selfdestruct_test_contracts/coordinator.yul b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/coordinator.yul new file mode 100644 index 0000000000..54bd5c08f3 --- /dev/null +++ b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/coordinator.yul @@ -0,0 +1,20 @@ +object "Coordinator" { + code { + datacopy(0, dataoffset("Runtime"), datasize("Runtime")) + return(0, datasize("Runtime")) + } + object "Runtime" { + code { + let contractA := 0x00000000000000000000000000000000000000aa + let contractB := 0x00000000000000000000000000000000000000bb + + // First, call A (A will selfdestruct to B) + pop(call(gas(), contractA, 0, 0, 0, 0, 0)) + + // Then, call B (B will send funds back to A) + pop(call(gas(), contractB, 0, 0, 0, 0, 0)) + + stop() + } + } +} diff --git a/eth/tracers/internal/tracetest/selfdestruct_test_contracts/coordinatorSendAfter.yul b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/coordinatorSendAfter.yul new file mode 100644 index 0000000000..9473d1f3ef --- /dev/null +++ b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/coordinatorSendAfter.yul @@ -0,0 +1,27 @@ +object "CoordinatorSendAfter" { + code { + datacopy(0, dataoffset("Runtime"), datasize("Runtime")) + return(0, datasize("Runtime")) + } + object "Runtime" { + code { + let contractAddr := 0x00000000000000000000000000000000000000aa + + // Call contract (triggers selfdestruct to self, burning its balance) + pop(call(gas(), contractAddr, 0, 0, 0, 0, 0)) + + // Check contract's balance immediately after selfdestruct + // Store in slot 0 to verify it's 0 (proving immediate burn) + sstore(0, balance(contractAddr)) + + // Send 50 wei to the contract (after it selfdestructed) + pop(call(gas(), contractAddr, 50, 0, 0, 0, 0)) + + // Check balance again after sending funds + // Store in slot 1 to verify it's 50 (new funds not burnt) + sstore(1, balance(contractAddr)) + + stop() + } + } +} diff --git a/eth/tracers/internal/tracetest/selfdestruct_test_contracts/factoryRefund.yul b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/factoryRefund.yul new file mode 100644 index 0000000000..f52a46fcc3 --- /dev/null +++ b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/factoryRefund.yul @@ -0,0 +1,28 @@ +object "FactoryRefund" { + code { + datacopy(0, dataoffset("Runtime"), datasize("Runtime")) + return(0, datasize("Runtime")) + } + object "Runtime" { + code { + let contractB := 0x00000000000000000000000000000000000000bb + + // Store the deploy bytecode for contract A in memory + // Full deploy bytecode from: solc --strict-assembly --evm-version paris contractA.yul --bin + // Including the 0xfe separator: 600c600d600039600c6000f3fe60003411600a5760bbff5b00 + // That's 25 bytes, padded to 32 bytes with 7 zero bytes at the front + mstore(0, 0x0000000000000000000000000000600c600d600039600c6000f3fe60003411600a5760bbff5b00) + + // CREATE contract A with 100 wei, using 25 bytes starting at position 7 + let contractA := create(100, 7, 25) + + // Call contract A (triggers selfdestruct to B) + pop(call(gas(), contractA, 0, 0, 0, 0, 0)) + + // Call contract B (B sends 50 wei back to A) + pop(call(gas(), contractB, 0, 0, 0, 0, 0)) + + stop() + } + } +} diff --git a/eth/tracers/internal/tracetest/selfdestruct_test_contracts/factorySelfDestructBalanceCheck.yul b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/factorySelfDestructBalanceCheck.yul new file mode 100644 index 0000000000..46f4628419 --- /dev/null +++ b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/factorySelfDestructBalanceCheck.yul @@ -0,0 +1,35 @@ +object "FactorySelfDestructBalanceCheck" { + code { + datacopy(0, dataoffset("Runtime"), datasize("Runtime")) + return(0, datasize("Runtime")) + } + object "Runtime" { + code { + // Get the full deploy bytecode for ContractSelfDestruct + // Compiled with: solc --strict-assembly --evm-version paris contractSelfDestruct.yul --bin + // Full bytecode: 6002600d60003960026000f3fe30ff + // That's 15 bytes total, padded to 32 bytes with 17 zero bytes at front + mstore(0, 0x0000000000000000000000000000000000000000006002600d60003960026000f3fe30ff) + + // CREATE contract with 100 wei, using deploy bytecode + // The bytecode is 15 bytes, starts at position 17 in the 32-byte word + let contractAddr := create(100, 17, 15) + + // Call the created contract (triggers selfdestruct to self) + pop(call(gas(), contractAddr, 0, 0, 0, 0, 0)) + + // Check contract's balance immediately after selfdestruct + // Store in slot 0 to verify it's 0 (proving immediate burn) + sstore(0, balance(contractAddr)) + + // Send 50 wei to the contract (after it selfdestructed) + pop(call(gas(), contractAddr, 50, 0, 0, 0, 0)) + + // Check balance again after sending funds + // Store in slot 1 to verify it's 0 (funds sent to destroyed contract are burnt) + sstore(1, balance(contractAddr)) + + stop() + } + } +} diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/calldata.json b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/calldata.json index e213777848..87797622e1 100644 --- a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/calldata.json +++ b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/calldata.json @@ -116,7 +116,8 @@ "0xe1c52dc63b719ade82e8bea94cc41a0d5d28e4aaf536adb5e9cccc9ff8c1aeda" ], "data": "0x0000000000000000000000004f5777744b500616697cb655dcb02ee6cd51deb5be96016bb57376da7a6d296e0a405ee1501778227dfa604df0a81cb1ae018598", - "position": "0x0" + "position": "0x0", + "index": "0x0" }, { "address": "0x200edd17f30485a8735878661960cd7a9a95733f", @@ -124,7 +125,8 @@ "0xacbdb084c721332ac59f9b8e392196c9eb0e4932862da8eb9beaf0dad4f550da" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000000", - "position": "0x0" + "position": "0x0", + "index": "0x1" } ], "value": "0x8ac7230489e80000", @@ -134,4 +136,4 @@ "value": "0x8ac7230489e80000", "type": "CALL" } -} +} \ No newline at end of file diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/delegatecall.json b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/delegatecall.json index 5078f854d6..afbdc36778 100644 --- a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/delegatecall.json +++ b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/delegatecall.json @@ -278,6 +278,7 @@ "0x0000000000000000000000006ca7f214ab2ddbb9a8e1a1e2c8550e3164e9dba5" ], "data": "0x00000000000000000000000000000000000000000000000080d29fa5cccfadac", + "index": "0x0", "position": "0x0" } ], @@ -300,6 +301,7 @@ "0x0000000000000000000000005aae5c59d642e5fd45b427df6ed478b49d55fefd" ], "data": "0x00000000000000000000000000000000000000000000000080d29fa5cccfadac", + "index": "0x1", "position": "0x0" } ], @@ -330,6 +332,7 @@ "0x0000000000000000000000005aae5c59d642e5fd45b427df6ed478b49d55fefd" ], "data": "0x00000000000000000000000000000000000000000000000080d29fa5cccfadac", + "index": "0x2", "position": "0x0" } ], @@ -352,6 +355,7 @@ "0x000000000000000000000000950ca4a06c78934a148b7a3ff3ea8fc366f77a06" ], "data": "0x0000000000000000000000000000000000000000000000000041f50e27d56848", + "index": "0x3", "position": "0x0" } ], @@ -416,6 +420,7 @@ "0x0000000000000000000000003de712784baf97260455ae25fb74f574ec9c1add" ], "data": "0x000000000000000000000000000000000000000000000000de0b6b3a76400000", + "index": "0x4", "position": "0x0" } ], @@ -430,4 +435,4 @@ "value": "0x0", "type": "CALL" } -} +} \ No newline at end of file diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/multi_contracts.json b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/multi_contracts.json index 6406c8d867..0545fad562 100644 --- a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/multi_contracts.json +++ b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/multi_contracts.json @@ -378,6 +378,7 @@ "0x0000000000000000000000004fd27b205895e698fa350f7ea57cec8a21927fcd" ], "data": "0x00000000000000000000000000000000000000000001819451f999d617dafa93", + "index": "0x0", "position": "0x0" } ], @@ -392,6 +393,7 @@ "0x69ca02dd4edd7bf0a4abb9ed3b7af3f14778db5d61921c7dc7cd545266326de2" ], "data": "0x0000000000000000000000004fd27b205895e698fa350f7ea57cec8a21927fcd00000000000000000000000000000000000000000001819451f999d617dafa93", + "index": "0x1", "position": "0x1" } ], @@ -514,6 +516,7 @@ "0x0000000000000000000000004fd27b205895e698fa350f7ea57cec8a21927fcd" ], "data": "0x00000000000000000000000000000000000000000001819451f999d617dafa76", + "index": "0x2", "position": "0x0" } ], @@ -528,6 +531,7 @@ "0x69ca02dd4edd7bf0a4abb9ed3b7af3f14778db5d61921c7dc7cd545266326de2" ], "data": "0x0000000000000000000000004fd27b205895e698fa350f7ea57cec8a21927fcd00000000000000000000000000000000000000000001819451f999d617dafa76", + "index": "0x3", "position": "0x1" } ], @@ -717,6 +721,7 @@ "0x0000000000000000000000006e715ab4f598eacf0016b9b35ef33e4141844ccc" ], "data": "0x0000000000000000000000000000000000000000000181a7ae53ea2f0bef8ccd", + "index": "0x5", "position": "0x0" } ], @@ -900,6 +905,7 @@ "0x0000000000000000000000004fd27b205895e698fa350f7ea57cec8a21927fcd" ], "data": "0x0000000000000000000000000000000000000000000181a7ae53ea2f0bef8ccc", + "index": "0x7", "position": "0x0" } ], @@ -919,6 +925,7 @@ "0x0000000000000000000000006e715ab4f598eacf0016b9b35ef33e4141844ccc" ], "data": "0x0000000000000000000000000000000000000000000000022b1c8c12279fffff", + "index": "0x8", "position": "0x1" } ], @@ -942,6 +949,7 @@ "0x0000000000000000000000006e715ab4f598eacf0016b9b35ef33e4141844ccc" ], "data": "0x0000000000000000000000000000000000000000000000022b1c8c12279fffff", + "index": "0x9", "position": "0x1" } ], @@ -968,6 +976,7 @@ "0x0000000000000000000000006dbfc63479ffc031f23e94dc91befa38bec2c25f" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000001", + "index": "0xa", "position": "0x0" } ], @@ -982,6 +991,7 @@ "0x07cf7e805770612a8b2ee8e0bcbba8aa908df5f85fbc4f9e2ef384cf75315038" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000000", + "index": "0x4", "position": "0x6" }, { @@ -990,6 +1000,7 @@ "0x7027eecbd2a688fc1fa281702b311ed7168571514adfd17014a55d828cb43382" ], "data": "0x000000000000000000000000000000000000000000000004563918244f400000", + "index": "0x6", "position": "0x8" } ], @@ -1067,6 +1078,7 @@ "0x0000000000000000000000006dbfc63479ffc031f23e94dc91befa38bec2c25f" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000063", + "index": "0xb", "position": "0x0" } ], @@ -1195,6 +1207,7 @@ "0x000000000000000000000000da4a4626d3e16e094de3225a751aab7128e96526" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000064", + "index": "0xe", "position": "0x0" } ], @@ -1209,6 +1222,7 @@ "0x4b0bc4f25f8d0b92d2e12b686ba96cd75e4e69325e6cf7b1f3119d14eaf2cbdf" ], "data": "0x000000000000000000000000da4a4626d3e16e094de3225a751aab7128e96526", + "index": "0xc", "position": "0x6" }, { @@ -1217,6 +1231,7 @@ "0xf340c079d598119636d42046c6a2d2faf7a68c04aecee516f0e0b8a9e79b8666" ], "data": "0x000000000000000000000000da4a4626d3e16e094de3225a751aab7128e9652600000000000000000000000000000000000000000000000000000000000000640000000000000000000000000000000000000000000000000000000000000000", + "index": "0xd", "position": "0x9" } ], @@ -1267,6 +1282,7 @@ "0x0000000000000000000000007498bb5749c9801f1f7e490baf5f966dbfe4e97b" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000001", + "index": "0xf", "position": "0x0" } ], @@ -1361,6 +1377,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000001" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x10", "position": "0x2" } ], @@ -1455,6 +1472,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000002" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x11", "position": "0x2" } ], @@ -1549,6 +1567,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000003" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x12", "position": "0x2" } ], @@ -1643,6 +1662,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000004" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x13", "position": "0x2" } ], @@ -1737,6 +1757,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000005" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x14", "position": "0x2" } ], @@ -1831,6 +1852,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000006" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x15", "position": "0x2" } ], @@ -1925,6 +1947,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000007" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x16", "position": "0x2" } ], @@ -2019,6 +2042,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000008" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x17", "position": "0x2" } ], @@ -2113,6 +2137,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000009" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x18", "position": "0x2" } ], @@ -2207,6 +2232,7 @@ "0x000000000000000000000000000000000000000000000000000000000000000a" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x19", "position": "0x2" } ], @@ -2260,6 +2286,7 @@ "0x0000000000000000000000007ccbc69292c7a6d7b538c91f3b283de97906cf30" ], "data": "0x00000000000000000000000000000000000000000001010d8bfbbbe40fe7518c", + "index": "0x1a", "position": "0x0" } ], @@ -2282,6 +2309,7 @@ "0x0000000000000000000000001b9ec8ba24630b75a7a958153ffff56dd6d4b6a2" ], "data": "0x00000000000000000000000000000000000000000001010d8bfbbbe40fe7518c", + "index": "0x1c", "position": "0x0" } ], @@ -2304,6 +2332,7 @@ "0x000000000000000000000000c3a2c744ad1f5253c736875b93bacce5b01b060b" ], "data": "0x00000000000000000000000000000000000000000001010d8bfbbbe40fe7518c", + "index": "0x1e", "position": "0x0" } ], @@ -2318,6 +2347,7 @@ "0xc6d8c0af6d21f291e7c359603aa97e0ed500f04db6e983b9fce75a91c6b8da6b" ], "data": "0x00000000000000000000000000000000000000000001010d8bfbbbe40fe7518c", + "index": "0x1b", "position": "0x2" }, { @@ -2326,6 +2356,7 @@ "0xc6d8c0af6d21f291e7c359603aa97e0ed500f04db6e983b9fce75a91c6b8da6b" ], "data": "0x00000000000000000000000000000000000000000001010d8bfbbbe40fe7518c", + "index": "0x1d", "position": "0x3" }, { @@ -2334,6 +2365,7 @@ "0xc6d8c0af6d21f291e7c359603aa97e0ed500f04db6e983b9fce75a91c6b8da6b" ], "data": "0x00000000000000000000000000000000000000000001010d8bfbbbe40fe7518c", + "index": "0x1f", "position": "0x4" } ], @@ -2344,4 +2376,4 @@ "value": "0x0", "type": "CALL" } -} +} \ No newline at end of file diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/multilogs.json b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/multilogs.json index 0ca4c8353f..ad3ce19b6f 100644 --- a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/multilogs.json +++ b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/multilogs.json @@ -199,6 +199,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034100000000000000000000000000000000000000000000000000000000000000390000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffefefe0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x0", "position": "0x0" }, { @@ -207,6 +208,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff00000000000000000000000000000000000000000000000000000000000000360000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffdfdfd0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x1", "position": "0x0" }, { @@ -215,6 +217,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034100000000000000000000000000000000000000000000000000000000000000360000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffebebeb0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x2", "position": "0x0" }, { @@ -223,6 +226,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff000000000000000000000000000000000000000000000000000000000000003a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8888880000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x3", "position": "0x0" }, { @@ -231,6 +235,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000341000000000000000000000000000000000000000000000000000000000000003a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffb3b3b30000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x4", "position": "0x0" }, { @@ -239,6 +244,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff00000000000000000000000000000000000000000000000000000000000000350000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffcfcfc0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x5", "position": "0x0" }, { @@ -247,6 +253,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034100000000000000000000000000000000000000000000000000000000000000350000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x6", "position": "0x0" }, { @@ -255,6 +262,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff000000000000000000000000000000000000000000000000000000000000003b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe3e3e30000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x7", "position": "0x0" }, { @@ -263,6 +271,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000341000000000000000000000000000000000000000000000000000000000000003b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff3e3e3e0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x8", "position": "0x0" }, { @@ -271,6 +280,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff00000000000000000000000000000000000000000000000000000000000000340000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x9", "position": "0x0" }, { @@ -279,6 +289,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034100000000000000000000000000000000000000000000000000000000000000340000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0xa", "position": "0x0" }, { @@ -287,6 +298,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff000000000000000000000000000000000000000000000000000000000000003c0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0xb", "position": "0x0" }, { @@ -295,6 +307,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000341000000000000000000000000000000000000000000000000000000000000003c0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffdbdbdb0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0xc", "position": "0x0" }, { @@ -303,6 +316,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff00000000000000000000000000000000000000000000000000000000000000330000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0xd", "position": "0x0" }, { @@ -311,6 +325,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034100000000000000000000000000000000000000000000000000000000000000330000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffff4f4f40000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0xe", "position": "0x0" }, { @@ -319,6 +334,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff000000000000000000000000000000000000000000000000000000000000003d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0xf", "position": "0x0" }, { @@ -327,6 +343,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000341000000000000000000000000000000000000000000000000000000000000003d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x10", "position": "0x0" }, { @@ -335,6 +352,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff00000000000000000000000000000000000000000000000000000000000000320000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x11", "position": "0x0" }, { @@ -343,6 +361,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034100000000000000000000000000000000000000000000000000000000000000320000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffbfbfb0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x12", "position": "0x0" }, { @@ -351,6 +370,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff000000000000000000000000000000000000000000000000000000000000003e0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x13", "position": "0x0" }, { @@ -359,6 +379,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000341000000000000000000000000000000000000000000000000000000000000003e0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x14", "position": "0x0" }, { @@ -367,6 +388,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe00000000000000000000000000000000000000000000000000000000000000380000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffb0b0b00000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x15", "position": "0x0" }, { @@ -375,7 +397,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034200000000000000000000000000000000000000000000000000000000000000380000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffefefe0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x16", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -383,7 +406,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe00000000000000000000000000000000000000000000000000000000000000370000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffa0a0a00000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x17", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -391,7 +415,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034200000000000000000000000000000000000000000000000000000000000000370000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff5b5b5b0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x18", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -399,7 +424,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe00000000000000000000000000000000000000000000000000000000000000390000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffbababa0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x19", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -407,7 +433,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034200000000000000000000000000000000000000000000000000000000000000390000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x1a", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -415,7 +442,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe00000000000000000000000000000000000000000000000000000000000000360000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffeaeaea0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x1b", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -423,7 +451,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034200000000000000000000000000000000000000000000000000000000000000360000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffa9a9a90000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x1c", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -431,7 +460,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe000000000000000000000000000000000000000000000000000000000000003a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffb9b9b90000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x1d", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -439,7 +469,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000342000000000000000000000000000000000000000000000000000000000000003a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffbfbfb0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x1e", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -447,7 +478,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe00000000000000000000000000000000000000000000000000000000000000350000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffefefe0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x1f", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -455,7 +487,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034200000000000000000000000000000000000000000000000000000000000000350000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffefefe0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x20", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -463,7 +496,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe000000000000000000000000000000000000000000000000000000000000003b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffbababa0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x21", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -471,7 +505,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000342000000000000000000000000000000000000000000000000000000000000003b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff6363630000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x22", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -479,7 +514,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe00000000000000000000000000000000000000000000000000000000000000340000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x23", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -487,7 +523,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034200000000000000000000000000000000000000000000000000000000000000340000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffff9f9f90000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x24", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -495,7 +532,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe000000000000000000000000000000000000000000000000000000000000003c0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffeaeaea0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x25", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -503,7 +541,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000342000000000000000000000000000000000000000000000000000000000000003c0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff9c9c9c0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x26", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -511,7 +550,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe00000000000000000000000000000000000000000000000000000000000000330000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8f8f80000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x27", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -519,7 +559,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034200000000000000000000000000000000000000000000000000000000000000330000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x28", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -527,7 +568,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe000000000000000000000000000000000000000000000000000000000000003d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffdfdfd0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x29", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -535,7 +577,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000342000000000000000000000000000000000000000000000000000000000000003d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x2a", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -543,7 +586,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe00000000000000000000000000000000000000000000000000000000000000320000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x2b", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -551,7 +595,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034200000000000000000000000000000000000000000000000000000000000000320000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffcfcfc0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x2c", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -559,7 +604,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe000000000000000000000000000000000000000000000000000000000000003e0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffdfdfd0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x2d", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -567,7 +613,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000342000000000000000000000000000000000000000000000000000000000000003e0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x2e", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -575,7 +622,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fd00000000000000000000000000000000000000000000000000000000000000380000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff4d4e530000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x2f", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -583,7 +631,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034300000000000000000000000000000000000000000000000000000000000000380000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x30", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -591,10 +640,11 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fd00000000000000000000000000000000000000000000000000000000000000370000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff4f494b0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x31", + "position": "0x0" } ], "value": "0x3782dace9d90000", "type": "CALL" } -} +} \ No newline at end of file diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/notopic.json b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/notopic.json index de8ee41af4..58f2c4b8c3 100644 --- a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/notopic.json +++ b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/notopic.json @@ -287,6 +287,7 @@ "0xaf30e4d66b2f1f23e63ef4591058a897f67e6867233e33ca3508b982dcc4129b" ], "data": "0x00000000000000000000000050739060a2c32dc076e507ae1a893aab28ecfe68d1b13c1538a940417bf0e73b2498634436753c854c7fb971224d971bd2ae3e8800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000120000000000000000000000000000000000000000000000000000000000000016000000000000000000000000000000000000000000000000000000000000001e000000000000000000000000000000000000000000000000000000000000249f011000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000355524c000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000436a736f6e2868747470733a2f2f6170692e72616e646f6d2e6f72672f6a736f6e2d7270632f312f696e766f6b65292e726573756c742e72616e646f6d2e646174612e300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000012c4244584a68725670424a35336f3243786c4a526c51745a4a4b5a714c5974354951652b37335944533448744e6a5335486f64624942337476666f773755717579416b303835566b4c6e4c3945704b67777157517a375a4c64477673516c526432734b78496f6c4e673944626e6650737047714c684c62625953566e4e38437776736a7041586353536f33632b34634e774339307946346f4e69626b764433797461706f5a37676f5453796f5559546677536a6e773374692b484a5648374e332b633069774f43715a6a4464734751556358336d33532f494857624f4f5151356f734f344c626a33476730783155644e7466557a5943465937396e7a596757495145464375524249306e364e42764251573732372b4f73445259304a2f392f676a74387563696248576963303d0000000000000000000000000000000000000000", + "index": "0x1", "position": "0x4" } ], @@ -299,10 +300,11 @@ "address": "0x50739060a2c32dc076e507ae1a893aab28ecfe68", "topics": [], "data": "0x62616e6b726f6c6c5f6d69736d61746368", + "index": "0x0", "position": "0x2" } ], "value": "0x429d069189e0000", "type": "CALL" } -} +} \ No newline at end of file diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/simple.json b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/simple.json index 8a68061191..5e79fccabd 100644 --- a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/simple.json +++ b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/simple.json @@ -96,10 +96,11 @@ "0x000000000000000000000000dbf03b407c01e7cd3cbea99509d93f8dddc8c6fb" ], "data": "0x0000000000000000000000000000000000000000000000000000000000989680", + "index": "0x0", "position": "0x0" } ], "value": "0x0", "type": "CALL" } -} +} \ No newline at end of file diff --git a/eth/tracers/js/tracer_test.go b/eth/tracers/js/tracer_test.go index 7f376a27fc..b21e104abc 100644 --- a/eth/tracers/js/tracer_test.go +++ b/eth/tracers/js/tracer_test.go @@ -46,7 +46,7 @@ type vmContext struct { } func testCtx() *vmContext { - return &vmContext{blockCtx: vm.BlockContext{BlockNumber: big.NewInt(1), BaseFee: big.NewInt(0)}, txCtx: vm.TxContext{GasPrice: big.NewInt(100000)}} + return &vmContext{blockCtx: vm.BlockContext{BlockNumber: big.NewInt(1), BaseFee: big.NewInt(0)}, txCtx: vm.TxContext{GasPrice: uint256.NewInt(100000)}} } func runTrace(tracer *tracers.Tracer, vmctx *vmContext, chaincfg *params.ChainConfig, contractCode []byte) (json.RawMessage, error) { @@ -63,7 +63,7 @@ func runTrace(tracer *tracers.Tracer, vmctx *vmContext, chaincfg *params.ChainCo contract.Code = contractCode } - tracer.OnTxStart(evm.GetVMContext(), types.NewTx(&types.LegacyTx{Gas: gasLimit, GasPrice: vmctx.txCtx.GasPrice}), contract.Caller()) + tracer.OnTxStart(evm.GetVMContext(), types.NewTx(&types.LegacyTx{Gas: gasLimit, GasPrice: vmctx.txCtx.GasPrice.ToBig()}), contract.Caller()) tracer.OnEnter(0, byte(vm.CALL), contract.Caller(), contract.Address(), []byte{}, startGas, value.ToBig()) ret, err := evm.Run(contract, []byte{}, false) tracer.OnExit(0, ret, startGas-contract.Gas, err, true) @@ -186,7 +186,7 @@ func TestHaltBetweenSteps(t *testing.T) { Contract: vm.NewContract(common.Address{}, common.Address{}, uint256.NewInt(0), 0, nil), } evm := vm.NewEVM(vm.BlockContext{BlockNumber: big.NewInt(1)}, &dummyStatedb{}, chainConfig, vm.Config{Tracer: tracer.Hooks}) - evm.SetTxContext(vm.TxContext{GasPrice: big.NewInt(1)}) + evm.SetTxContext(vm.TxContext{GasPrice: uint256.NewInt(1)}) tracer.OnTxStart(evm.GetVMContext(), types.NewTx(&types.LegacyTx{}), common.Address{}) tracer.OnEnter(0, byte(vm.CALL), common.Address{}, common.Address{}, []byte{}, 0, big.NewInt(0)) tracer.OnOpcode(0, 0, 0, 0, scope, nil, 0, nil) @@ -210,7 +210,7 @@ func TestNoStepExec(t *testing.T) { t.Fatal(err) } evm := vm.NewEVM(vm.BlockContext{BlockNumber: big.NewInt(1)}, &dummyStatedb{}, chainConfig, vm.Config{Tracer: tracer.Hooks}) - evm.SetTxContext(vm.TxContext{GasPrice: big.NewInt(100)}) + evm.SetTxContext(vm.TxContext{GasPrice: uint256.NewInt(100)}) tracer.OnTxStart(evm.GetVMContext(), types.NewTx(&types.LegacyTx{}), common.Address{}) tracer.OnEnter(0, byte(vm.CALL), common.Address{}, common.Address{}, []byte{}, 1000, big.NewInt(0)) tracer.OnExit(0, nil, 0, nil, false) @@ -240,7 +240,7 @@ func TestIsPrecompile(t *testing.T) { chaincfg.ByzantiumBlock = big.NewInt(100) chaincfg.IstanbulBlock = big.NewInt(200) chaincfg.BerlinBlock = big.NewInt(300) - txCtx := vm.TxContext{GasPrice: big.NewInt(100000)} + txCtx := vm.TxContext{GasPrice: uint256.NewInt(100000)} tracer, err := newJsTracer("{addr: toAddress('0000000000000000000000000000000000000009'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil, nil, chaincfg) if err != nil { t.Fatal(err) diff --git a/eth/tracers/logger/access_list_tracer.go b/eth/tracers/logger/access_list_tracer.go index 0d51f40522..749aade61b 100644 --- a/eth/tracers/logger/access_list_tracer.go +++ b/eth/tracers/logger/access_list_tracer.go @@ -18,6 +18,7 @@ package logger import ( "maps" + "slices" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/tracing" @@ -84,12 +85,17 @@ func (al accessList) equal(other accessList) bool { func (al accessList) accessList() types.AccessList { acl := make(types.AccessList, 0, len(al)) for addr, slots := range al { - tuple := types.AccessTuple{Address: addr, StorageKeys: []common.Hash{}} - for slot := range slots { - tuple.StorageKeys = append(tuple.StorageKeys, slot) + keys := slices.SortedFunc(maps.Keys(slots), common.Hash.Cmp) + // Ensure keys is never nil to avoid JSON serialization issues. + // When slots is empty, slices.SortedFunc returns nil, but JSON marshaling + // will serialize nil slice as null instead of [], which breaks clients + // that expect storageKeys to always be an array. + if keys == nil { + keys = []common.Hash{} } - acl = append(acl, tuple) + acl = append(acl, types.AccessTuple{Address: addr, StorageKeys: keys}) } + slices.SortFunc(acl, func(a, b types.AccessTuple) int { return a.Address.Cmp(b.Address) }) return acl } diff --git a/eth/tracers/logger/logger.go b/eth/tracers/logger/logger.go index 824a5e0c3e..67e07f78d0 100644 --- a/eth/tracers/logger/logger.go +++ b/eth/tracers/logger/logger.go @@ -94,8 +94,8 @@ func (s *StructLog) ErrorString() string { return "" } -// WriteTo writes the human-readable log data into the supplied writer. -func (s *StructLog) WriteTo(writer io.Writer) { +// Write writes the human-readable log data into the supplied writer. +func (s *StructLog) Write(writer io.Writer) { fmt.Fprintf(writer, "%-16spc=%08d gas=%v cost=%v", s.Op, s.Pc, s.Gas, s.GasCost) if s.Err != nil { fmt.Fprintf(writer, " ERROR: %v", s.Err) @@ -324,7 +324,7 @@ func (l *StructLogger) OnOpcode(pc uint64, opcode byte, gas, cost uint64, scope l.logs = append(l.logs, entry) return } - log.WriteTo(l.writer) + log.Write(l.writer) } // OnExit is called a call frame finishes processing. @@ -405,7 +405,7 @@ func (l *StructLogger) Output() []byte { return l.output } // @deprecated func WriteTrace(writer io.Writer, logs []StructLog) { for _, log := range logs { - log.WriteTo(writer) + log.Write(writer) } } diff --git a/eth/tracers/native/call.go b/eth/tracers/native/call.go index 5ab7ef6ede..e78d27ef36 100644 --- a/eth/tracers/native/call.go +++ b/eth/tracers/native/call.go @@ -42,6 +42,7 @@ type callLog struct { Address common.Address `json:"address"` Topics []common.Hash `json:"topics"` Data hexutil.Bytes `json:"data"` + Index hexutil.Uint `json:"index"` // Position of the log relative to subcalls within the same trace // See https://github.com/ethereum/go-ethereum/pull/28389 for details Position hexutil.Uint `json:"position"` @@ -265,6 +266,7 @@ func (t *callTracer) OnLog(log *types.Log) { Address: log.Address, Topics: log.Topics, Data: log.Data, + Index: hexutil.Uint(log.Index), Position: hexutil.Uint(len(t.callstack[len(t.callstack)-1].Calls)), } t.callstack[len(t.callstack)-1].Logs = append(t.callstack[len(t.callstack)-1].Logs, l) diff --git a/eth/tracers/native/erc7562.go b/eth/tracers/native/erc7562.go index 3ab98c7132..34e202f667 100644 --- a/eth/tracers/native/erc7562.go +++ b/eth/tracers/native/erc7562.go @@ -513,7 +513,7 @@ func defaultIgnoredOpcodes() []hexutil.Uint64 { ignored := make([]hexutil.Uint64, 0, 64) // Allow all PUSHx, DUPx and SWAPx opcodes as they have sequential codes - for op := vm.PUSH0; op < vm.SWAP16; op++ { + for op := vm.PUSH0; op <= vm.SWAP16; op++ { ignored = append(ignored, hexutil.Uint64(op)) } diff --git a/eth/tracers/native/mux.go b/eth/tracers/native/mux.go index 7100a2ddc9..709085cf67 100644 --- a/eth/tracers/native/mux.go +++ b/eth/tracers/native/mux.go @@ -28,7 +28,7 @@ import ( ) func init() { - tracers.DefaultDirectory.Register("muxTracer", newMuxTracer, false) + tracers.DefaultDirectory.Register("muxTracer", newMuxTracerFromConfig, false) } // muxTracer is a go implementation of the Tracer interface which @@ -38,8 +38,8 @@ type muxTracer struct { tracers []*tracers.Tracer } -// newMuxTracer returns a new mux tracer. -func newMuxTracer(ctx *tracers.Context, cfg json.RawMessage, chainConfig *params.ChainConfig) (*tracers.Tracer, error) { +// newMuxTracerFromConfig returns a new mux tracer. +func newMuxTracerFromConfig(ctx *tracers.Context, cfg json.RawMessage, chainConfig *params.ChainConfig) (*tracers.Tracer, error) { var config map[string]json.RawMessage if err := json.Unmarshal(cfg, &config); err != nil { return nil, err @@ -54,7 +54,16 @@ func newMuxTracer(ctx *tracers.Context, cfg json.RawMessage, chainConfig *params objects = append(objects, t) names = append(names, k) } + return NewMuxTracer(names, objects) +} +// NewMuxTracer creates a multiplexing tracer that fans out tracing hooks to +// multiple child tracers. Each hook invocation is forwarded to all children, +// in the order they are provided. +// +// The names parameter associates a label with each tracer, used as keys in +// the aggregated JSON result returned by GetResult. +func NewMuxTracer(names []string, objects []*tracers.Tracer) (*tracers.Tracer, error) { t := &muxTracer{names: names, tracers: objects} return &tracers.Tracer{ Hooks: &tracing.Hooks{ @@ -160,6 +169,14 @@ func (t *muxTracer) OnCodeChange(a common.Address, prevCodeHash common.Hash, pre } } +func (t *muxTracer) OnCodeChangeV2(a common.Address, prevCodeHash common.Hash, prev []byte, codeHash common.Hash, code []byte, reason tracing.CodeChangeReason) { + for _, t := range t.tracers { + if t.OnCodeChangeV2 != nil { + t.OnCodeChangeV2(a, prevCodeHash, prev, codeHash, code, reason) + } + } +} + func (t *muxTracer) OnStorageChange(a common.Address, k, prev, new common.Hash) { for _, t := range t.tracers { if t.OnStorageChange != nil { diff --git a/eth/tracers/native/opcode_counter.go b/eth/tracers/native/opcode_counter.go new file mode 100644 index 0000000000..d859b275f0 --- /dev/null +++ b/eth/tracers/native/opcode_counter.go @@ -0,0 +1,55 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package native + +import ( + "encoding/json" + + "github.com/ethereum/go-ethereum/core/tracing" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/eth/tracers" +) + +// opcodeCounter is a simple tracer that counts how many times each opcode is executed. +type opcodeCounter struct { + counts [256]uint64 +} + +// NewOpcodeCounter returns a new opcodeCounter tracer. +func NewOpcodeCounter() *tracers.Tracer { + c := &opcodeCounter{} + return &tracers.Tracer{ + Hooks: &tracing.Hooks{ + OnOpcode: func(pc uint64, op byte, gas, cost uint64, scope tracing.OpContext, rData []byte, depth int, err error) { + c.counts[op]++ + }, + }, + GetResult: c.getResult, + Stop: func(err error) {}, + } +} + +// getResult returns the opcode counts keyed by opcode name. +func (c *opcodeCounter) getResult() (json.RawMessage, error) { + out := make(map[string]uint64) + for op, count := range c.counts { + if count != 0 { + out[vm.OpCode(op).String()] = count + } + } + return json.Marshal(out) +} diff --git a/eth/tracers/native/prestate.go b/eth/tracers/native/prestate.go index abd2e4e7ce..6ff5b4f7b4 100644 --- a/eth/tracers/native/prestate.go +++ b/eth/tracers/native/prestate.go @@ -66,7 +66,7 @@ type prestateTracer struct { pre stateMap post stateMap to common.Address - config prestateTracerConfig + config PrestateTracerConfig chainConfig *params.ChainConfig interrupt atomic.Bool // Atomic flag to signal execution interruption reason error // Textual reason for the interruption @@ -74,7 +74,7 @@ type prestateTracer struct { deleted map[common.Address]bool } -type prestateTracerConfig struct { +type PrestateTracerConfig struct { DiffMode bool `json:"diffMode"` // If true, this tracer will return state modifications DisableCode bool `json:"disableCode"` // If true, this tracer will not return the contract code DisableStorage bool `json:"disableStorage"` // If true, this tracer will not return the contract storage @@ -82,7 +82,7 @@ type prestateTracerConfig struct { } func newPrestateTracer(ctx *tracers.Context, cfg json.RawMessage, chainConfig *params.ChainConfig) (*tracers.Tracer, error) { - var config prestateTracerConfig + var config PrestateTracerConfig if err := json.Unmarshal(cfg, &config); err != nil { return nil, err } diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index 5401a0536c..18e234e215 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -90,7 +90,7 @@ func BenchmarkTransactionTraceV2(b *testing.B) { evm.Config.Tracer = tracer snap := state.StateDB.Snapshot() - _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) + _, err := core.ApplyMessage(evm, msg, nil) if err != nil { b.Fatal(err) } diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 2c2fbf4550..1a6640f7f7 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -124,7 +124,7 @@ func (ec *Client) PeerCount(ctx context.Context) (uint64, error) { // BlockReceipts returns the receipts of a given block number or hash. func (ec *Client) BlockReceipts(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) ([]*types.Receipt, error) { var r []*types.Receipt - err := ec.c.CallContext(ctx, &r, "eth_getBlockReceipts", blockNrOrHash.String()) + err := ec.c.CallContext(ctx, &r, "eth_getBlockReceipts", blockNrOrHash) if err == nil && r == nil { return nil, ethereum.NotFound } @@ -518,9 +518,12 @@ func (ec *Client) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuer } func toFilterArg(q ethereum.FilterQuery) (interface{}, error) { - arg := map[string]interface{}{ - "address": q.Addresses, - "topics": q.Topics, + arg := map[string]interface{}{} + if q.Addresses != nil { + arg["address"] = q.Addresses + } + if q.Topics != nil { + arg["topics"] = q.Topics } if q.BlockHash != nil { arg["blockHash"] = *q.BlockHash @@ -747,9 +750,11 @@ func (ec *Client) SendRawTransactionSync( rawTx []byte, timeout *time.Duration, ) (*types.Receipt, error) { - var ms *hexutil.Uint64 + var ms *uint64 if timeout != nil { - if d := hexutil.Uint64(timeout.Milliseconds()); d > 0 { + msInt := timeout.Milliseconds() + if msInt > 0 { + d := uint64(msInt) ms = &d } } diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index 302ccf2e16..f9e761e412 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -687,6 +687,49 @@ func testTransactionSender(t *testing.T, client *rpc.Client) { } } +func TestBlockReceiptsPreservesCanonicalFlag(t *testing.T) { + srv := rpc.NewServer() + service := &blockReceiptsTestService{calls: make(chan rpc.BlockNumberOrHash, 1)} + if err := srv.RegisterName("eth", service); err != nil { + t.Fatalf("failed to register service: %v", err) + } + defer srv.Stop() + + client := rpc.DialInProc(srv) + defer client.Close() + + ec := ethclient.NewClient(client) + defer ec.Close() + + hash := common.HexToHash("0x01") + ref := rpc.BlockNumberOrHashWithHash(hash, true) + + if _, err := ec.BlockReceipts(context.Background(), ref); err != nil { + t.Fatalf("BlockReceipts returned error: %v", err) + } + + select { + case call := <-service.calls: + if call.BlockHash == nil || *call.BlockHash != hash { + t.Fatalf("unexpected block hash: got %v, want %v", call.BlockHash, hash) + } + if !call.RequireCanonical { + t.Fatalf("requireCanonical flag was lost: %+v", call) + } + default: + t.Fatal("service was not called") + } +} + +type blockReceiptsTestService struct { + calls chan rpc.BlockNumberOrHash +} + +func (s *blockReceiptsTestService) GetBlockReceipts(ctx context.Context, block rpc.BlockNumberOrHash) ([]*types.Receipt, error) { + s.calls <- block + return []*types.Receipt{}, nil +} + func newCanceledContext() context.Context { ctx, cancel := context.WithCancel(context.Background()) cancel() diff --git a/ethclient/gethclient/gen_callframe_json.go b/ethclient/gethclient/gen_callframe_json.go new file mode 100644 index 0000000000..48df2790cc --- /dev/null +++ b/ethclient/gethclient/gen_callframe_json.go @@ -0,0 +1,104 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package gethclient + +import ( + "encoding/json" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" +) + +var _ = (*callFrameMarshaling)(nil) + +// MarshalJSON marshals as JSON. +func (c CallFrame) MarshalJSON() ([]byte, error) { + type CallFrame0 struct { + Type string `json:"type"` + From common.Address `json:"from"` + Gas hexutil.Uint64 `json:"gas"` + GasUsed hexutil.Uint64 `json:"gasUsed"` + To *common.Address `json:"to,omitempty"` + Input hexutil.Bytes `json:"input"` + Output hexutil.Bytes `json:"output,omitempty"` + Error string `json:"error,omitempty"` + RevertReason string `json:"revertReason,omitempty"` + Calls []CallFrame `json:"calls,omitempty"` + Logs []CallLog `json:"logs,omitempty"` + Value *hexutil.Big `json:"value,omitempty"` + } + var enc CallFrame0 + enc.Type = c.Type + enc.From = c.From + enc.Gas = hexutil.Uint64(c.Gas) + enc.GasUsed = hexutil.Uint64(c.GasUsed) + enc.To = c.To + enc.Input = c.Input + enc.Output = c.Output + enc.Error = c.Error + enc.RevertReason = c.RevertReason + enc.Calls = c.Calls + enc.Logs = c.Logs + enc.Value = (*hexutil.Big)(c.Value) + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (c *CallFrame) UnmarshalJSON(input []byte) error { + type CallFrame0 struct { + Type *string `json:"type"` + From *common.Address `json:"from"` + Gas *hexutil.Uint64 `json:"gas"` + GasUsed *hexutil.Uint64 `json:"gasUsed"` + To *common.Address `json:"to,omitempty"` + Input *hexutil.Bytes `json:"input"` + Output *hexutil.Bytes `json:"output,omitempty"` + Error *string `json:"error,omitempty"` + RevertReason *string `json:"revertReason,omitempty"` + Calls []CallFrame `json:"calls,omitempty"` + Logs []CallLog `json:"logs,omitempty"` + Value *hexutil.Big `json:"value,omitempty"` + } + var dec CallFrame0 + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.Type != nil { + c.Type = *dec.Type + } + if dec.From != nil { + c.From = *dec.From + } + if dec.Gas != nil { + c.Gas = uint64(*dec.Gas) + } + if dec.GasUsed != nil { + c.GasUsed = uint64(*dec.GasUsed) + } + if dec.To != nil { + c.To = dec.To + } + if dec.Input != nil { + c.Input = *dec.Input + } + if dec.Output != nil { + c.Output = *dec.Output + } + if dec.Error != nil { + c.Error = *dec.Error + } + if dec.RevertReason != nil { + c.RevertReason = *dec.RevertReason + } + if dec.Calls != nil { + c.Calls = dec.Calls + } + if dec.Logs != nil { + c.Logs = dec.Logs + } + if dec.Value != nil { + c.Value = (*big.Int)(dec.Value) + } + return nil +} diff --git a/ethclient/gethclient/gen_calllog_json.go b/ethclient/gethclient/gen_calllog_json.go new file mode 100644 index 0000000000..50e25d4bb3 --- /dev/null +++ b/ethclient/gethclient/gen_calllog_json.go @@ -0,0 +1,61 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package gethclient + +import ( + "encoding/json" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" +) + +var _ = (*callLogMarshaling)(nil) + +// MarshalJSON marshals as JSON. +func (c CallLog) MarshalJSON() ([]byte, error) { + type CallLog struct { + Address common.Address `json:"address"` + Topics []common.Hash `json:"topics"` + Data hexutil.Bytes `json:"data"` + Index hexutil.Uint `json:"index"` + Position hexutil.Uint `json:"position"` + } + var enc CallLog + enc.Address = c.Address + enc.Topics = c.Topics + enc.Data = c.Data + enc.Index = hexutil.Uint(c.Index) + enc.Position = hexutil.Uint(c.Position) + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (c *CallLog) UnmarshalJSON(input []byte) error { + type CallLog struct { + Address *common.Address `json:"address"` + Topics []common.Hash `json:"topics"` + Data *hexutil.Bytes `json:"data"` + Index *hexutil.Uint `json:"index"` + Position *hexutil.Uint `json:"position"` + } + var dec CallLog + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.Address != nil { + c.Address = *dec.Address + } + if dec.Topics != nil { + c.Topics = dec.Topics + } + if dec.Data != nil { + c.Data = *dec.Data + } + if dec.Index != nil { + c.Index = uint(*dec.Index) + } + if dec.Position != nil { + c.Position = uint(*dec.Position) + } + return nil +} diff --git a/ethclient/gethclient/gethclient.go b/ethclient/gethclient/gethclient.go index 6a0f5eb312..e677e2bb21 100644 --- a/ethclient/gethclient/gethclient.go +++ b/ethclient/gethclient/gethclient.go @@ -19,10 +19,12 @@ package gethclient import ( "context" + "encoding/json" "fmt" "math/big" "runtime" "runtime/debug" + "time" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" @@ -104,7 +106,10 @@ func (ec *Client) GetProof(ctx context.Context, account common.Address, keys []s var res accountResult err := ec.c.CallContext(ctx, &res, "eth_getProof", account, keys, toBlockNumArg(blockNumber)) - // Turn hexutils back to normal datatypes + if err != nil { + return nil, err + } + // Turn hexutils back to normal data types storageResults := make([]StorageResult, 0, len(res.StorageProof)) for _, st := range res.StorageProof { storageResults = append(storageResults, StorageResult{ @@ -122,7 +127,7 @@ func (ec *Client) GetProof(ctx context.Context, account common.Address, keys []s StorageHash: res.StorageHash, StorageProof: storageResults, } - return &result, err + return &result, nil } // CallContract executes a message call transaction, which is directly executed in the VM @@ -226,6 +231,124 @@ func (ec *Client) TraceBlock(ctx context.Context, hash common.Hash, config *trac return result, nil } +// CallTracerConfig configures the call tracer for +// TraceTransactionWithCallTracer and TraceCallWithCallTracer. +type CallTracerConfig struct { + // OnlyTopCall, when true, limits tracing to the main (top-level) call only. + OnlyTopCall bool + // WithLog, when true, includes log emissions in the trace output. + WithLog bool + // Timeout is the maximum duration the tracer may run. + // Zero means the server default (5s). + Timeout time.Duration +} + +//go:generate go run github.com/fjl/gencodec -type CallLog -field-override callLogMarshaling -out gen_calllog_json.go + +// CallLog represents a log emitted during a traced call. +type CallLog struct { + Address common.Address `json:"address"` + Topics []common.Hash `json:"topics"` + Data []byte `json:"data"` + Index uint `json:"index"` + Position uint `json:"position"` +} + +type callLogMarshaling struct { + Data hexutil.Bytes + Index hexutil.Uint + Position hexutil.Uint +} + +//go:generate go run github.com/fjl/gencodec -type CallFrame -field-override callFrameMarshaling -out gen_callframe_json.go + +// CallFrame contains the result of a call tracer run. +type CallFrame struct { + Type string `json:"type"` + From common.Address `json:"from"` + Gas uint64 `json:"gas"` + GasUsed uint64 `json:"gasUsed"` + To *common.Address `json:"to,omitempty"` + Input []byte `json:"input"` + Output []byte `json:"output,omitempty"` + Error string `json:"error,omitempty"` + RevertReason string `json:"revertReason,omitempty"` + Calls []CallFrame `json:"calls,omitempty"` + Logs []CallLog `json:"logs,omitempty"` + Value *big.Int `json:"value,omitempty"` +} + +type callFrameMarshaling struct { + Gas hexutil.Uint64 + GasUsed hexutil.Uint64 + Input hexutil.Bytes + Output hexutil.Bytes + Value *hexutil.Big +} + +// TraceTransactionWithCallTracer traces a transaction with the call tracer +// and returns a typed CallFrame. If config is nil, defaults are used. +func (ec *Client) TraceTransactionWithCallTracer(ctx context.Context, txHash common.Hash, config *CallTracerConfig) (*CallFrame, error) { + var result CallFrame + err := ec.c.CallContext(ctx, &result, "debug_traceTransaction", txHash, callTracerConfig(config)) + if err != nil { + return nil, err + } + return &result, nil +} + +// TraceCallWithCallTracer executes a call with the call tracer and returns +// a typed CallFrame. blockNrOrHash selects the block context for the call. +// overrides specifies state overrides (nil for none), blockOverrides specifies +// block header overrides (nil for none), and config configures the tracer +// (nil for defaults). +func (ec *Client) TraceCallWithCallTracer(ctx context.Context, msg ethereum.CallMsg, blockNrOrHash rpc.BlockNumberOrHash, overrides map[common.Address]OverrideAccount, blockOverrides *BlockOverrides, config *CallTracerConfig) (*CallFrame, error) { + var result CallFrame + err := ec.c.CallContext(ctx, &result, "debug_traceCall", toCallArg(msg), blockNrOrHash, callTraceCallConfig(config, overrides, blockOverrides)) + if err != nil { + return nil, err + } + return &result, nil +} + +// callTracerConfig converts a CallTracerConfig to the wire-format TraceConfig. +func callTracerConfig(config *CallTracerConfig) *tracers.TraceConfig { + tracer := "callTracer" + tc := &tracers.TraceConfig{Tracer: &tracer} + if config != nil { + if config.OnlyTopCall || config.WithLog { + cfg, _ := json.Marshal(struct { + OnlyTopCall bool `json:"onlyTopCall"` + WithLog bool `json:"withLog"` + }{config.OnlyTopCall, config.WithLog}) + tc.TracerConfig = cfg + } + if config.Timeout != 0 { + s := config.Timeout.String() + tc.Timeout = &s + } + } + return tc +} + +// callTraceCallConfig builds the wire-format TraceCallConfig for debug_traceCall, +// bundling tracer settings with optional state and block overrides. +func callTraceCallConfig(config *CallTracerConfig, overrides map[common.Address]OverrideAccount, blockOverrides *BlockOverrides) interface{} { + tc := callTracerConfig(config) + // debug_traceCall expects a single config object that includes both + // tracer settings and any state/block overrides. + type traceCallConfig struct { + *tracers.TraceConfig + StateOverrides map[common.Address]OverrideAccount `json:"stateOverrides,omitempty"` + BlockOverrides *BlockOverrides `json:"blockOverrides,omitempty"` + } + return &traceCallConfig{ + TraceConfig: tc, + StateOverrides: overrides, + BlockOverrides: blockOverrides, + } +} + func toBlockNumArg(number *big.Int) string { if number == nil { return "latest" diff --git a/ethclient/gethclient/gethclient_test.go b/ethclient/gethclient/gethclient_test.go index 0eed63cacf..4d8ccfcb6f 100644 --- a/ethclient/gethclient/gethclient_test.go +++ b/ethclient/gethclient/gethclient_test.go @@ -34,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/tracers" + _ "github.com/ethereum/go-ethereum/eth/tracers/native" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" @@ -161,6 +162,12 @@ func TestGethClient(t *testing.T) { }, { "TestCallContractWithBlockOverrides", func(t *testing.T) { testCallContractWithBlockOverrides(t, client) }, + }, { + "TestTraceTransactionWithCallTracer", + func(t *testing.T) { testTraceTransactionWithCallTracer(t, client, txHashes) }, + }, { + "TestTraceCallWithCallTracer", + func(t *testing.T) { testTraceCallWithCallTracer(t, client) }, }, // The testaccesslist is a bit time-sensitive: the newTestBackend imports // one block. The `testAccessList` fails if the miner has not yet created a @@ -620,3 +627,60 @@ func testCallContractWithBlockOverrides(t *testing.T, client *rpc.Client) { t.Fatalf("unexpected result: %x", res) } } + +func testTraceTransactionWithCallTracer(t *testing.T, client *rpc.Client, txHashes []common.Hash) { + ec := New(client) + for _, txHash := range txHashes { + // With nil config (defaults). + result, err := ec.TraceTransactionWithCallTracer(context.Background(), txHash, nil) + if err != nil { + t.Fatalf("nil config: %v", err) + } + if result.Type != "CALL" { + t.Fatalf("unexpected type: %s", result.Type) + } + if result.From == (common.Address{}) { + t.Fatal("from is zero") + } + if result.Gas == 0 { + t.Fatal("gas is zero") + } + + // With explicit config. + result, err = ec.TraceTransactionWithCallTracer(context.Background(), txHash, + &CallTracerConfig{}, + ) + if err != nil { + t.Fatalf("explicit config: %v", err) + } + if result.Type != "CALL" { + t.Fatalf("unexpected type: %s", result.Type) + } + } +} + +func testTraceCallWithCallTracer(t *testing.T, client *rpc.Client) { + ec := New(client) + msg := ethereum.CallMsg{ + From: testAddr, + To: &common.Address{}, + Gas: 21000, + GasPrice: big.NewInt(1000000000), + Value: big.NewInt(1), + } + result, err := ec.TraceCallWithCallTracer(context.Background(), msg, + rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber), nil, nil, nil, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.Type != "CALL" { + t.Fatalf("unexpected type: %s", result.Type) + } + if result.From == (common.Address{}) { + t.Fatal("from is zero") + } + if result.Gas == 0 { + t.Fatal("gas is zero") + } +} diff --git a/ethclient/types_test.go b/ethclient/types_test.go index 02f9f21758..dcb9a579b7 100644 --- a/ethclient/types_test.go +++ b/ethclient/types_test.go @@ -41,6 +41,18 @@ func TestToFilterArg(t *testing.T) { output interface{} err error }{ + { + "without addresses", + ethereum.FilterQuery{ + FromBlock: big.NewInt(1), + ToBlock: big.NewInt(2), + }, + map[string]interface{}{ + "fromBlock": "0x1", + "toBlock": "0x2", + }, + nil, + }, { "without BlockHash", ethereum.FilterQuery{ diff --git a/ethdb/batch.go b/ethdb/batch.go index a2dd22078e..cfca592bb6 100644 --- a/ethdb/batch.go +++ b/ethdb/batch.go @@ -41,6 +41,9 @@ type Batch interface { // Replay replays the batch contents. Replay(w KeyValueWriter) error + + // Close closes the batch and releases all associated resources. + Close() } // Batcher wraps the NewBatch method of a backing data store. diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index 8cd3c78890..364ac7ed0a 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -518,6 +518,9 @@ func (b *batch) Replay(w ethdb.KeyValueWriter) error { return b.b.Replay(&replayer{writer: w}) } +// Close closes the batch and releases all associated resources. +func (b *batch) Close() {} + // replayer is a small wrapper to implement the correct replay methods. type replayer struct { writer ethdb.KeyValueWriter diff --git a/ethdb/memorydb/memorydb.go b/ethdb/memorydb/memorydb.go index cfbad77816..a62e76cf00 100644 --- a/ethdb/memorydb/memorydb.go +++ b/ethdb/memorydb/memorydb.go @@ -338,6 +338,9 @@ func (b *batch) Replay(w ethdb.KeyValueWriter) error { return nil } +// Close closes the batch and releases all associated resources. +func (b *batch) Close() {} + // iterator can walk over the (potentially partial) keyspace of a memory key // value store. Internally it is a deep copy of the entire iterated state, // sorted by keys. diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index 7502a6877a..2180c272d3 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -229,6 +229,8 @@ func New(file string, cache int, handles int, namespace string, readonly bool, e {TargetFileSize: 16 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, {TargetFileSize: 32 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, {TargetFileSize: 64 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, + + // Pebble doesn't use the Bloom filter at level6 for read efficiency. {TargetFileSize: 128 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, } } else { @@ -269,8 +271,8 @@ func New(file string, cache int, handles int, namespace string, readonly bool, e // limit unchanged allows writes to be flushed more smoothly. This helps // avoid compaction spikes and mitigates write stalls caused by heavy // compaction workloads. - memTableLimit := extraOptions.MemTableStopWritesThreshold - memTableSize := cache * 1024 * 1024 / 2 / memTableLimit + memTableNumber := extraOptions.MemTableStopWritesThreshold + memTableSize := cache * 1024 * 1024 / 2 / memTableNumber // The memory table size is currently capped at maxMemTableSize-1 due to a // known bug in the pebble where maxMemTableSize is not recognized as a @@ -299,12 +301,16 @@ func New(file string, cache int, handles int, namespace string, readonly bool, e // Note, there may have more than two memory tables in the system. MemTableSize: uint64(memTableSize), - // MemTableStopWritesThreshold places a hard limit on the size + // MemTableStopWritesThreshold places a hard limit on the number // of the existent MemTables(including the frozen one). + // // Note, this must be the number of tables not the size of all memtables // according to https://github.com/cockroachdb/pebble/blob/master/options.go#L738-L742 // and to https://github.com/cockroachdb/pebble/blob/master/db.go#L1892-L1903. - MemTableStopWritesThreshold: memTableLimit, + // + // MemTableStopWritesThreshold is set to twice the maximum number of + // allowed memtables to accommodate temporary spikes. + MemTableStopWritesThreshold: memTableNumber * 2, // The default compaction concurrency(1 thread), // Here use all available CPUs for faster compaction. @@ -852,6 +858,12 @@ func (b *batch) Replay(w ethdb.KeyValueWriter) error { } } +// Close closes the batch and releases all associated resources. After it is +// closed, any subsequent operations on this batch are undefined. +func (b *batch) Close() { + b.b.Close() +} + // pebbleIterator is a wrapper of underlying iterator in storage engine. // The purpose of this structure is to implement the missing APIs. // diff --git a/ethstats/ethstats.go b/ethstats/ethstats.go index b6191baa12..c17e225165 100644 --- a/ethstats/ethstats.go +++ b/ethstats/ethstats.go @@ -63,6 +63,7 @@ const ( type backend interface { SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription + SubscribeNewPayloadEvent(ch chan<- core.NewPayloadEvent) event.Subscription CurrentHeader() *types.Header HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error) Stats() (pending int, queued int) @@ -92,8 +93,9 @@ type Service struct { pongCh chan struct{} // Pong notifications are fed into this channel histCh chan []uint64 // History request block numbers are fed into this channel - headSub event.Subscription - txSub event.Subscription + headSub event.Subscription + txSub event.Subscription + newPayloadSub event.Subscription } // connWrapper is a wrapper to prevent concurrent-write or concurrent-read on the @@ -198,7 +200,9 @@ func (s *Service) Start() error { s.headSub = s.backend.SubscribeChainHeadEvent(chainHeadCh) txEventCh := make(chan core.NewTxsEvent, txChanSize) s.txSub = s.backend.SubscribeNewTxsEvent(txEventCh) - go s.loop(chainHeadCh, txEventCh) + newPayloadCh := make(chan core.NewPayloadEvent, chainHeadChanSize) + s.newPayloadSub = s.backend.SubscribeNewPayloadEvent(newPayloadCh) + go s.loop(chainHeadCh, txEventCh, newPayloadCh) log.Info("Stats daemon started") return nil @@ -208,18 +212,20 @@ func (s *Service) Start() error { func (s *Service) Stop() error { s.headSub.Unsubscribe() s.txSub.Unsubscribe() + s.newPayloadSub.Unsubscribe() log.Info("Stats daemon stopped") return nil } // loop keeps trying to connect to the netstats server, reporting chain events // until termination. -func (s *Service) loop(chainHeadCh chan core.ChainHeadEvent, txEventCh chan core.NewTxsEvent) { +func (s *Service) loop(chainHeadCh chan core.ChainHeadEvent, txEventCh chan core.NewTxsEvent, newPayloadCh chan core.NewPayloadEvent) { // Start a goroutine that exhausts the subscriptions to avoid events piling up var ( - quitCh = make(chan struct{}) - headCh = make(chan *types.Header, 1) - txCh = make(chan struct{}, 1) + quitCh = make(chan struct{}) + headCh = make(chan *types.Header, 1) + txCh = make(chan struct{}, 1) + newPayloadEvCh = make(chan core.NewPayloadEvent, 1) ) go func() { var lastTx mclock.AbsTime @@ -246,11 +252,20 @@ func (s *Service) loop(chainHeadCh chan core.ChainHeadEvent, txEventCh chan core default: } + // Notify of new payload events, but drop if too frequent + case ev := <-newPayloadCh: + select { + case newPayloadEvCh <- ev: + default: + } + // node stopped case <-s.txSub.Err(): break HandleLoop case <-s.headSub.Err(): break HandleLoop + case <-s.newPayloadSub.Err(): + break HandleLoop } } close(quitCh) @@ -336,6 +351,10 @@ func (s *Service) loop(chainHeadCh chan core.ChainHeadEvent, txEventCh chan core if err = s.reportPending(conn); err != nil { log.Warn("Post-block transaction stats report failed", "err", err) } + case ev := <-newPayloadEvCh: + if err = s.reportNewPayload(conn, ev); err != nil { + log.Warn("New payload stats report failed", "err", err) + } case <-txCh: if err = s.reportPending(conn); err != nil { log.Warn("Transaction stats report failed", "err", err) @@ -600,6 +619,33 @@ func (s uncleStats) MarshalJSON() ([]byte, error) { return []byte("[]"), nil } +// newPayloadStats is the information to report about new payload events. +type newPayloadStats struct { + Number uint64 `json:"number"` + Hash common.Hash `json:"hash"` + ProcessingTime uint64 `json:"processingTime"` // nanoseconds +} + +// reportNewPayload reports a new payload event to the stats server. +func (s *Service) reportNewPayload(conn *connWrapper, ev core.NewPayloadEvent) error { + details := &newPayloadStats{ + Number: ev.Number, + Hash: ev.Hash, + ProcessingTime: uint64(ev.ProcessingTime.Nanoseconds()), + } + + log.Trace("Sending new payload to ethstats", "number", details.Number, "hash", details.Hash) + + stats := map[string]interface{}{ + "id": s.node, + "block": details, + } + report := map[string][]interface{}{ + "emit": {"block_new_payload", stats}, + } + return conn.WriteJSON(report) +} + // reportBlock retrieves the current chain head and reports it to the stats server. func (s *Service) reportBlock(conn *connWrapper, header *types.Header) error { // Gather the block details from the header or block chain diff --git a/go.mod b/go.mod index b4dc9a281a..ab89f78505 100644 --- a/go.mod +++ b/go.mod @@ -14,18 +14,16 @@ require ( github.com/cespare/cp v0.1.0 github.com/cloudflare/cloudflare-go v0.114.0 github.com/cockroachdb/pebble v1.1.5 - github.com/consensys/gnark-crypto v0.18.0 - github.com/crate-crypto/go-eth-kzg v1.4.0 - github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a + github.com/consensys/gnark-crypto v0.18.1 + github.com/crate-crypto/go-eth-kzg v1.5.0 github.com/davecgh/go-spew v1.1.1 github.com/dchest/siphash v1.2.3 github.com/deckarep/golang-set/v2 v2.6.0 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 - github.com/ethereum/c-kzg-4844/v2 v2.1.5 + github.com/ethereum/c-kzg-4844/v2 v2.1.6 github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab - github.com/ethereum/go-verkle v0.2.2 github.com/fatih/color v1.16.0 github.com/ferranbt/fastssz v0.1.4 github.com/fsnotify/fsnotify v1.6.0 @@ -35,7 +33,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.2 github.com/golang/snappy v1.0.0 github.com/google/gofuzz v1.2.0 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.4.2 github.com/graph-gophers/graphql-go v1.3.0 github.com/hashicorp/go-bexpr v0.1.10 @@ -47,12 +45,12 @@ require ( github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c github.com/jackpal/go-nat-pmp v1.0.2 github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 - github.com/karalabe/hid v1.0.1-0.20240306101548-573246063e52 + github.com/karalabe/hid v1.0.1-0.20260315100226-f5d04adeffeb + github.com/klauspost/compress v1.17.8 github.com/kylelemons/godebug v1.1.0 github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-isatty v0.0.20 github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 - github.com/olekukonko/tablewriter v0.0.5 github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 github.com/pion/stun/v2 v2.0.0 github.com/pkg/errors v0.9.1 @@ -63,24 +61,42 @@ require ( github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible github.com/spf13/pflag v1.0.6 github.com/status-im/keycard-go v0.2.0 - github.com/stretchr/testify v1.10.0 - github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe + github.com/stretchr/testify v1.11.1 + github.com/supranational/blst v0.3.16 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/urfave/cli/v2 v2.27.5 + go.opentelemetry.io/otel v1.40.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0 + go.opentelemetry.io/otel/sdk v1.40.0 + go.opentelemetry.io/otel/trace v1.40.0 go.uber.org/automaxprocs v1.5.2 go.uber.org/goleak v1.3.0 - golang.org/x/crypto v0.36.0 + golang.org/x/crypto v0.44.0 golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df - golang.org/x/sync v0.12.0 - golang.org/x/sys v0.36.0 - golang.org/x/text v0.23.0 + golang.org/x/sync v0.18.0 + golang.org/x/sys v0.40.0 + golang.org/x/text v0.31.0 golang.org/x/time v0.9.0 - golang.org/x/tools v0.29.0 - google.golang.org/protobuf v1.34.2 + golang.org/x/tools v0.38.0 + google.golang.org/protobuf v1.36.11 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 ) +require ( + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/metric v1.40.0 // indirect + go.opentelemetry.io/proto/otlp v1.9.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect + google.golang.org/grpc v1.77.0 // indirect +) + require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect @@ -118,10 +134,11 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect + github.com/grafana/pyroscope-go v1.2.7 + github.com/grafana/pyroscope-go/godeltaprof v0.1.9 // indirect github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kilic/bls12-381 v0.1.0 // indirect - github.com/klauspost/compress v1.16.0 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect @@ -142,13 +159,13 @@ require ( github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect - github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect - golang.org/x/mod v0.22.0 // indirect - golang.org/x/net v0.38.0 // indirect + golang.org/x/mod v0.29.0 // indirect + golang.org/x/net v0.47.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 4ec857c99b..d55fa498cf 100644 --- a/go.sum +++ b/go.sum @@ -54,6 +54,8 @@ github.com/bits-and-blooms/bitset v1.20.0 h1:2F+rfL86jE2d/bmw7OhqUg2Sj/1rURkBn3M github.com/bits-and-blooms/bitset v1.20.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/ccoveille/go-safecast v1.1.0 h1:iHKNWaZm+OznO7Eh6EljXPjGfGQsSfa6/sxPlIEKO+g= github.com/ccoveille/go-safecast v1.1.0/go.mod h1:QqwNjxQ7DAqY0C721OIO9InMk9zCwcsO7tnRuHytad8= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -77,14 +79,12 @@ github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwP github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= -github.com/consensys/gnark-crypto v0.18.0 h1:vIye/FqI50VeAr0B3dx+YjeIvmc3LWz4yEfbWBpTUf0= -github.com/consensys/gnark-crypto v0.18.0/go.mod h1:L3mXGFTe1ZN+RSJ+CLjUt9x7PNdx8ubaYfDROyp2Z8c= +github.com/consensys/gnark-crypto v0.18.1 h1:RyLV6UhPRoYYzaFnPQA4qK3DyuDgkTgskDdoGqFt3fI= +github.com/consensys/gnark-crypto v0.18.1/go.mod h1:L3mXGFTe1ZN+RSJ+CLjUt9x7PNdx8ubaYfDROyp2Z8c= github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc= github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/crate-crypto/go-eth-kzg v1.4.0 h1:WzDGjHk4gFg6YzV0rJOAsTK4z3Qkz5jd4RE3DAvPFkg= -github.com/crate-crypto/go-eth-kzg v1.4.0/go.mod h1:J9/u5sWfznSObptgfa92Jq8rTswn6ahQWEuiLHOjCUI= -github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a h1:W8mUrRp6NOVl3J+MYp5kPMoUZPp7aOYHtaua31lwRHg= -github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a/go.mod h1:sTwzHBvIzm2RfVCGNEBZgRyjwK40bVoun3ZnGOCafNM= +github.com/crate-crypto/go-eth-kzg v1.5.0 h1:FYRiJMJG2iv+2Dy3fi14SVGjcPteZ5HAAUe4YWlJygc= +github.com/crate-crypto/go-eth-kzg v1.5.0/go.mod h1:J9/u5sWfznSObptgfa92Jq8rTswn6ahQWEuiLHOjCUI= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -115,12 +115,10 @@ github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/emicklei/dot v1.6.2 h1:08GN+DD79cy/tzN6uLCT84+2Wk9u+wvqP+Hkx/dIR8A= github.com/emicklei/dot v1.6.2/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= -github.com/ethereum/c-kzg-4844/v2 v2.1.5 h1:aVtoLK5xwJ6c5RiqO8g8ptJ5KU+2Hdquf6G3aXiHh5s= -github.com/ethereum/c-kzg-4844/v2 v2.1.5/go.mod h1:u59hRTTah4Co6i9fDWtiCjTrblJv0UwsqZKCc0GfgUs= +github.com/ethereum/c-kzg-4844/v2 v2.1.6 h1:xQymkKCT5E2Jiaoqf3v4wsNgjZLY0lRSkZn27fRjSls= +github.com/ethereum/c-kzg-4844/v2 v2.1.6/go.mod h1:8HMkUZ5JRv4hpw/XUrYWSQNAUzhHMg2UDb/U+5m+XNw= github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab h1:rvv6MJhy07IMfEKuARQ9TKojGqLVNxQajaXEp/BoqSk= github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab/go.mod h1:IuLm4IsPipXKF7CW5Lzf68PIbZ5yl7FFd74l/E0o9A8= -github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= -github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/ferranbt/fastssz v0.1.4 h1:OCDB+dYDEQDvAgtAGnTSidK1Pe2tW3nFV40XyMkTeDY= @@ -144,6 +142,11 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= @@ -178,21 +181,27 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20230207041349-798e818bf904 h1:4/hN5RUoecvl+RmJRE2YxKWtnnQls6rQjjW5oV7qg2U= github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grafana/pyroscope-go v1.2.7 h1:VWBBlqxjyR0Cwk2W6UrE8CdcdD80GOFNutj0Kb1T8ac= +github.com/grafana/pyroscope-go v1.2.7/go.mod h1:o/bpSLiJYYP6HQtvcoVKiE9s5RiNgjYTj1DhiddP2Pc= +github.com/grafana/pyroscope-go/godeltaprof v0.1.9 h1:c1Us8i6eSmkW+Ez05d3co8kasnuOY813tbMN8i/a3Og= +github.com/grafana/pyroscope-go/godeltaprof v0.1.9/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= github.com/graph-gophers/graphql-go v1.3.0 h1:Eb9x/q6MFpCLz7jBCiP/WTxjSDrYLR1QY41SORZyNJ0= github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4= github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= github.com/holiman/billy v0.0.0-20250707135307-f2f9b9aae7db h1:IZUYC/xb3giYwBLMnr8d0TGTzPKFGNTCGgGLoyeX330= @@ -220,14 +229,14 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/karalabe/hid v1.0.1-0.20240306101548-573246063e52 h1:msKODTL1m0wigztaqILOtla9HeW1ciscYG4xjLtvk5I= -github.com/karalabe/hid v1.0.1-0.20240306101548-573246063e52/go.mod h1:qk1sX/IBgppQNcGCRoj90u6EGC056EBoIc1oEjCWla8= +github.com/karalabe/hid v1.0.1-0.20260315100226-f5d04adeffeb h1:Ag83At00qa4FLkcdMgrwHVSakqky/eZczOlxd4q336E= +github.com/karalabe/hid v1.0.1-0.20260315100226-f5d04adeffeb/go.mod h1:qk1sX/IBgppQNcGCRoj90u6EGC056EBoIc1oEjCWla8= github.com/kilic/bls12-381 v0.1.0 h1:encrdjqKMEvabVQ7qYOKu1OvhqpK4s47wDYtNiPtlp4= github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= -github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -261,7 +270,6 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= @@ -278,8 +286,6 @@ github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 h1:shk/vn9oCoOTmwcou github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= -github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= @@ -333,8 +339,8 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -348,6 +354,8 @@ github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -356,10 +364,10 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe h1:nbdqkIGOGfUAD54q1s2YBcBz/WcsxCO9HUQ4aGV5hUw= -github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/supranational/blst v0.3.16 h1:bTDadT+3fK497EvLdWRQEjiGnUtzJ7jjIUMF0jqwYhE= +github.com/supranational/blst v0.3.16/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= @@ -376,6 +384,24 @@ github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBi github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= +go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 h1:f0cb2XPmrqn4XMy9PNliTgRKJgS5WcL/u0/WRYGz4t0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0/go.mod h1:vnakAaFckOMiMtOIhFI2MNH4FYrZzXCYxmb1LlhoGz8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0 h1:Ckwye2FpXkYgiHX7fyVrN1uA/UYd9ounqqTuSNAv0k4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0/go.mod h1:teIFJh5pW2y+AN7riv6IBPX2DuesS3HgP39mwOspKwU= +go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= +go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= +go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= +go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= +go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= +go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= +go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= +go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= +go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= +go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/automaxprocs v1.5.2 h1:2LxUOGiR3O6tw8ui5sZa2LAaHnsviZdVOUZw4fvbnME= go.uber.org/automaxprocs v1.5.2/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -388,16 +414,16 @@ golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU= +golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc= golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= -golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -413,8 +439,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -423,8 +449,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -457,8 +483,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= -golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -476,8 +502,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= @@ -489,21 +515,29 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= -golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b h1:uA40e2M6fYRBf0+8uN5mLlqUtV192iiksiICIBkYJ1E= +google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:Xa7le7qx2vmqB/SzWUBa7KdMjpdpAHlh5QCSnjessQk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM= +google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/graphql/graphql.go b/graphql/graphql.go index aff9205ad9..797ab306dd 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -279,7 +279,7 @@ func (t *Transaction) GasPrice(ctx context.Context) hexutil.Big { return hexutil.Big{} } switch tx.Type() { - case types.DynamicFeeTxType: + case types.DynamicFeeTxType, types.BlobTxType, types.SetCodeTxType: if block != nil { if baseFee, _ := block.BaseFeePerGas(ctx); baseFee != nil { // price = min(gasTipCap + baseFee, gasFeeCap) @@ -588,6 +588,9 @@ func (t *Transaction) getLogs(ctx context.Context, hash common.Hash) (*[]*Log, e func (t *Transaction) Type(ctx context.Context) *hexutil.Uint64 { tx, _ := t.resolve(ctx) + if tx == nil { + return nil + } txType := hexutil.Uint64(tx.Type()) return &txType } @@ -717,6 +720,9 @@ func (b *Block) resolveHeader(ctx context.Context) (*types.Header, error) { if err != nil { return nil, err } + if b.header == nil { + return nil, nil + } if b.hash == (common.Hash{}) { b.hash = b.header.Hash() } @@ -1100,6 +1106,18 @@ func (b *Block) ExcessBlobGas(ctx context.Context) (*hexutil.Uint64, error) { return &ret, nil } +func (b *Block) SlotNumber(ctx context.Context) (*hexutil.Uint64, error) { + header, err := b.resolveHeader(ctx) + if err != nil { + return nil, err + } + if header.SlotNumber == nil { + return nil, nil + } + ret := hexutil.Uint64(*header.SlotNumber) + return &ret, nil +} + // BlockFilterCriteria encapsulates criteria passed to a `logs` accessor inside // a block. type BlockFilterCriteria struct { @@ -1441,7 +1459,7 @@ func (r *Resolver) Logs(ctx context.Context, args struct{ Filter FilterCriteria topics = *args.Filter.Topics } // Construct the range filter - filter := r.filterSystem.NewRangeFilter(begin, end, addresses, topics) + filter := r.filterSystem.NewRangeFilter(begin, end, addresses, topics, 0) return runFilter(ctx, r, filter) } diff --git a/internal/build/file.go b/internal/build/file.go index 2cd090c42c..b7c00eb842 100644 --- a/internal/build/file.go +++ b/internal/build/file.go @@ -21,20 +21,21 @@ import ( "io" "os" "path/filepath" - "sort" - "strings" + "slices" ) // HashFolder iterates all files under the given directory, computing the hash // of each. -func HashFolder(folder string, exlude []string) (map[string][32]byte, error) { +func HashFolder(folder string, excludes []string) (map[string][32]byte, error) { res := make(map[string][32]byte) err := filepath.WalkDir(folder, func(path string, d os.DirEntry, _ error) error { // Skip anything that's exluded or not a regular file - for _, skip := range exlude { - if strings.HasPrefix(path, filepath.FromSlash(skip)) { + // Skip anything that's excluded or not a regular file + if slices.Contains(excludes, path) { + if d.IsDir() { return filepath.SkipDir } + return nil } if !d.Type().IsRegular() { return nil @@ -71,6 +72,6 @@ func DiffHashes(a map[string][32]byte, b map[string][32]byte) []string { updates = append(updates, file) } } - sort.Strings(updates) + slices.Sort(updates) return updates } diff --git a/internal/debug/api.go b/internal/debug/api.go index 1bac36e908..5a2781cc77 100644 --- a/internal/debug/api.go +++ b/internal/debug/api.go @@ -252,7 +252,7 @@ func (*HandlerT) SetGCPercent(v int) int { // - Geth also allocates memory off-heap, particularly for fastCache and Pebble, // which can be non-trivial (a few gigabytes by default). func (*HandlerT) SetMemoryLimit(limit int64) int64 { - log.Info("Setting memory limit", "size", common.PrettyDuration(limit)) + log.Info("Setting memory limit", "size", common.StorageSize(limit)) return debug.SetMemoryLimit(limit) } diff --git a/internal/debug/flags.go b/internal/debug/flags.go index 3b865bd1df..0639f57bb0 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -162,6 +162,11 @@ var Flags = []cli.Flag{ blockprofilerateFlag, cpuprofileFlag, traceFlag, + pyroscopeFlag, + pyroscopeServerFlag, + pyroscopeAuthUsernameFlag, + pyroscopeAuthPasswordFlag, + pyroscopeTagsFlag, } var ( @@ -300,6 +305,14 @@ func Setup(ctx *cli.Context) error { // It cannot be imported because it will cause a cyclical dependency. StartPProf(address, !ctx.IsSet("metrics.addr")) } + + // Pyroscope profiling + if ctx.Bool(pyroscopeFlag.Name) { + if err := startPyroscope(ctx); err != nil { + return err + } + } + if len(logFile) > 0 || rotation { log.Info("Logging configured", context...) } @@ -323,6 +336,7 @@ func StartPProf(address string, withMetrics bool) { // Exit stops all running profiles, flushing their output to the // respective file. func Exit() { + stopPyroscope() Handler.StopCPUProfile() Handler.StopGoTrace() if logOutputFile != nil { diff --git a/internal/debug/pyroscope.go b/internal/debug/pyroscope.go new file mode 100644 index 0000000000..d0804cb891 --- /dev/null +++ b/internal/debug/pyroscope.go @@ -0,0 +1,134 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package debug + +import ( + "fmt" + "strings" + + "github.com/ethereum/go-ethereum/internal/flags" + "github.com/ethereum/go-ethereum/log" + "github.com/grafana/pyroscope-go" + "github.com/urfave/cli/v2" +) + +var ( + pyroscopeFlag = &cli.BoolFlag{ + Name: "pyroscope", + Usage: "Enable Pyroscope profiling", + Value: false, + Category: flags.LoggingCategory, + } + pyroscopeServerFlag = &cli.StringFlag{ + Name: "pyroscope.server", + Usage: "Pyroscope server URL to push profiling data to", + Value: "http://localhost:4040", + Category: flags.LoggingCategory, + } + pyroscopeAuthUsernameFlag = &cli.StringFlag{ + Name: "pyroscope.username", + Usage: "Pyroscope basic authentication username", + Value: "", + Category: flags.LoggingCategory, + } + pyroscopeAuthPasswordFlag = &cli.StringFlag{ + Name: "pyroscope.password", + Usage: "Pyroscope basic authentication password", + Value: "", + Category: flags.LoggingCategory, + } + pyroscopeTagsFlag = &cli.StringFlag{ + Name: "pyroscope.tags", + Usage: "Comma separated list of key=value tags to add to profiling data", + Value: "", + Category: flags.LoggingCategory, + } +) + +// This holds the globally-configured Pyroscope instance. +var pyroscopeProfiler *pyroscope.Profiler + +func startPyroscope(ctx *cli.Context) error { + server := ctx.String(pyroscopeServerFlag.Name) + authUsername := ctx.String(pyroscopeAuthUsernameFlag.Name) + authPassword := ctx.String(pyroscopeAuthPasswordFlag.Name) + + rawTags := ctx.String(pyroscopeTagsFlag.Name) + tags := make(map[string]string) + for tag := range strings.SplitSeq(rawTags, ",") { + tag = strings.TrimSpace(tag) + if tag == "" { + continue + } + k, v, _ := strings.Cut(tag, "=") + tags[k] = v + } + + config := pyroscope.Config{ + ApplicationName: "geth", + ServerAddress: server, + BasicAuthUser: authUsername, + BasicAuthPassword: authPassword, + Logger: &pyroscopeLogger{Logger: log.Root()}, + Tags: tags, + ProfileTypes: []pyroscope.ProfileType{ + // Enabling all profile types + pyroscope.ProfileCPU, + pyroscope.ProfileAllocObjects, + pyroscope.ProfileAllocSpace, + pyroscope.ProfileInuseObjects, + pyroscope.ProfileInuseSpace, + pyroscope.ProfileGoroutines, + pyroscope.ProfileMutexCount, + pyroscope.ProfileMutexDuration, + pyroscope.ProfileBlockCount, + pyroscope.ProfileBlockDuration, + }, + } + + profiler, err := pyroscope.Start(config) + if err != nil { + return err + } + pyroscopeProfiler = profiler + log.Info("Enabled Pyroscope") + return nil +} + +func stopPyroscope() { + if pyroscopeProfiler != nil { + pyroscopeProfiler.Stop() + pyroscopeProfiler = nil + } +} + +// Small wrapper for log.Logger to satisfy pyroscope.Logger interface +type pyroscopeLogger struct { + Logger log.Logger +} + +func (l *pyroscopeLogger) Infof(format string, v ...any) { + l.Logger.Info(fmt.Sprintf("Pyroscope: "+format, v...)) +} + +func (l *pyroscopeLogger) Debugf(format string, v ...any) { + l.Logger.Debug(fmt.Sprintf("Pyroscope: "+format, v...)) +} + +func (l *pyroscopeLogger) Errorf(format string, v ...any) { + l.Logger.Error(fmt.Sprintf("Pyroscope: "+format, v...)) +} diff --git a/internal/download/download.go b/internal/download/download.go index 26c7795ce5..c59c8a90c3 100644 --- a/internal/download/download.go +++ b/internal/download/download.go @@ -205,7 +205,10 @@ func (db *ChecksumDB) DownloadFile(url, dstPath string) error { if err != nil { return err } - dst := newDownloadWriter(fd, resp.ContentLength) + var dst io.WriteCloser = fd + if resp.ContentLength > 0 { + dst = newDownloadWriter(fd, resp.ContentLength) + } _, err = io.Copy(dst, resp.Body) dst.Close() if err != nil { diff --git a/internal/era/accumulator.go b/internal/era/accumulator.go index 83a761f1fd..72e36fe00f 100644 --- a/internal/era/accumulator.go +++ b/internal/era/accumulator.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "math/big" + "slices" "github.com/ethereum/go-ethereum/common" ssz "github.com/ferranbt/fastssz" @@ -31,8 +32,8 @@ func ComputeAccumulator(hashes []common.Hash, tds []*big.Int) (common.Hash, erro if len(hashes) != len(tds) { return common.Hash{}, errors.New("must have equal number hashes as td values") } - if len(hashes) > MaxEra1Size { - return common.Hash{}, fmt.Errorf("too many records: have %d, max %d", len(hashes), MaxEra1Size) + if len(hashes) > MaxSize { + return common.Hash{}, fmt.Errorf("too many records: have %d, max %d", len(hashes), MaxSize) } hh := ssz.NewHasher() for i := range hashes { @@ -43,7 +44,7 @@ func ComputeAccumulator(hashes []common.Hash, tds []*big.Int) (common.Hash, erro } hh.Append(root[:]) } - hh.MerkleizeWithMixin(0, uint64(len(hashes)), uint64(MaxEra1Size)) + hh.MerkleizeWithMixin(0, uint64(len(hashes)), uint64(MaxSize)) return hh.HashRoot() } @@ -69,23 +70,15 @@ func (h *headerRecord) HashTreeRoot() ([32]byte, error) { // HashTreeRootWith ssz hashes the headerRecord object with a hasher. func (h *headerRecord) HashTreeRootWith(hh ssz.HashWalker) (err error) { hh.PutBytes(h.Hash[:]) - td := bigToBytes32(h.TotalDifficulty) + td := BigToBytes32(h.TotalDifficulty) hh.PutBytes(td[:]) hh.Merkleize(0) return } // bigToBytes32 converts a big.Int into a little-endian 32-byte array. -func bigToBytes32(n *big.Int) (b [32]byte) { +func BigToBytes32(n *big.Int) (b [32]byte) { n.FillBytes(b[:]) - reverseOrder(b[:]) + slices.Reverse(b[:]) return } - -// reverseOrder reverses the byte order of a slice. -func reverseOrder(b []byte) []byte { - for i := 0; i < 16; i++ { - b[i], b[32-i-1] = b[32-i-1], b[i] - } - return b -} diff --git a/internal/era/era.go b/internal/era/era.go index 118c67abfd..a3c8465bc4 100644 --- a/internal/era/era.go +++ b/internal/era/era.go @@ -1,4 +1,4 @@ -// Copyright 2024 The go-ethereum Authors +// Copyright 2025 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -17,7 +17,6 @@ package era import ( - "encoding/binary" "fmt" "io" "math/big" @@ -25,293 +24,132 @@ import ( "path" "strconv" "strings" - "sync" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/internal/era/e2store" - "github.com/ethereum/go-ethereum/rlp" - "github.com/golang/snappy" ) +// Type constants for the e2store entries in the Era1 and EraE formats. var ( - TypeVersion uint16 = 0x3265 - TypeCompressedHeader uint16 = 0x03 - TypeCompressedBody uint16 = 0x04 - TypeCompressedReceipts uint16 = 0x05 - TypeTotalDifficulty uint16 = 0x06 - TypeAccumulator uint16 = 0x07 - TypeBlockIndex uint16 = 0x3266 - - MaxEra1Size = 8192 + TypeVersion uint16 = 0x3265 + TypeCompressedHeader uint16 = 0x03 + TypeCompressedBody uint16 = 0x04 + TypeCompressedReceipts uint16 = 0x05 + TypeTotalDifficulty uint16 = 0x06 + TypeAccumulator uint16 = 0x07 + TypeCompressedSlimReceipts uint16 = 0x0a // uses eth/69 encoding + TypeProof uint16 = 0x0b + TypeBlockIndex uint16 = 0x3266 + TypeComponentIndex uint16 = 0x3267 + + MaxSize = 8192 + // headerSize uint64 = 8 ) -// Filename returns a recognizable Era1-formatted file name for the specified -// epoch and network. -func Filename(network string, epoch int, root common.Hash) string { - return fmt.Sprintf("%s-%05d-%s.era1", network, epoch, root.Hex()[2:10]) +type ReadAtSeekCloser interface { + io.ReaderAt + io.Seeker + io.Closer } -// ReadDir reads all the era1 files in a directory for a given network. -// Format: --.era1 +// Iterator provides sequential access to blocks in an era file. +type Iterator interface { + // Next advances to the next block. Returns true if a block is available, + // false when iteration is complete or an error occurred. + Next() bool + + // Number returns the block number of the current block. + Number() uint64 + + // Block returns the current block. + Block() (*types.Block, error) + + // BlockAndReceipts returns the current block and its receipts. + BlockAndReceipts() (*types.Block, types.Receipts, error) + + // Receipts returns the receipts for the current block. + Receipts() (types.Receipts, error) + + // Error returns any error encountered during iteration. + Error() error +} + +// Builder constructs era files from blocks and receipts. +// +// Builders handle three epoch types automatically: +// - Pre-merge: all blocks have difficulty > 0, TD is stored for each block +// - Transition: starts pre-merge, ends post-merge; TD stored for all blocks +// - Post-merge: all blocks have difficulty == 0, no TD stored +type Builder interface { + // Add appends a block and its receipts to the era file. + // For pre-merge blocks, td must be provided. + // For post-merge blocks, td should be nil. + Add(block *types.Block, receipts types.Receipts, td *big.Int) error + + // AddRLP appends RLP-encoded block components to the era file. + // For pre-merge blocks, td and difficulty must be provided. + // For post-merge blocks, td and difficulty should be nil. + AddRLP(header, body, receipts []byte, number uint64, hash common.Hash, td, difficulty *big.Int) error + + // Finalize writes all collected entries and returns the epoch identifier. + // For Era1 (onedb): returns the accumulator root. + // For EraE (execdb): returns the last block hash. + Finalize() (common.Hash, error) + + // Accumulator returns the accumulator root after Finalize has been called. + // Returns nil for post-merge epochs where no accumulator exists. + Accumulator() *common.Hash +} + +// Era represents the interface for reading era data. +type Era interface { + Close() error + Start() uint64 + Count() uint64 + Iterator() (Iterator, error) + GetBlockByNumber(num uint64) (*types.Block, error) + GetRawBodyByNumber(num uint64) ([]byte, error) + GetRawReceiptsByNumber(num uint64) ([]byte, error) + InitialTD() (*big.Int, error) + Accumulator() (common.Hash, error) +} + +// ReadDir reads all the era files in a directory for a given network. +// Format: --.erae or --.era1 func ReadDir(dir, network string) ([]string, error) { entries, err := os.ReadDir(dir) + if err != nil { return nil, fmt.Errorf("error reading directory %s: %w", dir, err) } var ( - next = uint64(0) - eras []string + next = uint64(0) + eras []string + dirType string ) for _, entry := range entries { - if path.Ext(entry.Name()) != ".era1" { + ext := path.Ext(entry.Name()) + if ext != ".erae" && ext != ".era1" { continue } + if dirType == "" { + dirType = ext + } parts := strings.Split(entry.Name(), "-") if len(parts) != 3 || parts[0] != network { - // Invalid era1 filename, skip. + // Invalid era filename, skip. continue } if epoch, err := strconv.ParseUint(parts[1], 10, 64); err != nil { - return nil, fmt.Errorf("malformed era1 filename: %s", entry.Name()) + return nil, fmt.Errorf("malformed era filenames: %s", entry.Name()) } else if epoch != next { return nil, fmt.Errorf("missing epoch %d", next) } + if dirType != ext { + return nil, fmt.Errorf("directory %s contains mixed era file formats: want %s, have %s", dir, dirType, ext) + } next += 1 eras = append(eras, entry.Name()) } return eras, nil } - -type ReadAtSeekCloser interface { - io.ReaderAt - io.Seeker - io.Closer -} - -// Era reads and Era1 file. -type Era struct { - f ReadAtSeekCloser // backing era1 file - s *e2store.Reader // e2store reader over f - m metadata // start, count, length info - mu *sync.Mutex // lock for buf - buf [8]byte // buffer reading entry offsets -} - -// From returns an Era backed by f. -func From(f ReadAtSeekCloser) (*Era, error) { - m, err := readMetadata(f) - if err != nil { - return nil, err - } - return &Era{ - f: f, - s: e2store.NewReader(f), - m: m, - mu: new(sync.Mutex), - }, nil -} - -// Open returns an Era backed by the given filename. -func Open(filename string) (*Era, error) { - f, err := os.Open(filename) - if err != nil { - return nil, err - } - return From(f) -} - -func (e *Era) Close() error { - return e.f.Close() -} - -// GetBlockByNumber returns the block for the given block number. -func (e *Era) GetBlockByNumber(num uint64) (*types.Block, error) { - if e.m.start > num || e.m.start+e.m.count <= num { - return nil, fmt.Errorf("out-of-bounds: %d not in [%d, %d)", num, e.m.start, e.m.start+e.m.count) - } - off, err := e.readOffset(num) - if err != nil { - return nil, err - } - r, n, err := newSnappyReader(e.s, TypeCompressedHeader, off) - if err != nil { - return nil, err - } - var header types.Header - if err := rlp.Decode(r, &header); err != nil { - return nil, err - } - off += n - r, _, err = newSnappyReader(e.s, TypeCompressedBody, off) - if err != nil { - return nil, err - } - var body types.Body - if err := rlp.Decode(r, &body); err != nil { - return nil, err - } - return types.NewBlockWithHeader(&header).WithBody(body), nil -} - -// GetRawBodyByNumber returns the RLP-encoded body for the given block number. -func (e *Era) GetRawBodyByNumber(num uint64) ([]byte, error) { - if e.m.start > num || e.m.start+e.m.count <= num { - return nil, fmt.Errorf("out-of-bounds: %d not in [%d, %d)", num, e.m.start, e.m.start+e.m.count) - } - off, err := e.readOffset(num) - if err != nil { - return nil, err - } - off, err = e.s.SkipN(off, 1) - if err != nil { - return nil, err - } - r, _, err := newSnappyReader(e.s, TypeCompressedBody, off) - if err != nil { - return nil, err - } - return io.ReadAll(r) -} - -// GetRawReceiptsByNumber returns the RLP-encoded receipts for the given block number. -func (e *Era) GetRawReceiptsByNumber(num uint64) ([]byte, error) { - if e.m.start > num || e.m.start+e.m.count <= num { - return nil, fmt.Errorf("out-of-bounds: %d not in [%d, %d)", num, e.m.start, e.m.start+e.m.count) - } - off, err := e.readOffset(num) - if err != nil { - return nil, err - } - - // Skip over header and body. - off, err = e.s.SkipN(off, 2) - if err != nil { - return nil, err - } - - r, _, err := newSnappyReader(e.s, TypeCompressedReceipts, off) - if err != nil { - return nil, err - } - return io.ReadAll(r) -} - -// Accumulator reads the accumulator entry in the Era1 file. -func (e *Era) Accumulator() (common.Hash, error) { - entry, err := e.s.Find(TypeAccumulator) - if err != nil { - return common.Hash{}, err - } - return common.BytesToHash(entry.Value), nil -} - -// InitialTD returns initial total difficulty before the difficulty of the -// first block of the Era1 is applied. -func (e *Era) InitialTD() (*big.Int, error) { - var ( - r io.Reader - header types.Header - rawTd []byte - n int64 - off int64 - err error - ) - - // Read first header. - if off, err = e.readOffset(e.m.start); err != nil { - return nil, err - } - if r, n, err = newSnappyReader(e.s, TypeCompressedHeader, off); err != nil { - return nil, err - } - if err := rlp.Decode(r, &header); err != nil { - return nil, err - } - off += n - - // Skip over header and body. - off, err = e.s.SkipN(off, 2) - if err != nil { - return nil, err - } - - // Read total difficulty after first block. - if r, _, err = e.s.ReaderAt(TypeTotalDifficulty, off); err != nil { - return nil, err - } - rawTd, err = io.ReadAll(r) - if err != nil { - return nil, err - } - td := new(big.Int).SetBytes(reverseOrder(rawTd)) - return td.Sub(td, header.Difficulty), nil -} - -// Start returns the listed start block. -func (e *Era) Start() uint64 { - return e.m.start -} - -// Count returns the total number of blocks in the Era1. -func (e *Era) Count() uint64 { - return e.m.count -} - -// readOffset reads a specific block's offset from the block index. The value n -// is the absolute block number desired. -func (e *Era) readOffset(n uint64) (int64, error) { - var ( - blockIndexRecordOffset = e.m.length - 24 - int64(e.m.count)*8 // skips start, count, and header - firstIndex = blockIndexRecordOffset + 16 // first index after header / start-num - indexOffset = int64(n-e.m.start) * 8 // desired index * size of indexes - offOffset = firstIndex + indexOffset // offset of block offset - ) - e.mu.Lock() - defer e.mu.Unlock() - clear(e.buf[:]) - if _, err := e.f.ReadAt(e.buf[:], offOffset); err != nil { - return 0, err - } - // Since the block offset is relative from the start of the block index record - // we need to add the record offset to it's offset to get the block's absolute - // offset. - return blockIndexRecordOffset + int64(binary.LittleEndian.Uint64(e.buf[:])), nil -} - -// newSnappyReader returns a snappy.Reader for the e2store entry value at off. -func newSnappyReader(e *e2store.Reader, expectedType uint16, off int64) (io.Reader, int64, error) { - r, n, err := e.ReaderAt(expectedType, off) - if err != nil { - return nil, 0, err - } - return snappy.NewReader(r), int64(n), err -} - -// metadata wraps the metadata in the block index. -type metadata struct { - start uint64 - count uint64 - length int64 -} - -// readMetadata reads the metadata stored in an Era1 file's block index. -func readMetadata(f ReadAtSeekCloser) (m metadata, err error) { - // Determine length of reader. - if m.length, err = f.Seek(0, io.SeekEnd); err != nil { - return - } - b := make([]byte, 16) - // Read count. It's the last 8 bytes of the file. - if _, err = f.ReadAt(b[:8], m.length-8); err != nil { - return - } - m.count = binary.LittleEndian.Uint64(b) - // Read start. It's at the offset -sizeof(m.count) - - // count*sizeof(indexEntry) - sizeof(m.start) - if _, err = f.ReadAt(b[8:], m.length-16-int64(m.count*8)); err != nil { - return - } - m.start = binary.LittleEndian.Uint64(b[8:]) - return -} diff --git a/internal/era/eradl/eradl.go b/internal/era/eradl/eradl.go index 30bd2bc0d5..375b9ad15b 100644 --- a/internal/era/eradl/eradl.go +++ b/internal/era/eradl/eradl.go @@ -86,8 +86,8 @@ func (l *Loader) DownloadAll(destDir string) error { // DownloadBlockRange fetches the era1 files for the given block range. func (l *Loader) DownloadBlockRange(start, end uint64, destDir string) error { - startEpoch := start / uint64(era.MaxEra1Size) - endEpoch := end / uint64(era.MaxEra1Size) + startEpoch := start / uint64(era.MaxSize) + endEpoch := end / uint64(era.MaxSize) return l.DownloadEpochRange(startEpoch, endEpoch, destDir) } diff --git a/internal/era/execdb/builder.go b/internal/era/execdb/builder.go new file mode 100644 index 0000000000..6246b9caae --- /dev/null +++ b/internal/era/execdb/builder.go @@ -0,0 +1,332 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package execdb + +// EraE file format specification. +// +// The format can be summarized with the following expression: +// +// eraE := Version | CompressedHeader* | CompressedBody* | CompressedSlimReceipts* | TotalDifficulty* | other-entries* | Accumulator? | ComponentIndex +// +// Each basic element is its own e2store entry: +// +// Version = { type: 0x3265, data: nil } +// CompressedHeader = { type: 0x03, data: snappyFramed(rlp(header)) } +// CompressedBody = { type: 0x04, data: snappyFramed(rlp(body)) } +// CompressedSlimReceipts = { type: 0x0a, data: snappyFramed(rlp([tx-type, post-state-or-status, cumulative-gas, logs])) } +// TotalDifficulty = { type: 0x06, data: uint256 (header.total_difficulty) } +// AccumulatorRoot = { type: 0x07, data: hash_tree_root(List(HeaderRecord, 8192)) } +// ComponentIndex = { type: 0x3267, data: component-index } +// +// Notes: +// - TotalDifficulty is present for pre-merge and merge transition epochs. +// For pure post-merge epochs, TotalDifficulty is omitted entirely. +// - In merge transition epochs, post-merge blocks store the final total +// difficulty (the TD at which the merge occurred). +// - AccumulatorRoot is only written for pre-merge epochs. +// - HeaderRecord is defined in the Portal Network specification. +// - Proofs (type 0x09) are defined in the spec but not yet supported in this implementation. +// +// ComponentIndex stores relative offsets to each block's components: +// +// component-index := starting-number | indexes | indexes | ... | component-count | count +// indexes := header-offset | body-offset | receipts-offset | td-offset? +// +// All values are little-endian uint64. +// +// Due to the accumulator size limit of 8192, the maximum number of blocks in an +// EraE file is also 8192. + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/era/e2store" + "github.com/ethereum/go-ethereum/rlp" + "github.com/golang/snappy" +) + +// Builder is used to build an EraE e2store file. It collects block entries and +// writes them to the underlying e2store.Writer. +type Builder struct { + w *e2store.Writer + + headers [][]byte + hashes []common.Hash // only pre-merge block hashes, for accumulator + bodies [][]byte + receipts [][]byte + tds []*big.Int + + startNum *uint64 + ttd *big.Int // terminal total difficulty + last common.Hash // hash of last block added + accumulator *common.Hash // accumulator root, set by Finalize (nil for post-merge) + + written uint64 + + buf *bytes.Buffer + snappy *snappy.Writer +} + +// NewBuilder returns a new Builder instance. +func NewBuilder(w io.Writer) era.Builder { + return &Builder{ + w: e2store.NewWriter(w), + } +} + +// Add writes a block entry and its receipts into the e2store file. +func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int) error { + eh, err := rlp.EncodeToBytes(block.Header()) + if err != nil { + return fmt.Errorf("encode header: %w", err) + } + eb, err := rlp.EncodeToBytes(block.Body()) + if err != nil { + return fmt.Errorf("encode body: %w", err) + } + + rs := make([]*types.SlimReceipt, len(receipts)) + for i, receipt := range receipts { + rs[i] = (*types.SlimReceipt)(receipt) + } + er, err := rlp.EncodeToBytes(rs) + if err != nil { + return fmt.Errorf("encode receipts: %w", err) + } + + return b.AddRLP(eh, eb, er, block.Number().Uint64(), block.Hash(), td, block.Difficulty()) +} + +// AddRLP takes the RLP encoded block components and writes them to the underlying e2store file. +// The builder automatically handles transition epochs where both pre and post-merge blocks exist. +func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, blockHash common.Hash, td, difficulty *big.Int) error { + if len(b.headers) >= era.MaxSize { + return fmt.Errorf("exceeds max size %d", era.MaxSize) + } + // Set starting block number on first add. + if b.startNum == nil { + b.startNum = new(uint64) + *b.startNum = number + } + + if difficulty == nil { + return fmt.Errorf("invalid block: difficulty is nil") + } + hasDifficulty := difficulty.Sign() > 0 + // Expect td to be nil for post-merge blocks + // and non-nil for pre-merge blocks. + if hasDifficulty != (td != nil) { + return fmt.Errorf("TD and difficulty mismatch: expected both nil or both non-nil") + } + // After the merge, difficulty must be nil. + post := (b.tds == nil && len(b.headers) > 0) || b.ttd != nil + if post && hasDifficulty { + return fmt.Errorf("post-merge epoch: cannot accept total difficulty for block %d", number) + } + + // If this marks the start of the transition, record final total + // difficulty value. + if b.ttd == nil && len(b.tds) > 0 && !hasDifficulty { + b.ttd = new(big.Int).Set(b.tds[len(b.tds)-1]) + } + + // Record block data. + b.headers = append(b.headers, header) + b.bodies = append(b.bodies, body) + b.receipts = append(b.receipts, receipts) + b.last = blockHash + + // Conditionally write the total difficulty and block hashes. + // - Pre-merge: store total difficulty and block hashes. + // - Transition: only store total difficulty. + // - Post-merge: store neither. + if hasDifficulty { + b.hashes = append(b.hashes, blockHash) + b.tds = append(b.tds, new(big.Int).Set(td)) + } else if b.ttd != nil { + b.tds = append(b.tds, new(big.Int).Set(b.ttd)) + } else { + // Post-merge: no TD or block hashes stored. + } + + return nil +} + +// Accumulator returns the accumulator root after Finalize has been called. +// Returns nil for post-merge epochs where no accumulator exists. +func (b *Builder) Accumulator() *common.Hash { + return b.accumulator +} + +type offsets struct { + headers []uint64 + bodies []uint64 + receipts []uint64 + tds []uint64 +} + +// Finalize writes all collected block entries to the e2store file. +// For pre-merge or transition epochs, the accumulator root is computed over +// pre-merge blocks and written. For pure post-merge epochs, no accumulator +// is written. Always returns the last block hash as the epoch identifier. +func (b *Builder) Finalize() (common.Hash, error) { + if b.startNum == nil { + return common.Hash{}, errors.New("no blocks added, cannot finalize") + } + // Write version before writing any blocks. + if n, err := b.w.Write(era.TypeVersion, nil); err != nil { + return common.Hash{}, fmt.Errorf("write version entry: %w", err) + } else { + b.written += uint64(n) + } + + // Convert TD values to byte-level LE representation. + var tds [][]byte + for _, td := range b.tds { + tds = append(tds, uint256LE(td)) + } + + // Create snappy writer. + b.buf = bytes.NewBuffer(nil) + b.snappy = snappy.NewBufferedWriter(b.buf) + + var o offsets + for _, section := range []struct { + typ uint16 + data [][]byte + compressed bool + offsets *[]uint64 + }{ + {era.TypeCompressedHeader, b.headers, true, &o.headers}, + {era.TypeCompressedBody, b.bodies, true, &o.bodies}, + {era.TypeCompressedSlimReceipts, b.receipts, true, &o.receipts}, + {era.TypeTotalDifficulty, tds, false, &o.tds}, + } { + for _, data := range section.data { + *section.offsets = append(*section.offsets, b.written) + if section.compressed { + // Write snappy compressed data. + if err := b.snappyWrite(section.typ, data); err != nil { + return common.Hash{}, err + } + } else { + // Directly write uncompressed data. + n, err := b.w.Write(section.typ, data) + if err != nil { + return common.Hash{}, err + } + b.written += uint64(n) + } + } + } + + // Compute and write accumulator root only for epochs that started pre-merge. + // The accumulator is computed over only the pre-merge blocks (b.hashes). + // Pure post-merge epochs have no accumulator. + if len(b.tds) > 0 { + accRoot, err := era.ComputeAccumulator(b.hashes, b.tds[:len(b.hashes)]) + if err != nil { + return common.Hash{}, fmt.Errorf("compute accumulator: %w", err) + } + if n, err := b.w.Write(era.TypeAccumulator, accRoot[:]); err != nil { + return common.Hash{}, fmt.Errorf("write accumulator: %w", err) + } else { + b.written += uint64(n) + } + b.accumulator = &accRoot + if err := b.writeIndex(&o); err != nil { + return common.Hash{}, err + } + return b.last, nil + } + + // Pure post-merge epoch: no accumulator. + if err := b.writeIndex(&o); err != nil { + return common.Hash{}, err + } + return b.last, nil +} + +// uin256LE writes 32 byte big integers to little endian. +func uint256LE(v *big.Int) []byte { + b := v.FillBytes(make([]byte, 32)) + for i := 0; i < 16; i++ { + b[i], b[31-i] = b[31-i], b[i] + } + return b +} + +// SnappyWrite compresses the input data using snappy and writes it to the e2store file. +func (b *Builder) snappyWrite(typ uint16, in []byte) error { + b.buf.Reset() + b.snappy.Reset(b.buf) + if _, err := b.snappy.Write(in); err != nil { + return fmt.Errorf("error snappy encoding: %w", err) + } + if err := b.snappy.Flush(); err != nil { + return fmt.Errorf("error flushing snappy encoding: %w", err) + } + n, err := b.w.Write(typ, b.buf.Bytes()) + b.written += uint64(n) + if err != nil { + return fmt.Errorf("error writing e2store entry: %w", err) + } + return nil +} + +// writeIndex writes the component index to the file. +func (b *Builder) writeIndex(o *offsets) error { + count := len(o.headers) + + // Post-merge, we only index headers, bodies, and receipts. Pre-merge, we also + // need to index the total difficulties. + componentCount := 3 + if len(o.tds) > 0 { + componentCount++ + } + + // Offsets are stored relative to the index position (negative, stored as uint64). + base := int64(b.written) + rel := func(abs uint64) uint64 { return uint64(int64(abs) - base) } + + var buf bytes.Buffer + write := func(v uint64) { binary.Write(&buf, binary.LittleEndian, v) } + + write(*b.startNum) + for i := range o.headers { + write(rel(o.headers[i])) + write(rel(o.bodies[i])) + write(rel(o.receipts[i])) + if len(o.tds) > 0 { + write(rel(o.tds[i])) + } + } + write(uint64(componentCount)) + write(uint64(count)) + + n, err := b.w.Write(era.TypeComponentIndex, buf.Bytes()) + b.written += uint64(n) + return err +} diff --git a/internal/era/execdb/era_test.go b/internal/era/execdb/era_test.go new file mode 100644 index 0000000000..f66931b9ed --- /dev/null +++ b/internal/era/execdb/era_test.go @@ -0,0 +1,348 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package execdb + +import ( + "bytes" + "fmt" + "io" + "math/big" + "os" + "slices" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +func TestEraE(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + start uint64 + preMerge int + postMerge int + accumulator bool // whether accumulator should exist + }{ + { + name: "pre-merge", + start: 0, + preMerge: 128, + postMerge: 0, + accumulator: true, + }, + { + name: "post-merge", + start: 0, + preMerge: 0, + postMerge: 64, + accumulator: false, + }, + { + name: "transition", + start: 0, + preMerge: 32, + postMerge: 32, + accumulator: true, + }, + { + name: "non-zero-start", + start: 8192, + preMerge: 64, + postMerge: 0, + accumulator: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + f, err := os.CreateTemp(t.TempDir(), "erae-test") + if err != nil { + t.Fatalf("error creating temp file: %v", err) + } + defer f.Close() + + // Build test data. + type blockData struct { + header, body, receipts []byte + hash common.Hash + td *big.Int + difficulty *big.Int + } + var ( + builder = NewBuilder(f) + blocks []blockData + totalBlocks = tt.preMerge + tt.postMerge + finalTD = big.NewInt(int64(tt.preMerge)) + ) + + // Add pre-merge blocks. + for i := 0; i < tt.preMerge; i++ { + num := tt.start + uint64(i) + blk := blockData{ + header: mustEncode(&types.Header{Number: big.NewInt(int64(num)), Difficulty: big.NewInt(1)}), + body: mustEncode(&types.Body{Transactions: []*types.Transaction{types.NewTransaction(0, common.Address{byte(i)}, nil, 0, nil, nil)}}), + receipts: mustEncode([]types.SlimReceipt{{CumulativeGasUsed: uint64(i)}}), + hash: common.Hash{byte(i)}, + td: big.NewInt(int64(i + 1)), + difficulty: big.NewInt(1), + } + blocks = append(blocks, blk) + if err := builder.AddRLP(blk.header, blk.body, blk.receipts, num, blk.hash, blk.td, blk.difficulty); err != nil { + t.Fatalf("error adding pre-merge block %d: %v", i, err) + } + } + + // Add post-merge blocks. + for i := 0; i < tt.postMerge; i++ { + idx := tt.preMerge + i + num := tt.start + uint64(idx) + blk := blockData{ + header: mustEncode(&types.Header{Number: big.NewInt(int64(num)), Difficulty: big.NewInt(0)}), + body: mustEncode(&types.Body{}), + receipts: mustEncode([]types.SlimReceipt{}), + hash: common.Hash{byte(idx)}, + difficulty: big.NewInt(0), + } + blocks = append(blocks, blk) + if err := builder.AddRLP(blk.header, blk.body, blk.receipts, num, blk.hash, nil, big.NewInt(0)); err != nil { + t.Fatalf("error adding post-merge block %d: %v", idx, err) + } + } + + // Finalize and check return values. + epochID, err := builder.Finalize() + if err != nil { + t.Fatalf("error finalizing: %v", err) + } + + // Verify epoch ID is always the last block hash. + expectedLastHash := blocks[len(blocks)-1].hash + if epochID != expectedLastHash { + t.Fatalf("wrong epoch ID: want %s, got %s", expectedLastHash.Hex(), epochID.Hex()) + } + + // Verify accumulator presence. + if tt.accumulator { + if builder.Accumulator() == nil { + t.Fatal("expected non-nil accumulator") + } + } else { + if builder.Accumulator() != nil { + t.Fatalf("expected nil accumulator, got %s", builder.Accumulator().Hex()) + } + } + + // Open and verify the era file. + e, err := Open(f.Name()) + if err != nil { + t.Fatalf("failed to open era: %v", err) + } + defer e.Close() + + // Verify metadata. + if e.Start() != tt.start { + t.Fatalf("wrong start block: want %d, got %d", tt.start, e.Start()) + } + if e.Count() != uint64(totalBlocks) { + t.Fatalf("wrong block count: want %d, got %d", totalBlocks, e.Count()) + } + + // Verify accumulator in file. + if tt.accumulator { + accRoot, err := e.Accumulator() + if err != nil { + t.Fatalf("error getting accumulator: %v", err) + } + if accRoot != *builder.Accumulator() { + t.Fatalf("accumulator mismatch: builder has %s, file contains %s", + builder.Accumulator().Hex(), accRoot.Hex()) + } + } else { + if _, err := e.Accumulator(); err == nil { + t.Fatal("expected error when reading accumulator from post-merge epoch") + } + } + + // Verify blocks via raw iterator. + it, err := NewRawIterator(e) + if err != nil { + t.Fatalf("failed to make iterator: %v", err) + } + for i := 0; i < totalBlocks; i++ { + if !it.Next() { + t.Fatalf("expected more entries at %d", i) + } + if it.Error() != nil { + t.Fatalf("unexpected error: %v", it.Error()) + } + + // Check header. + rawHeader, err := io.ReadAll(it.Header) + if err != nil { + t.Fatalf("error reading header: %v", err) + } + if !bytes.Equal(rawHeader, blocks[i].header) { + t.Fatalf("mismatched header at %d", i) + } + + // Check body. + rawBody, err := io.ReadAll(it.Body) + if err != nil { + t.Fatalf("error reading body: %v", err) + } + if !bytes.Equal(rawBody, blocks[i].body) { + t.Fatalf("mismatched body at %d", i) + } + + // Check receipts. + rawReceipts, err := io.ReadAll(it.Receipts) + if err != nil { + t.Fatalf("error reading receipts: %v", err) + } + if !bytes.Equal(rawReceipts, blocks[i].receipts) { + t.Fatalf("mismatched receipts at %d", i) + } + + // Check TD (only for epochs that have TD stored). + if tt.preMerge > 0 && it.TotalDifficulty != nil { + rawTd, err := io.ReadAll(it.TotalDifficulty) + if err != nil { + t.Fatalf("error reading TD: %v", err) + } + slices.Reverse(rawTd) + td := new(big.Int).SetBytes(rawTd) + var expectedTD *big.Int + if i < tt.preMerge { + expectedTD = blocks[i].td + } else { + // Post-merge blocks in transition epoch use final TD. + expectedTD = finalTD + } + if td.Cmp(expectedTD) != 0 { + t.Fatalf("mismatched TD at %d: want %s, got %s", i, expectedTD, td) + } + } + } + + // Verify random access. + for _, blockNum := range []uint64{tt.start, tt.start + uint64(totalBlocks) - 1} { + blk, err := e.GetBlockByNumber(blockNum) + if err != nil { + t.Fatalf("error getting block %d: %v", blockNum, err) + } + if blk.Number().Uint64() != blockNum { + t.Fatalf("wrong block number: want %d, got %d", blockNum, blk.Number().Uint64()) + } + } + + // Verify out-of-range access fails. + if _, err := e.GetBlockByNumber(tt.start + uint64(totalBlocks)); err == nil { + t.Fatal("expected error for out-of-range block") + } + if tt.start > 0 { + if _, err := e.GetBlockByNumber(tt.start - 1); err == nil { + t.Fatal("expected error for block before start") + } + } + + // Verify high-level iterator. + hlIt, err := e.Iterator() + if err != nil { + t.Fatalf("failed to create iterator: %v", err) + } + count := 0 + for hlIt.Next() { + blk, err := hlIt.Block() + if err != nil { + t.Fatalf("error getting block: %v", err) + } + if blk.Number().Uint64() != tt.start+uint64(count) { + t.Fatalf("wrong block number: want %d, got %d", tt.start+uint64(count), blk.Number().Uint64()) + } + count++ + } + if hlIt.Error() != nil { + t.Fatalf("iterator error: %v", hlIt.Error()) + } + if count != totalBlocks { + t.Fatalf("wrong iteration count: want %d, got %d", totalBlocks, count) + } + }) + } +} + +// TestInitialTD tests the InitialTD calculation separately since it requires +// specific TD/difficulty values. +func TestInitialTD(t *testing.T) { + t.Parallel() + + f, err := os.CreateTemp(t.TempDir(), "erae-initial-td-test") + if err != nil { + t.Fatalf("error creating temp file: %v", err) + } + defer f.Close() + + builder := NewBuilder(f) + + // First block: difficulty=5, TD=10, so initial TD = 10-5 = 5. + header := mustEncode(&types.Header{Number: big.NewInt(0), Difficulty: big.NewInt(5)}) + body := mustEncode(&types.Body{}) + receipts := mustEncode([]types.SlimReceipt{}) + + if err := builder.AddRLP(header, body, receipts, 0, common.Hash{0}, big.NewInt(10), big.NewInt(5)); err != nil { + t.Fatalf("error adding block: %v", err) + } + + // Second block: difficulty=3, TD=13. + header2 := mustEncode(&types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(3)}) + if err := builder.AddRLP(header2, body, receipts, 1, common.Hash{1}, big.NewInt(13), big.NewInt(3)); err != nil { + t.Fatalf("error adding block: %v", err) + } + + if _, err := builder.Finalize(); err != nil { + t.Fatalf("error finalizing: %v", err) + } + + e, err := Open(f.Name()) + if err != nil { + t.Fatalf("failed to open era: %v", err) + } + defer e.Close() + + initialTD, err := e.InitialTD() + if err != nil { + t.Fatalf("error getting initial TD: %v", err) + } + + // Initial TD should be TD[0] - Difficulty[0] = 10 - 5 = 5. + if initialTD.Cmp(big.NewInt(5)) != 0 { + t.Fatalf("wrong initial TD: want 5, got %s", initialTD) + } +} + +func mustEncode(obj any) []byte { + b, err := rlp.EncodeToBytes(obj) + if err != nil { + panic(fmt.Sprintf("failed to encode obj: %v", err)) + } + return b +} diff --git a/internal/era/execdb/iterator.go b/internal/era/execdb/iterator.go new file mode 100644 index 0000000000..8d17ac00a9 --- /dev/null +++ b/internal/era/execdb/iterator.go @@ -0,0 +1,240 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package execdb + +import ( + "errors" + "io" + "math/big" + "slices" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/era/e2store" + "github.com/ethereum/go-ethereum/rlp" + "github.com/klauspost/compress/snappy" +) + +type Iterator struct { + inner *RawIterator + block *types.Block // cache for decoded block +} + +// NewIterator returns a header/body/receipt iterator over the archive. +// Call Next immediately to position on the first block. +func NewIterator(e era.Era) (era.Iterator, error) { + inner, err := NewRawIterator(e.(*Era)) + if err != nil { + return nil, err + } + return &Iterator{inner: inner}, nil +} + +// Next advances to the next block entry. +func (it *Iterator) Next() bool { + it.block = nil + return it.inner.Next() +} + +// Number is the number of the block currently loaded. +func (it *Iterator) Number() uint64 { return it.inner.next - 1 } + +// Error returns any iteration error (EOF is reported as nil, identical +// to the Era‑1 iterator behaviour). +func (it *Iterator) Error() error { return it.inner.Error() } + +// Block decodes the current header+body into a *types.Block. +func (it *Iterator) Block() (*types.Block, error) { + if it.block != nil { + return it.block, nil + } + if it.inner.Header == nil || it.inner.Body == nil { + return nil, errors.New("header and body must be non‑nil") + } + var ( + h types.Header + b types.Body + ) + if err := rlp.Decode(it.inner.Header, &h); err != nil { + return nil, err + } + if err := rlp.Decode(it.inner.Body, &b); err != nil { + return nil, err + } + it.block = types.NewBlockWithHeader(&h).WithBody(b) + return it.block, nil +} + +// Receipts decodes receipts for the current block. +func (it *Iterator) Receipts() (types.Receipts, error) { + block, err := it.Block() + if err != nil { + return nil, err + } + if it.inner.Receipts == nil { + return nil, errors.New("receipts must be non‑nil") + } + var rs []*types.SlimReceipt + if err := rlp.Decode(it.inner.Receipts, &rs); err != nil { + return nil, err + } + if len(rs) != len(block.Transactions()) { + return nil, errors.New("number of txs does not match receipts") + } + receipts := make([]*types.Receipt, len(rs)) + for i, receipt := range rs { + receipts[i] = (*types.Receipt)(receipt) + receipts[i].Bloom = types.CreateBloom(receipts[i]) + } + return receipts, nil +} + +// BlockAndReceipts is a convenience wrapper. +func (it *Iterator) BlockAndReceipts() (*types.Block, types.Receipts, error) { + b, err := it.Block() + if err != nil { + return nil, nil, err + } + r, err := it.Receipts() + if err != nil { + return nil, nil, err + } + return b, r, nil +} + +// TotalDifficulty returns the TD at the current position (if present). +func (it *Iterator) TotalDifficulty() (*big.Int, error) { + if it.inner.TotalDifficulty == nil { + return nil, errors.New("total‑difficulty stream is nil") + } + tdBytes, err := io.ReadAll(it.inner.TotalDifficulty) + if err != nil { + return nil, err + } + slices.Reverse(tdBytes) + return new(big.Int).SetBytes(tdBytes), nil +} + +// ----------------------------------------------------------------------------- +// Low‑level iterator (raw TLV/offset handling, no decoding) +// ----------------------------------------------------------------------------- + +type RawIterator struct { + e *Era + next uint64 // next block to pull + err error + + Header io.Reader + Body io.Reader + Receipts io.Reader + TotalDifficulty io.Reader // nil when archive omits TD +} + +// NewRawIterator creates an iterator positioned *before* the first block. +func NewRawIterator(e *Era) (*RawIterator, error) { + return &RawIterator{e: e, next: e.m.start}, nil +} + +// Next loads the next block’s components; returns false on EOF or error. +func (it *RawIterator) Next() bool { + it.err = nil // clear previous error + + if it.next >= it.e.m.start+it.e.m.count { + it.clear() + return false + } + + headerOffset, err := it.e.headerOff(it.next) + if err != nil { + it.setErr(err) + return false + } + it.Header, _, err = newSnappyReader(it.e.s, era.TypeCompressedHeader, headerOffset) + if err != nil { + it.setErr(err) + return false + } + + bodyOffset, err := it.e.bodyOff(it.next) + if err != nil { + it.setErr(err) + return false + } + it.Body, _, err = newSnappyReader(it.e.s, era.TypeCompressedBody, bodyOffset) + if err != nil { + it.setErr(err) + return false + } + + receiptsOffset, err := it.e.receiptOff(it.next) + if err != nil { + it.setErr(err) + return false + } + it.Receipts, _, err = newSnappyReader(it.e.s, era.TypeCompressedSlimReceipts, receiptsOffset) + if err != nil { + it.setErr(err) + return false + } + + // Check if TD component is present in this file (pre-merge or merge-transition epoch). + if int(td) < int(it.e.m.components) { + tdOffset, err := it.e.tdOff(it.next) + if err != nil { + it.setErr(err) + return false + } + it.TotalDifficulty, _, err = it.e.s.ReaderAt(era.TypeTotalDifficulty, tdOffset) + if err != nil { + it.setErr(err) + return false + } + } else { + it.TotalDifficulty = nil + } + + it.next++ + return true +} + +func (it *RawIterator) Number() uint64 { return it.next - 1 } + +func (it *RawIterator) Error() error { + if it.err == io.EOF { + return nil + } + return it.err +} + +func (it *RawIterator) setErr(err error) { + it.err = err + it.clear() +} + +func (it *RawIterator) clear() { + it.Header, it.Body, it.Receipts, it.TotalDifficulty = nil, nil, nil, nil +} + +// newSnappyReader behaves like era.newSnappyReader: returns a snappy.Reader +// plus the length of the underlying TLV payload so callers can advance offsets. +func newSnappyReader(r *e2store.Reader, typ uint16, off int64) (io.Reader, int64, error) { + raw, n, err := r.ReaderAt(typ, off) + if err != nil { + return nil, 0, err + } + return snappy.NewReader(raw), int64(n), nil +} diff --git a/internal/era/execdb/reader.go b/internal/era/execdb/reader.go new file mode 100644 index 0000000000..d0aaad1748 --- /dev/null +++ b/internal/era/execdb/reader.go @@ -0,0 +1,296 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package execdb + +import ( + "encoding/binary" + "fmt" + "io" + "math/big" + "os" + "slices" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/era/e2store" + "github.com/ethereum/go-ethereum/rlp" + "github.com/klauspost/compress/snappy" +) + +// Era object represents an era file that contains blocks and their components. +type Era struct { + f era.ReadAtSeekCloser + s *e2store.Reader + m metadata // metadata for the Era file +} + +// Filename returns a recognizable filename for an EraE file. +// The filename uses the last block hash to uniquely identify the epoch's content. +func Filename(network string, epoch int, lastBlockHash common.Hash) string { + return fmt.Sprintf("%s-%05d-%s.erae", network, epoch, lastBlockHash.Hex()[2:10]) +} + +// Open accesses the era file. +func Open(path string) (*Era, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + e := &Era{f: f, s: e2store.NewReader(f)} + if err := e.loadIndex(); err != nil { + f.Close() + return nil, err + } + return e, nil +} + +// Close closes the era file safely. +func (e *Era) Close() error { + if e.f == nil { + return nil + } + err := e.f.Close() + e.f = nil + return err +} + +// From returns an Era backed by f. +func From(f era.ReadAtSeekCloser) (era.Era, error) { + e := &Era{f: f, s: e2store.NewReader(f)} + if err := e.loadIndex(); err != nil { + f.Close() + return nil, err + } + return e, nil +} + +// Start retrieves the starting block number. +func (e *Era) Start() uint64 { + return e.m.start +} + +// Count retrieves the count of blocks present. +func (e *Era) Count() uint64 { + return e.m.count +} + +// Iterator returns an iterator over the era file. +func (e *Era) Iterator() (era.Iterator, error) { + return NewIterator(e) +} + +// GetBlockByNumber retrieves the block if present within the era file. +func (e *Era) GetBlockByNumber(blockNum uint64) (*types.Block, error) { + h, err := e.GetHeader(blockNum) + if err != nil { + return nil, err + } + b, err := e.GetBody(blockNum) + if err != nil { + return nil, err + } + return types.NewBlockWithHeader(h).WithBody(*b), nil +} + +// GetHeader retrieves the header from the era file through the cached offset table. +func (e *Era) GetHeader(num uint64) (*types.Header, error) { + off, err := e.headerOff(num) + if err != nil { + return nil, err + } + + r, _, err := e.s.ReaderAt(era.TypeCompressedHeader, off) + if err != nil { + return nil, err + } + + r = snappy.NewReader(r) + var h types.Header + return &h, rlp.Decode(r, &h) +} + +// GetBody retrieves the body from the era file through cached offset table. +func (e *Era) GetBody(num uint64) (*types.Body, error) { + off, err := e.bodyOff(num) + if err != nil { + return nil, err + } + + r, _, err := e.s.ReaderAt(era.TypeCompressedBody, off) + if err != nil { + return nil, err + } + + r = snappy.NewReader(r) + var b types.Body + return &b, rlp.Decode(r, &b) +} + +// GetTD retrieves the td from the era file through cached offset table. +func (e *Era) GetTD(blockNum uint64) (*big.Int, error) { + off, err := e.tdOff(blockNum) + if err != nil { + return nil, err + } + r, _, err := e.s.ReaderAt(era.TypeTotalDifficulty, off) + if err != nil { + return nil, err + } + buf, _ := io.ReadAll(r) + slices.Reverse(buf) + td := new(big.Int).SetBytes(buf) + return td, nil +} + +// GetRawBodyByNumber returns the RLP-encoded body for the given block number. +func (e *Era) GetRawBodyByNumber(blockNum uint64) ([]byte, error) { + off, err := e.bodyOff(blockNum) + if err != nil { + return nil, err + } + r, _, err := e.s.ReaderAt(era.TypeCompressedBody, off) + if err != nil { + return nil, err + } + r = snappy.NewReader(r) + return io.ReadAll(r) +} + +// GetRawReceiptsByNumber returns the RLP-encoded receipts for the given block number. +func (e *Era) GetRawReceiptsByNumber(blockNum uint64) ([]byte, error) { + off, err := e.receiptOff(blockNum) + if err != nil { + return nil, err + } + r, _, err := e.s.ReaderAt(era.TypeCompressedSlimReceipts, off) + if err != nil { + return nil, err + } + r = snappy.NewReader(r) + return io.ReadAll(r) +} + +// InitialTD returns initial total difficulty before the difficulty of the +// first block of the Era is applied. Returns an error if TD is not available +// (e.g., post-merge epoch). +func (e *Era) InitialTD() (*big.Int, error) { + // Check if TD component exists. + if int(td) >= int(e.m.components) { + return nil, fmt.Errorf("total difficulty not available in this epoch") + } + + // Get first header to read its difficulty. + header, err := e.GetHeader(e.m.start) + if err != nil { + return nil, fmt.Errorf("read first header: %w", err) + } + + // Get TD after first block using the index. + firstTD, err := e.GetTD(e.m.start) + if err != nil { + return nil, fmt.Errorf("read first TD: %w", err) + } + + // Initial TD = TD[0] - Difficulty[0] + return new(big.Int).Sub(firstTD, header.Difficulty), nil +} + +// Accumulator reads the accumulator entry in the EraE file if it exists. +// Note that one premerge erae files will contain an accumulator entry. +func (e *Era) Accumulator() (common.Hash, error) { + entry, err := e.s.Find(era.TypeAccumulator) + if err != nil { + return common.Hash{}, err + } + return common.BytesToHash(entry.Value), nil +} + +// loadIndex loads in the index table containing all offsets and caches it. +func (e *Era) loadIndex() error { + var err error + e.m.length, err = e.f.Seek(0, io.SeekEnd) + if err != nil { + return err + } + + b := make([]byte, 16) + if _, err = e.f.ReadAt(b, e.m.length-16); err != nil { + return err + } + e.m.components = binary.LittleEndian.Uint64(b[0:8]) + e.m.count = binary.LittleEndian.Uint64(b[8:16]) + + payloadlen := 8 + 8*e.m.count*e.m.components + 16 // 8 for start block, 8 per property per block, 16 for the number of properties and the number of blocks + tlvstart := e.m.length - int64(payloadlen) - 8 + _, err = e.f.ReadAt(b[:8], tlvstart+8) + if err != nil { + return err + } + + e.m.start = binary.LittleEndian.Uint64(b[:8]) + return nil +} + +// headerOff, bodyOff, receiptOff, and tdOff return the offsets of the respective components for a given block number. +func (e *Era) headerOff(num uint64) (int64, error) { return e.indexOffset(num, header) } +func (e *Era) bodyOff(num uint64) (int64, error) { return e.indexOffset(num, body) } +func (e *Era) receiptOff(num uint64) (int64, error) { return e.indexOffset(num, receipts) } +func (e *Era) tdOff(num uint64) (int64, error) { return e.indexOffset(num, td) } + +// indexOffset calculates offset to a certain component for a block number within a file. +func (e *Era) indexOffset(n uint64, component componentType) (int64, error) { + if n < e.m.start || n >= e.m.start+e.m.count { + return 0, fmt.Errorf("block %d out of range [%d,%d)", n, e.m.start, e.m.start+e.m.count) + } + if int(component) >= int(e.m.components) { + return 0, fmt.Errorf("component %d not present", component) + } + + payloadlen := 8 + 8*e.m.count*e.m.components + 16 // 8 for start block, 8 per property per block, 16 for the number of properties and the number of blocks + indstart := e.m.length - int64(payloadlen) - 8 + + rec := (n-e.m.start)*e.m.components + uint64(component) + pos := indstart + 8 + 8 + int64(rec*8) + + var buf [8]byte + if _, err := e.f.ReadAt(buf[:], pos); err != nil { + return 0, err + } + rel := binary.LittleEndian.Uint64(buf[:]) + return int64(rel) + indstart, nil +} + +// metadata contains the information about the era file that is written into the file. +type metadata struct { + start uint64 // start block number + count uint64 // number of blocks in the era + components uint64 // number of properties + length int64 // length of the file in bytes +} + +// componentType represents the integer form of a specific type that can be present in the era file. +type componentType int + +// header, body, receipts, td, and proof are the different types of components that can be present in the era file. +const ( + header componentType = iota + body + receipts + td + proof +) diff --git a/internal/era/builder.go b/internal/era/onedb/builder.go similarity index 84% rename from internal/era/builder.go rename to internal/era/onedb/builder.go index 975561564c..6497bf4f60 100644 --- a/internal/era/builder.go +++ b/internal/era/onedb/builder.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package era +package onedb import ( "bytes" @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era" "github.com/ethereum/go-ethereum/internal/era/e2store" "github.com/ethereum/go-ethereum/rlp" "github.com/golang/snappy" @@ -72,20 +73,22 @@ import ( // Due to the accumulator size limit of 8192, the maximum number of blocks in // an Era1 batch is also 8192. type Builder struct { - w *e2store.Writer - startNum *uint64 - startTd *big.Int - indexes []uint64 - hashes []common.Hash - tds []*big.Int - written int + w *e2store.Writer + startNum *uint64 + startTd *big.Int + indexes []uint64 + hashes []common.Hash + tds []*big.Int + accumulator *common.Hash // accumulator root, set by Finalize + + written int buf *bytes.Buffer snappy *snappy.Writer } // NewBuilder returns a new Builder instance. -func NewBuilder(w io.Writer) *Builder { +func NewBuilder(w io.Writer) era.Builder { buf := bytes.NewBuffer(nil) return &Builder{ w: e2store.NewWriter(w), @@ -117,7 +120,7 @@ func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int) func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, hash common.Hash, td, difficulty *big.Int) error { // Write Era1 version entry before first block. if b.startNum == nil { - n, err := b.w.Write(TypeVersion, nil) + n, err := b.w.Write(era.TypeVersion, nil) if err != nil { return err } @@ -126,8 +129,8 @@ func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, hash comm b.startTd = new(big.Int).Sub(td, difficulty) b.written += n } - if len(b.indexes) >= MaxEra1Size { - return fmt.Errorf("exceeds maximum batch size of %d", MaxEra1Size) + if len(b.indexes) >= era.MaxSize { + return fmt.Errorf("exceeds maximum batch size of %d", era.MaxSize) } b.indexes = append(b.indexes, uint64(b.written)) @@ -135,19 +138,19 @@ func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, hash comm b.tds = append(b.tds, td) // Write block data. - if err := b.snappyWrite(TypeCompressedHeader, header); err != nil { + if err := b.snappyWrite(era.TypeCompressedHeader, header); err != nil { return err } - if err := b.snappyWrite(TypeCompressedBody, body); err != nil { + if err := b.snappyWrite(era.TypeCompressedBody, body); err != nil { return err } - if err := b.snappyWrite(TypeCompressedReceipts, receipts); err != nil { + if err := b.snappyWrite(era.TypeCompressedReceipts, receipts); err != nil { return err } // Also write total difficulty, but don't snappy encode. - btd := bigToBytes32(td) - n, err := b.w.Write(TypeTotalDifficulty, btd[:]) + btd := era.BigToBytes32(td) + n, err := b.w.Write(era.TypeTotalDifficulty, btd[:]) b.written += n if err != nil { return err @@ -157,21 +160,24 @@ func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, hash comm } // Finalize computes the accumulator and block index values, then writes the -// corresponding e2store entries. +// corresponding e2store entries. Era1 always has an accumulator, so this +// always returns a valid hash. func (b *Builder) Finalize() (common.Hash, error) { if b.startNum == nil { return common.Hash{}, errors.New("finalize called on empty builder") } // Compute accumulator root and write entry. - root, err := ComputeAccumulator(b.hashes, b.tds) + root, err := era.ComputeAccumulator(b.hashes, b.tds) if err != nil { return common.Hash{}, fmt.Errorf("error calculating accumulator root: %w", err) } - n, err := b.w.Write(TypeAccumulator, root[:]) + n, err := b.w.Write(era.TypeAccumulator, root[:]) b.written += n if err != nil { return common.Hash{}, fmt.Errorf("error writing accumulator: %w", err) } + b.accumulator = &root + // Get beginning of index entry to calculate block relative offset. base := int64(b.written) @@ -196,13 +202,19 @@ func (b *Builder) Finalize() (common.Hash, error) { binary.LittleEndian.PutUint64(index[8+count*8:], uint64(count)) // Finally, write the block index entry. - if _, err := b.w.Write(TypeBlockIndex, index); err != nil { + if _, err := b.w.Write(era.TypeBlockIndex, index); err != nil { return common.Hash{}, fmt.Errorf("unable to write block index: %w", err) } return root, nil } +// Accumulator returns the accumulator root after Finalize has been called. +// For Era1, this always returns a non-nil value since all blocks are pre-merge. +func (b *Builder) Accumulator() *common.Hash { + return b.accumulator +} + // snappyWrite is a small helper to take care snappy encoding and writing an e2store entry. func (b *Builder) snappyWrite(typ uint16, in []byte) error { var ( diff --git a/internal/era/era_test.go b/internal/era/onedb/builder_test.go similarity index 95% rename from internal/era/era_test.go rename to internal/era/onedb/builder_test.go index 31fa0076a6..bc7a1d9e63 100644 --- a/internal/era/era_test.go +++ b/internal/era/onedb/builder_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package era +package onedb import ( "bytes" @@ -22,6 +22,7 @@ import ( "io" "math/big" "os" + "slices" "testing" "github.com/ethereum/go-ethereum/common" @@ -82,7 +83,11 @@ func TestEra1Builder(t *testing.T) { t.Fatalf("failed to open era: %v", err) } defer e.Close() - it, err := NewRawIterator(e) + eraPtr, ok := e.(*Era) + if !ok { + t.Fatalf("failed to assert *Era type") + } + it, err := NewRawIterator(eraPtr) if err != nil { t.Fatalf("failed to make iterator: %s", err) } @@ -119,7 +124,7 @@ func TestEra1Builder(t *testing.T) { if !bytes.Equal(rawReceipts, chain.receipts[i]) { t.Fatalf("mismatched receipts: want %s, got %s", chain.receipts[i], rawReceipts) } - receipts, err := getReceiptsByNumber(e, i) + receipts, err := getReceiptsByNumber(eraPtr, i) if err != nil { t.Fatalf("error reading receipts: %v", err) } @@ -136,7 +141,8 @@ func TestEra1Builder(t *testing.T) { if err != nil { t.Fatalf("error reading td: %v", err) } - td := new(big.Int).SetBytes(reverseOrder(rawTd)) + slices.Reverse(rawTd) + td := new(big.Int).SetBytes(rawTd) if td.Cmp(chain.tds[i]) != 0 { t.Fatalf("mismatched tds: want %s, got %s", chain.tds[i], td) } diff --git a/internal/era/iterator.go b/internal/era/onedb/iterator.go similarity index 89% rename from internal/era/iterator.go rename to internal/era/onedb/iterator.go index 3c4f82d850..b80fbabbc5 100644 --- a/internal/era/iterator.go +++ b/internal/era/onedb/iterator.go @@ -14,14 +14,16 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package era +package onedb import ( "errors" "io" "math/big" + "slices" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era" "github.com/ethereum/go-ethereum/rlp" ) @@ -32,8 +34,8 @@ type Iterator struct { // NewIterator returns a new Iterator instance. Next must be immediately // called on new iterators to load the first item. -func NewIterator(e *Era) (*Iterator, error) { - inner, err := NewRawIterator(e) +func NewIterator(e era.Era) (era.Iterator, error) { + inner, err := NewRawIterator(e.(*Era)) if err != nil { return nil, err } @@ -107,7 +109,8 @@ func (it *Iterator) TotalDifficulty() (*big.Int, error) { if err != nil { return nil, err } - return new(big.Int).SetBytes(reverseOrder(td)), nil + slices.Reverse(td) + return new(big.Int).SetBytes(td), nil } // RawIterator reads an RLP-encode Era1 entries. @@ -151,22 +154,22 @@ func (it *RawIterator) Next() bool { return false } var n int64 - if it.Header, n, it.err = newSnappyReader(it.e.s, TypeCompressedHeader, off); it.err != nil { + if it.Header, n, it.err = newSnappyReader(it.e.s, era.TypeCompressedHeader, off); it.err != nil { it.clear() return true } off += n - if it.Body, n, it.err = newSnappyReader(it.e.s, TypeCompressedBody, off); it.err != nil { + if it.Body, n, it.err = newSnappyReader(it.e.s, era.TypeCompressedBody, off); it.err != nil { it.clear() return true } off += n - if it.Receipts, n, it.err = newSnappyReader(it.e.s, TypeCompressedReceipts, off); it.err != nil { + if it.Receipts, n, it.err = newSnappyReader(it.e.s, era.TypeCompressedReceipts, off); it.err != nil { it.clear() return true } off += n - if it.TotalDifficulty, _, it.err = it.e.s.ReaderAt(TypeTotalDifficulty, off); it.err != nil { + if it.TotalDifficulty, _, it.err = it.e.s.ReaderAt(era.TypeTotalDifficulty, off); it.err != nil { it.clear() return true } diff --git a/internal/era/onedb/reader.go b/internal/era/onedb/reader.go new file mode 100644 index 0000000000..df93ca8211 --- /dev/null +++ b/internal/era/onedb/reader.go @@ -0,0 +1,279 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package onedb + +import ( + "encoding/binary" + "fmt" + "io" + "math/big" + "os" + "slices" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/era/e2store" + "github.com/ethereum/go-ethereum/rlp" + "github.com/golang/snappy" +) + +// Filename returns a recognizable Era1-formatted file name for the specified +// epoch and network. +func Filename(network string, epoch int, root common.Hash) string { + return fmt.Sprintf("%s-%05d-%s.era1", network, epoch, root.Hex()[2:10]) +} + +type ReadAtSeekCloser interface { + io.ReaderAt + io.Seeker + io.Closer +} + +// Era reads and Era1 file. +type Era struct { + f ReadAtSeekCloser // backing era1 file + s *e2store.Reader // e2store reader over f + m metadata // start, count, length info + mu *sync.Mutex // lock for buf + buf [8]byte // buffer reading entry offsets +} + +// From returns an Era backed by f. +func From(f era.ReadAtSeekCloser) (era.Era, error) { + m, err := readMetadata(f) + if err != nil { + return nil, err + } + return &Era{ + f: f, + s: e2store.NewReader(f), + m: m, + mu: new(sync.Mutex), + }, nil +} + +// Open returns an Era backed by the given filename. +func Open(filename string) (era.Era, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + return From(f) +} + +func (e *Era) Close() error { + return e.f.Close() +} + +// Iterator returns an iterator over the era file. +func (e *Era) Iterator() (era.Iterator, error) { + return NewIterator(e) +} + +// GetBlockByNumber returns the block for the given block number. +func (e *Era) GetBlockByNumber(num uint64) (*types.Block, error) { + if e.m.start > num || e.m.start+e.m.count <= num { + return nil, fmt.Errorf("out-of-bounds: %d not in [%d, %d)", num, e.m.start, e.m.start+e.m.count) + } + off, err := e.readOffset(num) + if err != nil { + return nil, err + } + r, n, err := newSnappyReader(e.s, era.TypeCompressedHeader, off) + if err != nil { + return nil, err + } + var header types.Header + if err := rlp.Decode(r, &header); err != nil { + return nil, err + } + off += n + r, _, err = newSnappyReader(e.s, era.TypeCompressedBody, off) + if err != nil { + return nil, err + } + var body types.Body + if err := rlp.Decode(r, &body); err != nil { + return nil, err + } + return types.NewBlockWithHeader(&header).WithBody(body), nil +} + +// GetRawBodyByNumber returns the RLP-encoded body for the given block number. +func (e *Era) GetRawBodyByNumber(num uint64) ([]byte, error) { + if e.m.start > num || e.m.start+e.m.count <= num { + return nil, fmt.Errorf("out-of-bounds: %d not in [%d, %d)", num, e.m.start, e.m.start+e.m.count) + } + off, err := e.readOffset(num) + if err != nil { + return nil, err + } + off, err = e.s.SkipN(off, 1) + if err != nil { + return nil, err + } + r, _, err := newSnappyReader(e.s, era.TypeCompressedBody, off) + if err != nil { + return nil, err + } + return io.ReadAll(r) +} + +// GetRawReceiptsByNumber returns the RLP-encoded receipts for the given block number. +func (e *Era) GetRawReceiptsByNumber(num uint64) ([]byte, error) { + if e.m.start > num || e.m.start+e.m.count <= num { + return nil, fmt.Errorf("out-of-bounds: %d not in [%d, %d)", num, e.m.start, e.m.start+e.m.count) + } + off, err := e.readOffset(num) + if err != nil { + return nil, err + } + + // Skip over header and body. + off, err = e.s.SkipN(off, 2) + if err != nil { + return nil, err + } + + r, _, err := newSnappyReader(e.s, era.TypeCompressedReceipts, off) + if err != nil { + return nil, err + } + return io.ReadAll(r) +} + +// Accumulator reads the accumulator entry in the Era1 file. +func (e *Era) Accumulator() (common.Hash, error) { + entry, err := e.s.Find(era.TypeAccumulator) + if err != nil { + return common.Hash{}, err + } + return common.BytesToHash(entry.Value), nil +} + +// InitialTD returns initial total difficulty before the difficulty of the +// first block of the Era1 is applied. +func (e *Era) InitialTD() (*big.Int, error) { + var ( + r io.Reader + header types.Header + rawTd []byte + n int64 + off int64 + err error + ) + + // Read first header. + if off, err = e.readOffset(e.m.start); err != nil { + return nil, err + } + if r, n, err = newSnappyReader(e.s, era.TypeCompressedHeader, off); err != nil { + return nil, err + } + if err := rlp.Decode(r, &header); err != nil { + return nil, err + } + off += n + + // Skip over header and body. + off, err = e.s.SkipN(off, 2) + if err != nil { + return nil, err + } + + // Read total difficulty after first block. + if r, _, err = e.s.ReaderAt(era.TypeTotalDifficulty, off); err != nil { + return nil, err + } + rawTd, err = io.ReadAll(r) + if err != nil { + return nil, err + } + slices.Reverse(rawTd) + td := new(big.Int).SetBytes(rawTd) + return td.Sub(td, header.Difficulty), nil +} + +// Start returns the listed start block. +func (e *Era) Start() uint64 { + return e.m.start +} + +// Count returns the total number of blocks in the Era1. +func (e *Era) Count() uint64 { + return e.m.count +} + +// readOffset reads a specific block's offset from the block index. The value n +// is the absolute block number desired. +func (e *Era) readOffset(n uint64) (int64, error) { + var ( + blockIndexRecordOffset = e.m.length - 24 - int64(e.m.count)*8 // skips start, count, and header + firstIndex = blockIndexRecordOffset + 16 // first index after header / start-num + indexOffset = int64(n-e.m.start) * 8 // desired index * size of indexes + offOffset = firstIndex + indexOffset // offset of block offset + ) + e.mu.Lock() + defer e.mu.Unlock() + clear(e.buf[:]) + if _, err := e.f.ReadAt(e.buf[:], offOffset); err != nil { + return 0, err + } + // Since the block offset is relative from the start of the block index record + // we need to add the record offset to it's offset to get the block's absolute + // offset. + return blockIndexRecordOffset + int64(binary.LittleEndian.Uint64(e.buf[:])), nil +} + +// newSnappyReader returns a snappy.Reader for the e2store entry value at off. +func newSnappyReader(e *e2store.Reader, expectedType uint16, off int64) (io.Reader, int64, error) { + r, n, err := e.ReaderAt(expectedType, off) + if err != nil { + return nil, 0, err + } + return snappy.NewReader(r), int64(n), err +} + +// metadata wraps the metadata in the block index. +type metadata struct { + start uint64 + count uint64 + length int64 +} + +// readMetadata reads the metadata stored in an Era1 file's block index. +func readMetadata(f ReadAtSeekCloser) (m metadata, err error) { + // Determine length of reader. + if m.length, err = f.Seek(0, io.SeekEnd); err != nil { + return + } + b := make([]byte, 16) + // Read count. It's the last 8 bytes of the file. + if _, err = f.ReadAt(b[:8], m.length-8); err != nil { + return + } + m.count = binary.LittleEndian.Uint64(b) + // Read start. It's at the offset -sizeof(m.count) - + // count*sizeof(indexEntry) - sizeof(m.start) + if _, err = f.ReadAt(b[8:], m.length-16-int64(m.count*8)); err != nil { + return + } + m.start = binary.LittleEndian.Uint64(b[8:]) + return +} diff --git a/internal/era/proof.go b/internal/era/proof.go new file mode 100644 index 0000000000..464d9e6fe5 --- /dev/null +++ b/internal/era/proof.go @@ -0,0 +1,36 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . +package era + +import ( + "io" + + "github.com/ethereum/go-ethereum/rlp" +) + +type ProofVariant uint16 + +const ( + ProofNone ProofVariant = iota +) + +// Proof is the interface for all block proof types in the package. +// It's a stub for later integration into Era. +type Proof interface { + EncodeRLP(w io.Writer) error + DecodeRLP(s *rlp.Stream) error + Variant() ProofVariant +} diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 0b2e3ad4dc..6d1f9a14e2 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -49,9 +49,16 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/trie" ) +// maxGetStorageSlots is the maximum total number of storage slots that can +// be requested in a single eth_getStorageValues call. +const maxGetStorageSlots = 1024 + +// maxGetProofKeys is the maximum number of storage keys that can be +// requested in a single eth_getProof call. +const maxGetProofKeys = 1024 + var errBlobTxNotSupported = errors.New("signing blob transactions not supported") var errSubClosed = errors.New("chain subscription closed") @@ -179,6 +186,15 @@ func NewTxPoolAPI(b Backend) *TxPoolAPI { return &TxPoolAPI{b} } +// flattenTxs builds the RPC transaction map keyed by nonce for a set of pool txs. +func flattenTxs(txs types.Transactions, header *types.Header, cfg *params.ChainConfig) map[string]*RPCTransaction { + dump := make(map[string]*RPCTransaction, len(txs)) + for _, tx := range txs { + dump[fmt.Sprintf("%d", tx.Nonce())] = NewRPCPendingTransaction(tx, header, cfg) + } + return dump +} + // Content returns the transactions contained within the transaction pool. func (api *TxPoolAPI) Content() map[string]map[string]map[string]*RPCTransaction { pending, queue := api.b.TxPoolContent() @@ -189,19 +205,11 @@ func (api *TxPoolAPI) Content() map[string]map[string]map[string]*RPCTransaction curHeader := api.b.CurrentHeader() // Flatten the pending transactions for account, txs := range pending { - dump := make(map[string]*RPCTransaction, len(txs)) - for _, tx := range txs { - dump[fmt.Sprintf("%d", tx.Nonce())] = NewRPCPendingTransaction(tx, curHeader, api.b.ChainConfig()) - } - content["pending"][account.Hex()] = dump + content["pending"][account.Hex()] = flattenTxs(txs, curHeader, api.b.ChainConfig()) } // Flatten the queued transactions for account, txs := range queue { - dump := make(map[string]*RPCTransaction, len(txs)) - for _, tx := range txs { - dump[fmt.Sprintf("%d", tx.Nonce())] = NewRPCPendingTransaction(tx, curHeader, api.b.ChainConfig()) - } - content["queued"][account.Hex()] = dump + content["queued"][account.Hex()] = flattenTxs(txs, curHeader, api.b.ChainConfig()) } return content } @@ -213,18 +221,10 @@ func (api *TxPoolAPI) ContentFrom(addr common.Address) map[string]map[string]*RP curHeader := api.b.CurrentHeader() // Build the pending transactions - dump := make(map[string]*RPCTransaction, len(pending)) - for _, tx := range pending { - dump[fmt.Sprintf("%d", tx.Nonce())] = NewRPCPendingTransaction(tx, curHeader, api.b.ChainConfig()) - } - content["pending"] = dump + content["pending"] = flattenTxs(pending, curHeader, api.b.ChainConfig()) // Build the queued transactions - dump = make(map[string]*RPCTransaction, len(queue)) - for _, tx := range queue { - dump[fmt.Sprintf("%d", tx.Nonce())] = NewRPCPendingTransaction(tx, curHeader, api.b.ChainConfig()) - } - content["queued"] = dump + content["queued"] = flattenTxs(queue, curHeader, api.b.ChainConfig()) return content } @@ -364,6 +364,9 @@ func (n *proofList) Delete(key []byte) error { // GetProof returns the Merkle-proof for a given account and optionally some storage keys. func (api *BlockChainAPI) GetProof(ctx context.Context, address common.Address, storageKeys []string, blockNrOrHash rpc.BlockNumberOrHash) (*AccountResult, error) { + if len(storageKeys) > maxGetProofKeys { + return nil, &invalidParamsError{fmt.Sprintf("too many storage keys requested (max %d, got %d)", maxGetProofKeys, len(storageKeys))} + } var ( keys = make([]common.Hash, len(storageKeys)) keyLengths = make([]int, len(storageKeys)) @@ -372,9 +375,9 @@ func (api *BlockChainAPI) GetProof(ctx context.Context, address common.Address, // Deserialize all keys. This prevents state access on invalid input. for i, hexKey := range storageKeys { var err error - keys[i], keyLengths[i], err = decodeHash(hexKey) + keys[i], keyLengths[i], err = decodeStorageKey(hexKey) if err != nil { - return nil, err + return nil, &invalidParamsError{fmt.Sprintf("%v: %q", err, hexKey)} } } statedb, header, err := api.b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) @@ -387,8 +390,7 @@ func (api *BlockChainAPI) GetProof(ctx context.Context, address common.Address, if len(keys) > 0 { var storageTrie state.Trie if storageRoot != types.EmptyRootHash && storageRoot != (common.Hash{}) { - id := trie.StorageTrieID(header.Root, crypto.Keccak256Hash(address.Bytes()), storageRoot) - st, err := trie.NewStateTrie(id, statedb.Database().TrieDB()) + st, err := statedb.Database().OpenStorageTrie(header.Root, address, storageRoot, nil) if err != nil { return nil, err } @@ -396,6 +398,9 @@ func (api *BlockChainAPI) GetProof(ctx context.Context, address common.Address, } // Create the proofs for the storageKeys. for i, key := range keys { + if err := ctx.Err(); err != nil { + return nil, err + } // Output key encoding is a bit special: if the input was a 32-byte hash, it is // returned as such. Otherwise, we apply the QUANTITY encoding mandated by the // JSON-RPC spec for getProof. This behavior exists to preserve backwards @@ -419,7 +424,7 @@ func (api *BlockChainAPI) GetProof(ctx context.Context, address common.Address, } } // Create the accountProof. - tr, err := trie.NewStateTrie(trie.StateTrieID(header.Root), statedb.Database().TrieDB()) + tr, err := statedb.Database().OpenTrie(header.Root) if err != nil { return nil, err } @@ -439,9 +444,10 @@ func (api *BlockChainAPI) GetProof(ctx context.Context, address common.Address, }, statedb.Error() } -// decodeHash parses a hex-encoded 32-byte hash. The input may optionally -// be prefixed by 0x and can have a byte length up to 32. -func decodeHash(s string) (h common.Hash, inputLength int, err error) { +// decodeStorageKey parses a hex-encoded 32-byte hash. +// For legacy compatibility reasons, we parse these keys leniently, +// with the 0x prefix being optional. +func decodeStorageKey(s string) (h common.Hash, inputLength int, err error) { if strings.HasPrefix(s, "0x") || strings.HasPrefix(s, "0X") { s = s[2:] } @@ -449,11 +455,11 @@ func decodeHash(s string) (h common.Hash, inputLength int, err error) { s = "0" + s } if len(s) > 64 { - return common.Hash{}, len(s) / 2, errors.New("hex string too long, want at most 32 bytes") + return common.Hash{}, len(s) / 2, errors.New("storage key too long (want at most 32 bytes)") } b, err := hex.DecodeString(s) if err != nil { - return common.Hash{}, 0, errors.New("hex string invalid") + return common.Hash{}, 0, errors.New("invalid hex in storage key") } return common.BytesToHash(b), len(b), nil } @@ -601,9 +607,9 @@ func (api *BlockChainAPI) GetCode(ctx context.Context, address common.Address, b // block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta block // numbers are also allowed. func (api *BlockChainAPI) GetStorageAt(ctx context.Context, address common.Address, hexKey string, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { - key, _, err := decodeHash(hexKey) + key, _, err := decodeStorageKey(hexKey) if err != nil { - return nil, fmt.Errorf("unable to decode storage key: %s", err) + return nil, &invalidParamsError{fmt.Sprintf("%v: %q", err, hexKey)} } state, _, err := api.b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) if state == nil || err != nil { @@ -618,6 +624,41 @@ func (api *BlockChainAPI) GetStorageAt(ctx context.Context, address common.Addre return res[:], state.Error() } +// GetStorageValues returns multiple storage slot values for multiple accounts +// at the given block. +func (api *BlockChainAPI) GetStorageValues(ctx context.Context, requests map[common.Address][]common.Hash, blockNrOrHash rpc.BlockNumberOrHash) (map[common.Address][]hexutil.Bytes, error) { + // Count total slots requested. + var totalSlots int + for _, keys := range requests { + totalSlots += len(keys) + if totalSlots > maxGetStorageSlots { + return nil, &clientLimitExceededError{message: fmt.Sprintf("too many slots (max %d)", maxGetStorageSlots)} + } + } + if totalSlots == 0 { + return nil, &invalidParamsError{message: "empty request"} + } + + state, _, err := api.b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) + if state == nil || err != nil { + return nil, err + } + + result := make(map[common.Address][]hexutil.Bytes, len(requests)) + for addr, keys := range requests { + vals := make([]hexutil.Bytes, len(keys)) + for i, key := range keys { + v := state.GetState(addr, key) + vals[i] = v[:] + } + if err := state.Error(); err != nil { + return nil, err + } + result[addr] = vals + } + return result, nil +} + // GetBlockReceipts returns the block receipts for the given block hash or number or tag. func (api *BlockChainAPI) GetBlockReceipts(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) ([]map[string]interface{}, error) { var ( @@ -740,11 +781,10 @@ func doCall(ctx context.Context, b Backend, args TransactionArgs, state *state.S // Make sure the context is cancelled when the call has completed // this makes sure resources are cleaned up. defer cancel() - gp := new(core.GasPool) + + gp := core.NewGasPool(globalGasCap) if globalGasCap == 0 { - gp.AddGas(gomath.MaxUint64) - } else { - gp.AddGas(globalGasCap) + gp = core.NewGasPool(gomath.MaxUint64) } return applyMessage(ctx, b, args, state, header, timeout, gp, &blockCtx, &vm.Config{NoBaseFee: true}, precompiles, runCtx) } @@ -763,14 +803,14 @@ func applyMessage(ctx context.Context, b Backend, args TransactionArgs, state *s return nil, err } if gp.Gas() < gomath.MaxUint64-postingGas { - gp.AddGas(postingGas) + _ = gp.ReturnGas(postingGas, 0) // error can be ignored as we are checking for overflow already } else { log.Debug("gas pool overflow", "gpGas", gp.Gas(), "baseFee", header.BaseFee, "postingGas", postingGas, "msgDataLen", len(msg.Data)) - gp.SetGas(gomath.MaxUint64) + gp = core.NewGasPool(gomath.MaxUint64) } } if msg.GasLimit > gp.Gas() { - gp.SetGas(msg.GasLimit) + gp = core.NewGasPool(msg.GasLimit) } // Arbitrum: support NodeInterface.sol by swapping out the message if needed var res *core.ExecutionResult @@ -895,6 +935,16 @@ func (api *BlockChainAPI) SimulateV1(ctx context.Context, opts simOpts, blockNrO } else if len(opts.BlockStateCalls) > maxSimulateBlocks { return nil, &clientLimitExceededError{message: "too many blocks"} } + var totalCalls int + for _, block := range opts.BlockStateCalls { + if len(block.Calls) > maxSimulateCallsPerBlock { + return nil, &clientLimitExceededError{message: fmt.Sprintf("too many calls in block: %d > %d", len(block.Calls), maxSimulateCallsPerBlock)} + } + totalCalls += len(block.Calls) + if totalCalls > maxSimulateTotalCalls { + return nil, &clientLimitExceededError{message: fmt.Sprintf("too many calls: %d > %d", totalCalls, maxSimulateTotalCalls)} + } + } if blockNrOrHash == nil { n := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) blockNrOrHash = &n @@ -903,17 +953,12 @@ func (api *BlockChainAPI) SimulateV1(ctx context.Context, opts simOpts, blockNrO if state == nil || err != nil { return nil, err } - gasCap := api.b.RPCGasCap() - if gasCap == 0 { - gasCap = gomath.MaxUint64 - } sim := &simulator{ - b: api.b, - state: state, - base: base, - chainConfig: api.b.ChainConfig(), - // Each tx and all the series of txes shouldn't consume more gas than cap - gp: new(core.GasPool).AddGas(gasCap), + b: api.b, + state: state, + base: base, + chainConfig: api.b.ChainConfig(), + budget: newGasBudget(api.b.RPCGasCap()), traceTransfers: opts.TraceTransfers, validate: opts.Validation, fullTx: opts.ReturnFullTransactions, @@ -1047,6 +1092,9 @@ func RPCMarshalHeader(head *types.Header) map[string]interface{} { if head.RequestsHash != nil { result["requestsHash"] = head.RequestsHash } + if head.SlotNumber != nil { + result["slotNumber"] = hexutil.Uint64(*head.SlotNumber) + } return result } @@ -1139,6 +1187,7 @@ func (api *BlockChainAPI) arbClassicL1BlockNumber(ctx context.Context, block *ty type RPCTransaction struct { BlockHash *common.Hash `json:"blockHash"` BlockNumber *hexutil.Big `json:"blockNumber"` + BlockTimestamp *hexutil.Uint64 `json:"blockTimestamp"` From common.Address `json:"from"` Gas hexutil.Uint64 `json:"gas"` GasPrice *hexutil.Big `json:"gasPrice"` @@ -1199,6 +1248,7 @@ func newRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber if blockHash != (common.Hash{}) { result.BlockHash = &blockHash result.BlockNumber = (*hexutil.Big)(new(big.Int).SetUint64(blockNumber)) + result.BlockTimestamp = (*hexutil.Uint64)(&blockTime) result.TransactionIndex = (*hexutil.Uint64)(&index) } @@ -1544,7 +1594,7 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH if msg.BlobGasFeeCap != nil && msg.BlobGasFeeCap.BitLen() == 0 { evm.Context.BlobBaseFee = new(big.Int) } - res, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(msg.GasLimit)) + res, err := core.ApplyMessage(evm, msg, nil) if err != nil { return nil, 0, nil, fmt.Errorf("failed to apply transaction: %v err: %v", args.ToTransaction(types.LegacyTxType).Hash(), err) } @@ -1866,7 +1916,7 @@ func (api *TransactionAPI) SendTransaction(ctx context.Context, args Transaction return common.Hash{}, err } // Assemble the transaction and sign with the wallet - tx := args.ToTransaction(types.LegacyTxType) + tx := args.ToTransaction(types.DynamicFeeTxType) signed, err := wallet.SignTx(account, tx, api.b.ChainConfig().ChainID) if err != nil { @@ -1888,7 +1938,7 @@ func (api *TransactionAPI) FillTransaction(ctx context.Context, args Transaction return nil, err } // Assemble the transaction and obtain rlp - tx := args.ToTransaction(types.LegacyTxType) + tx := args.ToTransaction(types.DynamicFeeTxType) data, err := tx.MarshalBinary() if err != nil { return nil, err @@ -1929,7 +1979,7 @@ func (api *TransactionAPI) SendRawTransaction(ctx context.Context, input hexutil // SendRawTransactionSync will add the signed transaction to the transaction pool // and wait until the transaction has been included in a block and return the receipt, or the timeout. -func (api *TransactionAPI) SendRawTransactionSync(ctx context.Context, input hexutil.Bytes, timeoutMs *hexutil.Uint64) (map[string]interface{}, error) { +func (api *TransactionAPI) SendRawTransactionSync(ctx context.Context, input hexutil.Bytes, timeoutMs *uint64) (map[string]interface{}, error) { tx := new(types.Transaction) if err := tx.UnmarshalBinary(input); err != nil { return nil, err @@ -2095,7 +2145,7 @@ func (api *TransactionAPI) SignTransaction(ctx context.Context, args Transaction return nil, err } // Before actually sign the transaction, ensure the transaction fee is reasonable. - tx := args.ToTransaction(types.LegacyTxType) + tx := args.ToTransaction(types.DynamicFeeTxType) if err := checkTxFee(tx.GasPrice(), tx.Gas(), api.b.RPCTxFeeCap()); err != nil { return nil, err } @@ -2151,7 +2201,7 @@ func (api *TransactionAPI) Resend(ctx context.Context, sendArgs TransactionArgs, if err := sendArgs.setDefaults(ctx, api.b, sidecarConfig{}); err != nil { return common.Hash{}, err } - matchTx := sendArgs.ToTransaction(types.LegacyTxType) + matchTx := sendArgs.ToTransaction(types.DynamicFeeTxType) // Before replacing the old transaction, ensure the _new_ transaction fee is reasonable. price := matchTx.GasPrice() @@ -2181,7 +2231,7 @@ func (api *TransactionAPI) Resend(ctx context.Context, sendArgs TransactionArgs, if gasLimit != nil && *gasLimit != 0 { sendArgs.Gas = gasLimit } - signedTx, err := api.sign(sendArgs.from(), sendArgs.ToTransaction(types.LegacyTxType)) + signedTx, err := api.sign(sendArgs.from(), sendArgs.ToTransaction(types.DynamicFeeTxType)) if err != nil { return common.Hash{}, err } diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 0b62dd8c44..0ab80f37ad 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -24,7 +24,6 @@ import ( "encoding/json" "errors" "fmt" - "math" "math/big" "os" "path/filepath" @@ -141,6 +140,7 @@ func allTransactionTypes(addr common.Address, config *params.ChainConfig) []txDa Want: `{ "blockHash": null, "blockNumber": null, + "blockTimestamp": null, "from": "0x71562b71999873db5b286df957af199ec94617f7", "gas": "0x7", "gasPrice": "0x6", @@ -171,6 +171,7 @@ func allTransactionTypes(addr common.Address, config *params.ChainConfig) []txDa Want: `{ "blockHash": null, "blockNumber": null, + "blockTimestamp": null, "from": "0x71562b71999873db5b286df957af199ec94617f7", "gas": "0x7", "gasPrice": "0x6", @@ -209,6 +210,7 @@ func allTransactionTypes(addr common.Address, config *params.ChainConfig) []txDa Want: `{ "blockHash": null, "blockNumber": null, + "blockTimestamp": null, "from": "0x71562b71999873db5b286df957af199ec94617f7", "gas": "0x7", "gasPrice": "0x6", @@ -255,6 +257,7 @@ func allTransactionTypes(addr common.Address, config *params.ChainConfig) []txDa Want: `{ "blockHash": null, "blockNumber": null, + "blockTimestamp": null, "from": "0x71562b71999873db5b286df957af199ec94617f7", "gas": "0x7", "gasPrice": "0x6", @@ -302,6 +305,7 @@ func allTransactionTypes(addr common.Address, config *params.ChainConfig) []txDa Want: `{ "blockHash": null, "blockNumber": null, + "blockTimestamp": null, "from": "0x71562b71999873db5b286df957af199ec94617f7", "gas": "0x7", "gasPrice": "0x9", @@ -346,6 +350,7 @@ func allTransactionTypes(addr common.Address, config *params.ChainConfig) []txDa Want: `{ "blockHash": null, "blockNumber": null, + "blockTimestamp": null, "from": "0x71562b71999873db5b286df957af199ec94617f7", "gas": "0x7", "gasPrice": "0x9", @@ -388,6 +393,7 @@ func allBlobTxs(addr common.Address, config *params.ChainConfig) []txData { Want: `{ "blockHash": null, "blockNumber": null, + "blockTimestamp": null, "from": "0x71562b71999873db5b286df957af199ec94617f7", "gas": "0x6", "gasPrice": "0x5", @@ -2521,7 +2527,7 @@ func TestSimulateV1ChainLinkage(t *testing.T) { state: stateDB, base: baseHeader, chainConfig: backend.ChainConfig(), - gp: new(core.GasPool).AddGas(math.MaxUint64), + budget: newGasBudget(0), traceTransfers: false, validate: false, fullTx: false, @@ -2606,7 +2612,7 @@ func TestSimulateV1TxSender(t *testing.T) { state: stateDB, base: baseHeader, chainConfig: backend.ChainConfig(), - gp: new(core.GasPool).AddGas(math.MaxUint64), + budget: newGasBudget(0), traceTransfers: false, validate: false, fullTx: true, @@ -3159,6 +3165,7 @@ func TestRPCMarshalBlock(t *testing.T) { { "blockHash": "0x9b73c83b25d0faf7eab854e3684c7e394336d6e135625aafa5c183f27baa8fee", "blockNumber": "0x64", + "blockTimestamp": "0x0", "from": "0x0000000000000000000000000000000000000000", "gas": "0x457", "gasPrice": "0x2b67", @@ -3179,6 +3186,7 @@ func TestRPCMarshalBlock(t *testing.T) { { "blockHash": "0x9b73c83b25d0faf7eab854e3684c7e394336d6e135625aafa5c183f27baa8fee", "blockNumber": "0x64", + "blockTimestamp": "0x0", "from": "0x0000000000000000000000000000000000000000", "gas": "0x457", "gasPrice": "0x2b67", @@ -3197,6 +3205,7 @@ func TestRPCMarshalBlock(t *testing.T) { { "blockHash": "0x9b73c83b25d0faf7eab854e3684c7e394336d6e135625aafa5c183f27baa8fee", "blockNumber": "0x64", + "blockTimestamp": "0x0", "from": "0x0000000000000000000000000000000000000000", "gas": "0x457", "gasPrice": "0x2b67", @@ -3217,6 +3226,7 @@ func TestRPCMarshalBlock(t *testing.T) { { "blockHash": "0x9b73c83b25d0faf7eab854e3684c7e394336d6e135625aafa5c183f27baa8fee", "blockNumber": "0x64", + "blockTimestamp": "0x0", "from": "0x0000000000000000000000000000000000000000", "gas": "0x457", "gasPrice": "0x2b67", @@ -4048,7 +4058,7 @@ func TestSendRawTransactionSync_Timeout(t *testing.T) { raw, _ := makeSelfSignedRaw(t, api, b.acc.Address) - timeout := hexutil.Uint64(200) // 200ms + timeout := uint64(200) // 200ms receipt, err := api.SendRawTransactionSync(context.Background(), raw, &timeout) if receipt != nil { @@ -4076,3 +4086,91 @@ func TestSendRawTransactionSync_Timeout(t *testing.T) { t.Fatalf("expected ErrorData=%s, got %v", want, got) } } + +func TestGetStorageValues(t *testing.T) { + t.Parallel() + + var ( + addr1 = common.HexToAddress("0x1111") + addr2 = common.HexToAddress("0x2222") + slot0 = common.Hash{} + slot1 = common.BigToHash(big.NewInt(1)) + slot2 = common.BigToHash(big.NewInt(2)) + val0 = common.BigToHash(big.NewInt(42)) + val1 = common.BigToHash(big.NewInt(100)) + val2 = common.BigToHash(big.NewInt(200)) + + genesis = &core.Genesis{ + Config: params.MergedTestChainConfig, + Alloc: types.GenesisAlloc{ + addr1: { + Balance: big.NewInt(params.Ether), + Storage: map[common.Hash]common.Hash{ + slot0: val0, + slot1: val1, + }, + }, + addr2: { + Balance: big.NewInt(params.Ether), + Storage: map[common.Hash]common.Hash{ + slot2: val2, + }, + }, + }, + } + ) + api := NewBlockChainAPI(newTestBackend(t, 1, genesis, beacon.New(ethash.NewFaker()), func(i int, b *core.BlockGen) { + b.SetPoS() + })) + latest := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) + + // Happy path: multiple addresses, multiple slots. + result, err := api.GetStorageValues(context.Background(), map[common.Address][]common.Hash{ + addr1: {slot0, slot1}, + addr2: {slot2}, + }, latest) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result) != 2 { + t.Fatalf("expected 2 addresses in result, got %d", len(result)) + } + if got := common.BytesToHash(result[addr1][0]); got != val0 { + t.Errorf("addr1 slot0: want %x, got %x", val0, got) + } + if got := common.BytesToHash(result[addr1][1]); got != val1 { + t.Errorf("addr1 slot1: want %x, got %x", val1, got) + } + if got := common.BytesToHash(result[addr2][0]); got != val2 { + t.Errorf("addr2 slot2: want %x, got %x", val2, got) + } + + // Missing slot returns zero. + result, err = api.GetStorageValues(context.Background(), map[common.Address][]common.Hash{ + addr1: {common.HexToHash("0xff")}, + }, latest) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := common.BytesToHash(result[addr1][0]); got != (common.Hash{}) { + t.Errorf("missing slot: want zero, got %x", got) + } + + // Empty request returns error. + _, err = api.GetStorageValues(context.Background(), map[common.Address][]common.Hash{}, latest) + if err == nil { + t.Fatal("expected error for empty request") + } + + // Exceeding slot limit returns error. + tooMany := make([]common.Hash, maxGetStorageSlots+1) + for i := range tooMany { + tooMany[i] = common.BigToHash(big.NewInt(int64(i))) + } + _, err = api.GetStorageValues(context.Background(), map[common.Address][]common.Hash{ + addr1: tooMany, + }, latest) + if err == nil { + t.Fatal("expected error for exceeding slot limit") + } +} diff --git a/internal/ethapi/errors.go b/internal/ethapi/errors.go index e8e1393bc7..0353bf428c 100644 --- a/internal/ethapi/errors.go +++ b/internal/ethapi/errors.go @@ -120,7 +120,6 @@ const ( errCodeClientLimitExceeded = -38026 errCodeInternalError = -32603 errCodeInvalidParams = -32602 - errCodeReverted = -32000 errCodeVMError = -32015 errCodeTxSyncTimeout = 4 ) @@ -150,7 +149,7 @@ func txValidationError(err error) *invalidTxError { return &invalidTxError{Message: err.Error(), Code: errCodeIntrinsicGas} case errors.Is(err, core.ErrInsufficientFundsForTransfer): return &invalidTxError{Message: err.Error(), Code: errCodeInsufficientFunds} - case errors.Is(err, core.ErrMaxInitCodeSizeExceeded): + case errors.Is(err, vm.ErrMaxInitCodeSizeExceeded): return &invalidTxError{Message: err.Error(), Code: errCodeMaxInitCodeSizeExceeded} } return &invalidTxError{ diff --git a/internal/ethapi/override/override.go b/internal/ethapi/override/override.go index 63b0a9bb76..825ebe15dd 100644 --- a/internal/ethapi/override/override.go +++ b/internal/ethapi/override/override.go @@ -19,7 +19,9 @@ package override import ( "errors" "fmt" + "maps" "math/big" + "slices" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -58,12 +60,16 @@ func (diff *StateOverride) Apply(statedb *state.StateDB, precompiles vm.Precompi if diff == nil { return nil } + // Iterate in deterministic order so error messages and behavior are stable (e.g. for tests). + addrs := slices.SortedFunc(maps.Keys(*diff), common.Address.Cmp) + // Tracks destinations of precompiles that were moved. dirtyAddrs := make(map[common.Address]struct{}) - for addr, account := range *diff { + for _, addr := range addrs { if addr == types.ArbosStateAddress { return fmt.Errorf("overriding address %v not allowed", types.ArbosStateAddress) } + account := (*diff)[addr] // If a precompile was moved to this address already, it can't be overridden. if _, ok := dirtyAddrs[addr]; ok { return fmt.Errorf("account %s has already been overridden by a precompile", addr.Hex()) diff --git a/internal/ethapi/simulate.go b/internal/ethapi/simulate.go index 322e7c06bc..4385318b9a 100644 --- a/internal/ethapi/simulate.go +++ b/internal/ethapi/simulate.go @@ -21,6 +21,7 @@ import ( "encoding/json" "errors" "fmt" + "math" "math/big" "time" @@ -43,6 +44,14 @@ const ( // in a single request. maxSimulateBlocks = 256 + // maxSimulateCallsPerBlock is the maximum number of calls allowed in a + // single simulated block. + maxSimulateCallsPerBlock = 5000 + + // maxSimulateTotalCalls is the maximum total number of calls allowed + // across all simulated blocks in a single request. + maxSimulateTotalCalls = 10000 + // timestampIncrement is the default increment between block timestamps. timestampIncrement = 12 ) @@ -59,6 +68,7 @@ type simCallResult struct { ReturnValue hexutil.Bytes `json:"returnData"` Logs []*types.Log `json:"logs"` GasUsed hexutil.Uint64 `json:"gasUsed"` + MaxUsedGas hexutil.Uint64 `json:"maxUsedGas"` Status hexutil.Uint64 `json:"status"` Error *callError `json:"error,omitempty"` } @@ -149,6 +159,39 @@ func (m *simChainHeadReader) GetHeaderByHash(hash common.Hash) *types.Header { return header } +// gasBudget tracks the remaining gas allowed across all simulated blocks. +// It enforces the RPC-level gas cap to prevent DoS. +type gasBudget struct { + remaining uint64 +} + +// newGasBudget creates a gas budget with the given cap. +// A cap of 0 is treated as unlimited. +func newGasBudget(cap uint64) *gasBudget { + if cap == 0 { + cap = math.MaxUint64 + } + return &gasBudget{remaining: cap} +} + +// cap returns the given gas value clamped to the remaining budget. +func (b *gasBudget) cap(gas uint64) uint64 { + if gas > b.remaining { + return b.remaining + } + return gas +} + +// consume deducts the given amount from the budget. +// Returns an error if the amount exceeds the remaining budget. +func (b *gasBudget) consume(amount uint64) error { + if amount > b.remaining { + return fmt.Errorf("RPC gas cap exhausted: need %d, remaining %d", amount, b.remaining) + } + b.remaining -= amount + return nil +} + // simulator is a stateful object that simulates a series of blocks. // it is not safe for concurrent use. type simulator struct { @@ -156,7 +199,7 @@ type simulator struct { state *state.StateDB base *types.Header chainConfig *params.ChainConfig - gp *core.GasPool + budget *gasBudget traceTransfers bool validate bool fullTx bool @@ -200,7 +243,13 @@ func (sim *simulator) execute(ctx context.Context, blocks []simBlock) ([]*simBlo return nil, err } headers[bi] = result.Header() - results[bi] = &simBlockResult{fullTx: sim.fullTx, chainConfig: sim.chainConfig, Block: result, Calls: callResults, senders: senders} + results[bi] = &simBlockResult{ + fullTx: sim.fullTx, + chainConfig: sim.chainConfig, + Block: result, + Calls: callResults, + senders: senders, + } parent = result.Header() } return results, nil @@ -224,7 +273,7 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, } arbOSVersion := types.DeserializeHeaderExtraInformation(header).ArbOSFormatVersion parentArbOSVersion := types.DeserializeHeaderExtraInformation(parent).ArbOSFormatVersion - if sim.chainConfig.IsCancun(header.Number, header.Time, arbOSVersion) { + if !sim.chainConfig.IsArbitrum() && sim.chainConfig.IsCancun(header.Number, header.Time, arbOSVersion) { var excess uint64 if sim.chainConfig.IsCancun(parent.Number, parent.Time, parentArbOSVersion) { excess = eip4844.CalcExcessBlobGas(sim.chainConfig, parent, header.Time) @@ -235,16 +284,20 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, if block.BlockOverrides.BlobBaseFee != nil { blockContext.BlobBaseFee = block.BlockOverrides.BlobBaseFee.ToInt() } - precompiles := sim.activePrecompiles(sim.base) + precompiles := sim.activePrecompiles(header) + // State overrides are applied prior to execution of a block if err := block.StateOverrides.Apply(sim.state, precompiles); err != nil { return nil, nil, nil, err } var ( - gasUsed, blobGasUsed uint64 - txes = make([]*types.Transaction, len(block.Calls)) - callResults = make([]simCallResult, len(block.Calls)) - receipts = make([]*types.Receipt, len(block.Calls)) + gp = core.NewGasPool(blockContext.GasLimit) + blobGasUsed uint64 + + txes = make([]*types.Transaction, len(block.Calls)) + callResults = make([]simCallResult, len(block.Calls)) + receipts = make([]*types.Receipt, len(block.Calls)) + // Block hash will be repaired after execution. tracer = newTracer(sim.traceTransfers, blockContext.BlockNumber.Uint64(), blockContext.Time, common.Hash{}, common.Hash{}, 0) vmConfig = &vm.Config{ @@ -274,10 +327,12 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, } var allLogs []*types.Log for i, call := range block.Calls { + // Terminate if the context is cancelled if err := ctx.Err(); err != nil { return nil, nil, nil, err } - if err := sim.sanitizeCall(&call, sim.state, header, blockContext, &gasUsed); err != nil { + gasCapped, err := sim.sanitizeCall(&call, sim.state, header, gp) + if err != nil { return nil, nil, nil, err } var ( @@ -287,10 +342,11 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, txes[i] = tx senders[txHash] = call.from() tracer.reset(txHash, uint(i)) - sim.state.SetTxContext(txHash, i) + // EoA check is always skipped, even in validation mode. + sim.state.SetTxContext(txHash, i) msg := call.ToMessage(header.BaseFee, 0, nil, nil, core.NewMessageCommitContext(nil), !sim.validate) - result, err := applyMessageWithEVM(ctx, evm, msg, sim.state, timeout, sim.gp, sim.b, header, blockContext) + result, err := applyMessageWithEVM(ctx, evm, msg, sim.state, timeout, gp, sim.b, header, blockContext) if err != nil { txErr := txValidationError(err) return nil, nil, nil, txErr @@ -302,19 +358,29 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, } else { root = sim.state.IntermediateRoot(sim.chainConfig.IsEIP158(blockContext.BlockNumber)).Bytes() } - gasUsed += result.UsedGas - receipts[i] = core.MakeReceipt(evm, result, sim.state, blockContext.BlockNumber, common.Hash{}, blockContext.Time, tx, gasUsed, root) + receipts[i] = core.MakeReceipt(evm, result, sim.state, blockContext.BlockNumber, common.Hash{}, blockContext.Time, tx, gp.CumulativeUsed(), root) blobGasUsed += receipts[i].BlobGasUsed + + // Make sure the gas cap is still enforced. It's only for + // internally protection. + if err := sim.budget.consume(result.UsedGas); err != nil { + return nil, nil, nil, err + } + logs := tracer.Logs() - callRes := simCallResult{ReturnValue: result.Return(), Logs: logs, GasUsed: hexutil.Uint64(result.UsedGas)} + callRes := simCallResult{ReturnValue: result.Return(), Logs: logs, GasUsed: hexutil.Uint64(result.UsedGas), MaxUsedGas: hexutil.Uint64(result.MaxUsedGas)} if result.Failed() { callRes.Status = hexutil.Uint64(types.ReceiptStatusFailed) if errors.Is(result.Err, vm.ErrExecutionReverted) { // If the result contains a revert reason, try to unpack it. revertErr := newRevertError(result.Revert()) - callRes.Error = &callError{Message: revertErr.Error(), Code: errCodeReverted, Data: revertErr.ErrorData().(string)} + callRes.Error = &callError{Message: revertErr.Error(), Code: revertErr.ErrorCode(), Data: revertErr.ErrorData().(string)} } else { - callRes.Error = &callError{Message: result.Err.Error(), Code: errCodeVMError} + msg := result.Err.Error() + if gasCapped { + msg += " (gas limit was capped by the RPC server's global gas cap)" + } + callRes.Error = &callError{Message: msg, Code: errCodeVMError} } } else { callRes.Status = hexutil.Uint64(types.ReceiptStatusSuccessful) @@ -322,12 +388,14 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, } callResults[i] = callRes } - header.GasUsed = gasUsed + // Assign total consumed gas to the header + header.GasUsed = gp.Used() if sim.chainConfig.IsCancun(header.Number, header.Time, parentArbOSVersion) { header.BlobGasUsed = &blobGasUsed } - var requests [][]byte + // Process EIP-7685 requests unless Arbitrum is enabled. + var requests [][]byte if !sim.chainConfig.IsArbitrum() && sim.chainConfig.IsPrague(header.Number, header.Time, parentArbOSVersion) { requests = [][]byte{} // EIP-6110 @@ -347,9 +415,13 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, reqHash := types.CalcRequestsHash(requests) header.RequestsHash = &reqHash } - blockBody := &types.Body{Transactions: txes, Withdrawals: *block.BlockOverrides.Withdrawals} + + blockBody := &types.Body{ + Transactions: txes, + Withdrawals: *block.BlockOverrides.Withdrawals, + } chainHeadReader := &simChainHeadReader{ctx, sim.b} - b, err := sim.b.Engine().FinalizeAndAssemble(chainHeadReader, header, sim.state, blockBody, receipts) + b, err := sim.b.Engine().FinalizeAndAssemble(ctx, chainHeadReader, header, sim.state, blockBody, receipts) if err != nil { return nil, nil, nil, err } @@ -368,23 +440,25 @@ func repairLogs(calls []simCallResult, hash common.Hash) { } } -func (sim *simulator) sanitizeCall(call *TransactionArgs, state vm.StateDB, header *types.Header, blockContext vm.BlockContext, gasUsed *uint64) error { +func (sim *simulator) sanitizeCall(call *TransactionArgs, state vm.StateDB, header *types.Header, gp *core.GasPool) (bool, error) { if call.Nonce == nil { nonce := state.GetNonce(call.from()) call.Nonce = (*hexutil.Uint64)(&nonce) } // Let the call run wild unless explicitly specified. + remaining := gp.Gas() if call.Gas == nil { - remaining := blockContext.GasLimit - *gasUsed call.Gas = (*hexutil.Uint64)(&remaining) } - if *gasUsed+uint64(*call.Gas) > blockContext.GasLimit { - return &blockGasLimitReachedError{fmt.Sprintf("block gas limit reached: %d >= %d", gasUsed, blockContext.GasLimit)} + if remaining < uint64(*call.Gas) { + return false, &blockGasLimitReachedError{fmt.Sprintf("block gas limit reached: remaining: %d, required: %d", remaining, *call.Gas)} } - if err := call.CallDefaults(sim.gp.Gas(), header.BaseFee, sim.chainConfig.ChainID); err != nil { - return err - } - return nil + // Clamp to the cross-block gas budget. + gas := sim.budget.cap(uint64(*call.Gas)) + gasCapped := gas < uint64(*call.Gas) + call.Gas = (*hexutil.Uint64)(&gas) + + return gasCapped, call.CallDefaults(0, header.BaseFee, sim.chainConfig.ChainID) } func (sim *simulator) activePrecompiles(base *types.Header) vm.PrecompiledContracts { @@ -475,13 +549,14 @@ func (sim *simulator) makeHeaders(blocks []simBlock) ([]*types.Header, error) { } overrides := block.BlockOverrides - var withdrawalsHash *common.Hash number := overrides.Number.ToInt() timestamp := (uint64)(*overrides.Time) arbOsVersion := types.DeserializeHeaderExtraInformation(header).ArbOSFormatVersion + var withdrawalsHash *common.Hash if sim.chainConfig.IsShanghai(number, timestamp, arbOsVersion) { withdrawalsHash = &types.EmptyWithdrawalsHash } + var parentBeaconRoot *common.Hash if sim.chainConfig.IsCancun(number, timestamp, arbOsVersion) { parentBeaconRoot = &common.Hash{} @@ -513,7 +588,11 @@ func (sim *simulator) makeHeaders(blocks []simBlock) ([]*types.Header, error) { } func (sim *simulator) newSimulatedChainContext(ctx context.Context, headers []*types.Header) *ChainContext { - return NewChainContext(ctx, &simBackend{base: sim.base, b: sim.b, headers: headers}) + return NewChainContext(ctx, &simBackend{ + base: sim.base, + b: sim.b, + headers: headers, + }) } type simBackend struct { diff --git a/internal/ethapi/simulate_test.go b/internal/ethapi/simulate_test.go index 4f7ca4c8a8..2e3140eb83 100644 --- a/internal/ethapi/simulate_test.go +++ b/internal/ethapi/simulate_test.go @@ -80,7 +80,10 @@ func TestSimulateSanitizeBlockOrder(t *testing.T) { expected: []result{{number: 11, timestamp: 60}, {number: 12, timestamp: 72}, {number: 13, timestamp: 72}}, }, } { - sim := &simulator{base: &types.Header{Number: big.NewInt(int64(tc.baseNumber)), Time: tc.baseTimestamp}} + sim := &simulator{ + base: &types.Header{Number: big.NewInt(int64(tc.baseNumber)), Time: tc.baseTimestamp}, + budget: newGasBudget(0), + } res, err := sim.sanitizeChain(tc.blocks) if err != nil { if err.Error() == tc.err { diff --git a/internal/ethapi/testdata/eth_getBlockByHash-hash-latest-1-fullTx.json b/internal/ethapi/testdata/eth_getBlockByHash-hash-latest-1-fullTx.json index 3c8d42c9a9..6aac5b682e 100644 --- a/internal/ethapi/testdata/eth_getBlockByHash-hash-latest-1-fullTx.json +++ b/internal/ethapi/testdata/eth_getBlockByHash-hash-latest-1-fullTx.json @@ -20,6 +20,7 @@ { "blockHash": "0xedb9ccf3a85f67c095ad48abfb0fa09d47179bb0f902078d289042d12428aca5", "blockNumber": "0x9", + "blockTimestamp": "0x5a", "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", "gas": "0x5208", "gasPrice": "0x121a9cca", diff --git a/internal/ethapi/testdata/eth_getBlockByNumber-number-latest-1.json b/internal/ethapi/testdata/eth_getBlockByNumber-number-latest-1.json index 3c8d42c9a9..6aac5b682e 100644 --- a/internal/ethapi/testdata/eth_getBlockByNumber-number-latest-1.json +++ b/internal/ethapi/testdata/eth_getBlockByNumber-number-latest-1.json @@ -20,6 +20,7 @@ { "blockHash": "0xedb9ccf3a85f67c095ad48abfb0fa09d47179bb0f902078d289042d12428aca5", "blockNumber": "0x9", + "blockTimestamp": "0x5a", "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", "gas": "0x5208", "gasPrice": "0x121a9cca", diff --git a/internal/ethapi/testdata/eth_getBlockByNumber-tag-pending-fullTx.json b/internal/ethapi/testdata/eth_getBlockByNumber-tag-pending-fullTx.json index 2e323dcfe7..fc60111244 100644 --- a/internal/ethapi/testdata/eth_getBlockByNumber-tag-pending-fullTx.json +++ b/internal/ethapi/testdata/eth_getBlockByNumber-tag-pending-fullTx.json @@ -20,6 +20,7 @@ { "blockHash": "0xfda6c7cb7a3a712e0c424909a7724cab0448e89e286617fa8d5fd27f63f28bd2", "blockNumber": "0xb", + "blockTimestamp": "0x6e", "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", "gas": "0x5208", "gasPrice": "0xde56ab3", diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go index fd25c17220..cc54ab4ea9 100644 --- a/internal/ethapi/transaction_args.go +++ b/internal/ethapi/transaction_args.go @@ -162,6 +162,7 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend, config AccessList: args.AccessList, BlobFeeCap: args.BlobFeeCap, BlobHashes: args.BlobHashes, + AuthorizationList: args.AuthorizationList, } latestBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) estimated, err := DoEstimateGas(ctx, b, callArgs, latestBlockNr, nil, nil, b.RPCGasCap()) @@ -336,8 +337,8 @@ func (args *TransactionArgs) setBlobTxSidecar(ctx context.Context, config sideca commitments = make([]kzg4844.Commitment, n) proofs = make([]kzg4844.Proof, 0, proofLen) ) - for i, b := range args.Blobs { - c, err := kzg4844.BlobToCommitment(&b) + for i := range args.Blobs { + c, err := kzg4844.BlobToCommitment(&args.Blobs[i]) if err != nil { return fmt.Errorf("blobs[%d]: error computing commitment: %v", i, err) } @@ -345,13 +346,13 @@ func (args *TransactionArgs) setBlobTxSidecar(ctx context.Context, config sideca switch config.blobSidecarVersion { case types.BlobSidecarVersion0: - p, err := kzg4844.ComputeBlobProof(&b, c) + p, err := kzg4844.ComputeBlobProof(&args.Blobs[i], c) if err != nil { return fmt.Errorf("blobs[%d]: error computing proof: %v", i, err) } proofs = append(proofs, p) case types.BlobSidecarVersion1: - ps, err := kzg4844.ComputeCellProofs(&b) + ps, err := kzg4844.ComputeCellProofs(&args.Blobs[i]) if err != nil { return fmt.Errorf("blobs[%d]: error computing proof: %v", i, err) } @@ -363,8 +364,8 @@ func (args *TransactionArgs) setBlobTxSidecar(ctx context.Context, config sideca } else { switch config.blobSidecarVersion { case types.BlobSidecarVersion0: - for i, b := range args.Blobs { - if err := kzg4844.VerifyBlobProof(&b, args.Commitments[i], args.Proofs[i]); err != nil { + for i := range args.Blobs { + if err := kzg4844.VerifyBlobProof(&args.Blobs[i], args.Commitments[i], args.Proofs[i]); err != nil { return fmt.Errorf("failed to verify blob proof: %v", err) } } diff --git a/internal/flags/helpers.go b/internal/flags/helpers.go index fc84ae85da..e6a6966d9f 100644 --- a/internal/flags/helpers.go +++ b/internal/flags/helpers.go @@ -40,7 +40,7 @@ func NewApp(usage string) *cli.App { app.EnableBashCompletion = true app.Version = version.WithCommit(git.Commit, git.Date) app.Usage = usage - app.Copyright = "Copyright 2013-2025 The go-ethereum Authors" + app.Copyright = "Copyright 2013-2026 The go-ethereum Authors" app.Before = func(ctx *cli.Context) error { MigrateGlobalFlags(ctx) return nil diff --git a/core/rawdb/database_tablewriter_tinygo.go b/internal/tablewriter/database_tablewriter.go similarity index 92% rename from core/rawdb/database_tablewriter_tinygo.go rename to internal/tablewriter/database_tablewriter.go index 2f8e456fd5..c080a69c15 100644 --- a/core/rawdb/database_tablewriter_tinygo.go +++ b/internal/tablewriter/database_tablewriter.go @@ -14,12 +14,9 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// TODO: naive stub implementation for tablewriter +// Naive stub implementation for tablewriter -//go:build tinygo -// +build tinygo - -package rawdb +package tablewriter import ( "errors" @@ -40,7 +37,7 @@ type Table struct { rows [][]string } -func newTableWriter(w io.Writer) *Table { +func NewWriter(w io.Writer) *Table { return &Table{out: w} } @@ -61,12 +58,12 @@ func (t *Table) SetFooter(footer []string) { t.footer = footer } -// AppendBulk sets all data rows for the table at once, replacing any existing rows. +// AppendBulk appends one or more data rows to the table. // // Each row must have the same number of columns as the headers, or validation // will fail during Render(). func (t *Table) AppendBulk(rows [][]string) { - t.rows = rows + t.rows = append(t.rows, rows...) } // Render outputs the complete table to the configured writer. The table is rendered @@ -89,6 +86,7 @@ func (t *Table) render() error { rowSeparator := t.buildRowSeparator(widths) if len(t.headers) > 0 { + fmt.Fprintln(t.out, rowSeparator) t.printRow(t.headers, widths) fmt.Fprintln(t.out, rowSeparator) } @@ -100,6 +98,7 @@ func (t *Table) render() error { if len(t.footer) > 0 { fmt.Fprintln(t.out, rowSeparator) t.printRow(t.footer, widths) + fmt.Fprintln(t.out, rowSeparator) } return nil @@ -172,21 +171,22 @@ func (t *Table) calculateColumnWidths() []int { // // It generates a string with dashes (-) for each column width, joined by plus signs (+). // -// Example output: "----------+--------+-----------" +// Example output: "+----------+--------+-----------+" func (t *Table) buildRowSeparator(widths []int) string { parts := make([]string, len(widths)) for i, w := range widths { parts[i] = strings.Repeat("-", w) } - return strings.Join(parts, "+") + return "+" + strings.Join(parts, "+") + "+" } // printRow outputs a single row to the table writer. // // Each cell is padded with spaces and separated by pipe characters (|). // -// Example output: " Database | Size | Items " +// Example output: "| Database | Size | Items |" func (t *Table) printRow(row []string, widths []int) { + fmt.Fprintf(t.out, "|") for i, cell := range row { if i > 0 { fmt.Fprint(t.out, "|") @@ -204,5 +204,6 @@ func (t *Table) printRow(row []string, widths []int) { fmt.Fprintf(t.out, "%s%s%s", leftPadding, cell, rightPadding) } + fmt.Fprintf(t.out, "|") fmt.Fprintln(t.out) } diff --git a/core/rawdb/database_tablewriter_tinygo_test.go b/internal/tablewriter/database_tablewriter_test.go similarity index 93% rename from core/rawdb/database_tablewriter_tinygo_test.go rename to internal/tablewriter/database_tablewriter_test.go index 3bcf93832b..b915dcdda8 100644 --- a/core/rawdb/database_tablewriter_tinygo_test.go +++ b/internal/tablewriter/database_tablewriter_test.go @@ -14,10 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//go:build tinygo -// +build tinygo - -package rawdb +package tablewriter import ( "bytes" @@ -27,7 +24,7 @@ import ( func TestTableWriterTinyGo(t *testing.T) { var buf bytes.Buffer - table := newTableWriter(&buf) + table := NewWriter(&buf) headers := []string{"Database", "Size", "Items", "Status"} rows := [][]string{ @@ -51,7 +48,7 @@ func TestTableWriterValidationErrors(t *testing.T) { // Test missing headers t.Run("MissingHeaders", func(t *testing.T) { var buf bytes.Buffer - table := newTableWriter(&buf) + table := NewWriter(&buf) rows := [][]string{{"x", "y", "z"}} @@ -66,7 +63,7 @@ func TestTableWriterValidationErrors(t *testing.T) { t.Run("NotEnoughRowColumns", func(t *testing.T) { var buf bytes.Buffer - table := newTableWriter(&buf) + table := NewWriter(&buf) headers := []string{"A", "B", "C"} badRows := [][]string{ @@ -85,7 +82,7 @@ func TestTableWriterValidationErrors(t *testing.T) { t.Run("TooManyRowColumns", func(t *testing.T) { var buf bytes.Buffer - table := newTableWriter(&buf) + table := NewWriter(&buf) headers := []string{"A", "B", "C"} badRows := [][]string{ @@ -105,7 +102,7 @@ func TestTableWriterValidationErrors(t *testing.T) { // Test mismatched footer columns t.Run("MismatchedFooterColumns", func(t *testing.T) { var buf bytes.Buffer - table := newTableWriter(&buf) + table := NewWriter(&buf) headers := []string{"A", "B", "C"} rows := [][]string{{"x", "y", "z"}} diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go new file mode 100644 index 0000000000..27fe9b0a7a --- /dev/null +++ b/internal/telemetry/telemetry.go @@ -0,0 +1,109 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package telemetry + +import ( + "context" + "fmt" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + semconv "go.opentelemetry.io/otel/semconv/v1.38.0" + "go.opentelemetry.io/otel/trace" +) + +// Attribute is an alias for attribute.KeyValue. +type Attribute = attribute.KeyValue + +// StringAttribute creates an attribute with a string value. +func StringAttribute(key, val string) Attribute { + return attribute.String(key, val) +} + +// Int64Attribute creates an attribute with an int64 value. +func Int64Attribute(key string, val int64) Attribute { + return attribute.Int64(key, val) +} + +// BoolAttribute creates an attribute with a bool value. +func BoolAttribute(key string, val bool) Attribute { + return attribute.Bool(key, val) +} + +// StartSpan creates a SpanKind=INTERNAL span. +func StartSpan(ctx context.Context, spanName string, attributes ...Attribute) (context.Context, trace.Span, func(*error)) { + return StartSpanWithTracer(ctx, otel.Tracer(""), spanName, attributes...) +} + +// StartSpanWithTracer requires a tracer to be passed in and creates a SpanKind=INTERNAL span. +func StartSpanWithTracer(ctx context.Context, tracer trace.Tracer, name string, attributes ...Attribute) (context.Context, trace.Span, func(*error)) { + // Don't create a span if there's no parent span in the context. + parent := trace.SpanFromContext(ctx) + if !parent.SpanContext().IsValid() { + return ctx, parent, func(*error) {} + } + return startSpan(ctx, tracer, trace.SpanKindInternal, name, attributes...) +} + +// RPCInfo contains information about the RPC request. +type RPCInfo struct { + System string + Service string + Method string + RequestID string +} + +// StartServerSpan creates a SpanKind=SERVER span at the JSON-RPC boundary. +// The span name is formatted as $rpcSystem.$rpcService/$rpcMethod +// (e.g. "jsonrpc.engine/newPayloadV4") which follows the Open Telemetry +// semantic convensions: https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/#span-name. +func StartServerSpan(ctx context.Context, tracer trace.Tracer, rpc RPCInfo, others ...Attribute) (context.Context, func(*error)) { + var ( + name = fmt.Sprintf("%s.%s/%s", rpc.System, rpc.Service, rpc.Method) + attributes = append([]Attribute{ + semconv.RPCSystemKey.String(rpc.System), + semconv.RPCServiceKey.String(rpc.Service), + semconv.RPCMethodKey.String(rpc.Method), + semconv.RPCJSONRPCRequestID(rpc.RequestID), + }, + others..., + ) + ) + ctx, _, end := startSpan(ctx, tracer, trace.SpanKindServer, name, attributes...) + return ctx, end +} + +// startSpan creates a span with the given kind. +func startSpan(ctx context.Context, tracer trace.Tracer, kind trace.SpanKind, spanName string, attributes ...Attribute) (context.Context, trace.Span, func(*error)) { + ctx, span := tracer.Start(ctx, spanName, trace.WithSpanKind(kind)) + if len(attributes) > 0 { + span.SetAttributes(attributes...) + } + return ctx, span, endSpan(span) +} + +// endSpan ends the span and handles error recording. +func endSpan(span trace.Span) func(*error) { + return func(err *error) { + if err != nil && *err != nil { + span.RecordError(*err) + span.SetStatus(codes.Error, (*err).Error()) + } + span.End() + } +} diff --git a/internal/telemetry/telemetry_test.go b/internal/telemetry/telemetry_test.go new file mode 100644 index 0000000000..def85b735e --- /dev/null +++ b/internal/telemetry/telemetry_test.go @@ -0,0 +1,98 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package telemetry + +import ( + "context" + "testing" + + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "go.opentelemetry.io/otel/trace" +) + +// newTestTracer creates a TracerProvider backed by an in-memory exporter. +func newTestTracer(t *testing.T) (trace.Tracer, *sdktrace.TracerProvider, *tracetest.InMemoryExporter) { + t.Helper() + exporter := tracetest.NewInMemoryExporter() + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exporter)) + t.Cleanup(func() { _ = tp.Shutdown(context.Background()) }) + return tp.Tracer("test"), tp, exporter +} + +func TestStartSpanWithTracer_NoParent(t *testing.T) { + t.Parallel() + tracer, tp, exporter := newTestTracer(t) + + // Create a span without a parent. + ctx := context.Background() + retCtx, _, endSpan := StartSpanWithTracer(ctx, tracer, "should-not-exist") + endSpan(nil) + + // The returned context should be the original context (unchanged). + if retCtx != ctx { + t.Fatal("expected original context to be returned unchanged") + } + + // Flush and verify no spans were recorded. + if err := tp.ForceFlush(context.Background()); err != nil { + t.Fatalf("failed to flush: %v", err) + } + spans := exporter.GetSpans() + if len(spans) != 0 { + t.Fatalf("expected no spans, got %d", len(spans)) + } +} + +func TestStartSpanWithTracer_WithParent(t *testing.T) { + t.Parallel() + tracer, tp, exporter := newTestTracer(t) + + // Create a parent span to establish a valid span context. + ctx, parentSpan := tracer.Start(context.Background(), "parent") + defer parentSpan.End() + + // Should create a real child span. + _, _, endSpan := StartSpanWithTracer(ctx, tracer, "child") + endSpan(nil) + + // Flush and verify the child span was recorded. + parentSpan.End() + if err := tp.ForceFlush(context.Background()); err != nil { + t.Fatalf("failed to flush: %v", err) + } + spans := exporter.GetSpans() + var childSpan *tracetest.SpanStub + for i := range spans { + if spans[i].Name == "child" { + childSpan = &spans[i] + break + } + } + if childSpan == nil { + t.Fatal("child span not found") + } + + // Verify it is parented to the correct trace. + if childSpan.Parent.TraceID() != parentSpan.SpanContext().TraceID() { + t.Errorf("trace ID mismatch: got %s, want %s", + childSpan.Parent.TraceID(), parentSpan.SpanContext().TraceID()) + } + if childSpan.SpanKind != trace.SpanKindInternal { + t.Errorf("expected SpanKindInternal, got %v", childSpan.SpanKind) + } +} diff --git a/internal/telemetry/tracesetup/setup.go b/internal/telemetry/tracesetup/setup.go new file mode 100644 index 0000000000..444416dd26 --- /dev/null +++ b/internal/telemetry/tracesetup/setup.go @@ -0,0 +1,162 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package tracesetup + +import ( + "context" + "encoding/base64" + "fmt" + "net/url" + "strings" + "time" + + "github.com/ethereum/go-ethereum/internal/version" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.38.0" +) + +const startStopTimeout = 10 * time.Second + +// Service wraps the provider to implement node.Lifecycle. +type Service struct { + endpoint string + exporter *otlptrace.Exporter + provider *sdktrace.TracerProvider +} + +// Start implements node.Lifecycle. +func (t *Service) Start() error { + ctx, cancel := context.WithTimeout(context.Background(), startStopTimeout) + defer cancel() + if err := t.exporter.Start(ctx); err != nil { + log.Error("OpenTelemetry exporter didn't start", "endpoint", t.endpoint, "err", err) + return err + } + log.Info("OpenTelemetry trace export enabled", "endpoint", t.endpoint) + return nil +} + +// Stop implements node.Lifecycle. +func (t *Service) Stop() error { + ctx, cancel := context.WithTimeout(context.Background(), startStopTimeout) + defer cancel() + if err := t.provider.Shutdown(ctx); err != nil { + log.Error("Failed to stop OpenTelemetry service", "err", err) + return err + } + log.Debug("OpenTelemetry stopped") + return nil +} + +// SetupTelemetry initializes telemetry with the given parameters. +func SetupTelemetry(cfg node.OpenTelemetryConfig, stack *node.Node) error { + if !cfg.Enabled { + return nil + } + if cfg.SampleRatio < 0 || cfg.SampleRatio > 1 { + return fmt.Errorf("invalid sample ratio: %f", cfg.SampleRatio) + } + // Create exporter based on endpoint URL + u, err := url.Parse(cfg.Endpoint) + if err != nil { + return fmt.Errorf("invalid rpc tracing endpoint URL: %w", err) + } + var exporter *otlptrace.Exporter + switch u.Scheme { + case "http", "https": + opts := []otlptracehttp.Option{ + otlptracehttp.WithEndpoint(u.Host), + } + if u.Scheme == "http" { + opts = append(opts, otlptracehttp.WithInsecure()) + } + if u.Path != "" && u.Path != "/" { + opts = append(opts, otlptracehttp.WithURLPath(u.Path)) + } + if cfg.AuthUser != "" { + opts = append(opts, otlptracehttp.WithHeaders(map[string]string{ + "Authorization": "Basic " + base64.StdEncoding.EncodeToString([]byte(cfg.AuthUser+":"+cfg.AuthPassword)), + })) + } + exporter = otlptracehttp.NewUnstarted(opts...) + default: + return fmt.Errorf("unsupported telemetry url scheme: %s", u.Scheme) + } + + // Define sampler such that if no parent span is available, + // then sampleRatio of traces are sampled; otherwise, inherit + // the parent's sampling decision. + sampler := sdktrace.ParentBased(sdktrace.TraceIDRatioBased(cfg.SampleRatio)) + + // Define batch span processor options + batchOpts := []sdktrace.BatchSpanProcessorOption{ + // The maximum number of spans that can be queued before dropping + sdktrace.WithMaxQueueSize(sdktrace.DefaultMaxQueueSize), + // The maximum number of spans to export in a single batch + sdktrace.WithMaxExportBatchSize(sdktrace.DefaultMaxExportBatchSize), + // How long an export operation can take before timing out + sdktrace.WithExportTimeout(time.Duration(sdktrace.DefaultExportTimeout) * time.Millisecond), + // How often to export, even if the batch isn't full + sdktrace.WithBatchTimeout(time.Duration(sdktrace.DefaultScheduleDelay) * time.Millisecond), + } + + // Define resource attributes + var attr = []attribute.KeyValue{ + semconv.ServiceName("geth"), + attribute.String("client.name", version.ClientName("geth")), + } + // Add instance ID if provided + if cfg.InstanceID != "" { + attr = append(attr, semconv.ServiceInstanceID(cfg.InstanceID)) + } + // Add custom tags if provided + if cfg.Tags != "" { + for tag := range strings.SplitSeq(cfg.Tags, ",") { + key, value, ok := strings.Cut(tag, "=") + if ok { + attr = append(attr, attribute.String(key, value)) + } + } + } + res := resource.NewWithAttributes(semconv.SchemaURL, attr...) + + // Configure TracerProvider and set it as the global tracer provider + tp := sdktrace.NewTracerProvider( + sdktrace.WithSampler(sampler), + sdktrace.WithBatcher(exporter, batchOpts...), + sdktrace.WithResource(res), + ) + otel.SetTracerProvider(tp) + + // Set global propagator for context propagation + // Note: This is needed for distributed tracing + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + )) + service := &Service{endpoint: cfg.Endpoint, exporter: exporter, provider: tp} + stack.RegisterLifecycle(service) + return nil +} diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 0aedffe230..1d1b5fbcd1 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -427,11 +427,6 @@ web3._extend({ params: 2, inputFormatter:[null, null], }), - new web3._extend.Method({ - name: 'freezeClient', - call: 'debug_freezeClient', - params: 1, - }), new web3._extend.Method({ name: 'getAccessibleState', call: 'debug_getAccessibleState', @@ -474,6 +469,12 @@ web3._extend({ params: 1, inputFormatter: [null], }), + new web3._extend.Method({ + name: 'executionWitness', + call: 'debug_executionWitness', + params: 1, + inputFormatter: [null], + }), ], properties: [] }); @@ -567,6 +568,12 @@ web3._extend({ params: 3, inputFormatter: [web3._extend.formatters.inputAddressFormatter, null, web3._extend.formatters.inputBlockNumberFormatter] }), + new web3._extend.Method({ + name: 'getStorageValues', + call: 'eth_getStorageValues', + params: 2, + inputFormatter: [null, web3._extend.formatters.inputBlockNumberFormatter] + }), new web3._extend.Method({ name: 'createAccessList', call: 'eth_createAccessList', diff --git a/metrics/config.go b/metrics/config.go index 72f94dd194..3d3cb693fd 100644 --- a/metrics/config.go +++ b/metrics/config.go @@ -16,18 +16,21 @@ package metrics +import "time" + // Config contains the configuration for the metric collection. type Config struct { - Enabled bool `toml:",omitempty"` - EnabledExpensive bool `toml:"-"` - HTTP string `toml:",omitempty"` - Port int `toml:",omitempty"` - EnableInfluxDB bool `toml:",omitempty"` - InfluxDBEndpoint string `toml:",omitempty"` - InfluxDBDatabase string `toml:",omitempty"` - InfluxDBUsername string `toml:",omitempty"` - InfluxDBPassword string `toml:",omitempty"` - InfluxDBTags string `toml:",omitempty"` + Enabled bool `toml:",omitempty"` + EnabledExpensive bool `toml:"-"` + HTTP string `toml:",omitempty"` + Port int `toml:",omitempty"` + EnableInfluxDB bool `toml:",omitempty"` + InfluxDBEndpoint string `toml:",omitempty"` + InfluxDBDatabase string `toml:",omitempty"` + InfluxDBUsername string `toml:",omitempty"` + InfluxDBPassword string `toml:",omitempty"` + InfluxDBTags string `toml:",omitempty"` + InfluxDBInterval time.Duration `toml:",omitempty"` EnableInfluxDBV2 bool `toml:",omitempty"` InfluxDBToken string `toml:",omitempty"` @@ -47,6 +50,7 @@ var DefaultConfig = Config{ InfluxDBUsername: "test", InfluxDBPassword: "test", InfluxDBTags: "host=localhost", + InfluxDBInterval: 10 * time.Second, // influxdbv2-specific flags EnableInfluxDBV2: false, diff --git a/metrics/counter.go b/metrics/counter.go index c884e9a178..0e4d93bfbc 100644 --- a/metrics/counter.go +++ b/metrics/counter.go @@ -7,7 +7,10 @@ import ( // GetOrRegisterCounter returns an existing Counter or constructs and registers // a new Counter. func GetOrRegisterCounter(name string, r Registry) *Counter { - return getOrRegister(name, NewCounter, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewCounter() }).(*Counter) } // NewCounter constructs a new Counter. diff --git a/metrics/counter_float64.go b/metrics/counter_float64.go index 6cc73d89a2..caaaeb7be7 100644 --- a/metrics/counter_float64.go +++ b/metrics/counter_float64.go @@ -8,7 +8,10 @@ import ( // GetOrRegisterCounterFloat64 returns an existing *CounterFloat64 or constructs and registers // a new CounterFloat64. func GetOrRegisterCounterFloat64(name string, r Registry) *CounterFloat64 { - return getOrRegister(name, NewCounterFloat64, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewCounterFloat64() }).(*CounterFloat64) } // NewCounterFloat64 constructs a new CounterFloat64. diff --git a/metrics/gauge.go b/metrics/gauge.go index 20de95255b..39d5ccb3bc 100644 --- a/metrics/gauge.go +++ b/metrics/gauge.go @@ -11,7 +11,10 @@ func (g GaugeSnapshot) Value() int64 { return int64(g) } // GetOrRegisterGauge returns an existing Gauge or constructs and registers a // new Gauge. func GetOrRegisterGauge(name string, r Registry) *Gauge { - return getOrRegister(name, NewGauge, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewGauge() }).(*Gauge) } // NewGauge constructs a new Gauge. diff --git a/metrics/gauge_float64.go b/metrics/gauge_float64.go index 48524e4c3f..c7a1df0a23 100644 --- a/metrics/gauge_float64.go +++ b/metrics/gauge_float64.go @@ -8,7 +8,10 @@ import ( // GetOrRegisterGaugeFloat64 returns an existing GaugeFloat64 or constructs and registers a // new GaugeFloat64. func GetOrRegisterGaugeFloat64(name string, r Registry) *GaugeFloat64 { - return getOrRegister(name, NewGaugeFloat64, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewGaugeFloat64() }).(*GaugeFloat64) } // GaugeFloat64Snapshot is a read-only copy of a GaugeFloat64. diff --git a/metrics/gauge_info.go b/metrics/gauge_info.go index 34ac917919..30c5114085 100644 --- a/metrics/gauge_info.go +++ b/metrics/gauge_info.go @@ -16,7 +16,10 @@ func (val GaugeInfoValue) String() string { // GetOrRegisterGaugeInfo returns an existing GaugeInfo or constructs and registers a // new GaugeInfo. func GetOrRegisterGaugeInfo(name string, r Registry) *GaugeInfo { - return getOrRegister(name, NewGaugeInfo, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewGaugeInfo() }).(*GaugeInfo) } // NewGaugeInfo constructs a new GaugeInfo. diff --git a/metrics/histogram.go b/metrics/histogram.go index 18bf6e3d2b..467457abdb 100644 --- a/metrics/histogram.go +++ b/metrics/histogram.go @@ -23,13 +23,19 @@ type Histogram interface { // GetOrRegisterHistogram returns an existing Histogram or constructs and // registers a new StandardHistogram. func GetOrRegisterHistogram(name string, r Registry, s Sample) Histogram { - return getOrRegister(name, func() Histogram { return NewHistogram(s) }, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewHistogram(s) }).(Histogram) } // GetOrRegisterHistogramLazy returns an existing Histogram or constructs and // registers a new StandardHistogram. func GetOrRegisterHistogramLazy(name string, r Registry, s func() Sample) Histogram { - return getOrRegister(name, func() Histogram { return NewHistogram(s()) }, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewHistogram(s()) }).(Histogram) } // NewHistogram constructs a new StandardHistogram from a Sample. diff --git a/metrics/meter.go b/metrics/meter.go index ee23af10eb..2829774d8c 100644 --- a/metrics/meter.go +++ b/metrics/meter.go @@ -12,7 +12,10 @@ import ( // Be sure to unregister the meter from the registry once it is of no use to // allow for garbage collection. func GetOrRegisterMeter(name string, r Registry) *Meter { - return getOrRegister(name, NewMeter, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewMeter() }).(*Meter) } // NewMeter constructs a new Meter and launches a goroutine. diff --git a/metrics/registry.go b/metrics/registry.go index 6070f3d0e9..298598a9aa 100644 --- a/metrics/registry.go +++ b/metrics/registry.go @@ -143,6 +143,8 @@ func (r *StandardRegistry) GetAll() map[string]map[string]interface{} { values["value"] = metric.Snapshot().Value() case *GaugeFloat64: values["value"] = metric.Snapshot().Value() + case *GaugeInfo: + values["value"] = metric.Snapshot().Value() case *Healthcheck: values["error"] = nil metric.Check() @@ -186,6 +188,18 @@ func (r *StandardRegistry) GetAll() map[string]map[string]interface{} { values["5m.rate"] = t.Rate5() values["15m.rate"] = t.Rate15() values["mean.rate"] = t.RateMean() + case *ResettingTimer: + t := metric.Snapshot() + ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + values["count"] = t.Count() + values["min"] = t.Min() + values["max"] = t.Max() + values["mean"] = t.Mean() + values["median"] = ps[0] + values["75%"] = ps[1] + values["95%"] = ps[2] + values["99%"] = ps[3] + values["99.9%"] = ps[4] } data[name] = values }) @@ -331,13 +345,6 @@ func GetOrRegister(name string, i func() interface{}) interface{} { return DefaultRegistry.GetOrRegister(name, i) } -func getOrRegister[T any](name string, ctor func() T, r Registry) T { - if r == nil { - r = DefaultRegistry - } - return r.GetOrRegister(name, func() any { return ctor() }).(T) -} - // Register the given metric under the given name. Returns a ErrDuplicateMetric // if a metric by the given name is already registered. func Register(name string, i interface{}) error { diff --git a/metrics/registry_test.go b/metrics/registry_test.go index 6af0796da9..1aad7a0028 100644 --- a/metrics/registry_test.go +++ b/metrics/registry_test.go @@ -14,6 +14,31 @@ func BenchmarkRegistry(b *testing.B) { } } +func BenchmarkRegistryGetOrRegister(b *testing.B) { + sample := func() Sample { return nil } + tests := []struct { + name string + ctor func() any + }{ + {name: "counter", ctor: func() any { return GetOrRegisterCounter("counter", DefaultRegistry) }}, + {name: "gauge", ctor: func() any { return GetOrRegisterGauge("gauge", DefaultRegistry) }}, + {name: "gaugefloat64", ctor: func() any { return GetOrRegisterGaugeFloat64("gaugefloat64", DefaultRegistry) }}, + {name: "histogram", ctor: func() any { return GetOrRegisterHistogram("histogram", DefaultRegistry, sample()) }}, + {name: "meter", ctor: func() any { return GetOrRegisterMeter("meter", DefaultRegistry) }}, + {name: "timer", ctor: func() any { return GetOrRegisterTimer("timer", DefaultRegistry) }}, + {name: "gaugeinfo", ctor: func() any { return GetOrRegisterGaugeInfo("gaugeinfo", DefaultRegistry) }}, + {name: "resettingtimer", ctor: func() any { return GetOrRegisterResettingTimer("resettingtimer", DefaultRegistry) }}, + {name: "runtimehistogramlazy", ctor: func() any { return GetOrRegisterHistogramLazy("runtimehistogramlazy", DefaultRegistry, sample) }}, + } + for _, test := range tests { + b.Run(test.name, func(b *testing.B) { + for i := 0; i < b.N; i++ { + test.ctor() + } + }) + } +} + func BenchmarkRegistryGetOrRegisterParallel_8(b *testing.B) { benchmarkRegistryGetOrRegisterParallel(b, 8) } @@ -268,7 +293,7 @@ func TestPrefixedChildRegistryGet(t *testing.T) { } func TestChildPrefixedRegistryRegister(t *testing.T) { - r := NewPrefixedChildRegistry(DefaultRegistry, "prefix.") + r := NewPrefixedChildRegistry(NewRegistry(), "prefix.") err := r.Register("foo", NewCounter()) c := NewCounter() Register("bar", c) diff --git a/metrics/resetting_timer.go b/metrics/resetting_timer.go index 8aa7dc1488..a3f46e52e0 100644 --- a/metrics/resetting_timer.go +++ b/metrics/resetting_timer.go @@ -8,7 +8,10 @@ import ( // GetOrRegisterResettingTimer returns an existing ResettingTimer or constructs and registers a // new ResettingTimer. func GetOrRegisterResettingTimer(name string, r Registry) *ResettingTimer { - return getOrRegister(name, NewResettingTimer, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewResettingTimer() }).(*ResettingTimer) } // NewRegisteredResettingTimer constructs and registers a new ResettingTimer. diff --git a/metrics/runtimehistogram.go b/metrics/runtimehistogram.go index efbed498af..e975a570a4 100644 --- a/metrics/runtimehistogram.go +++ b/metrics/runtimehistogram.go @@ -8,8 +8,10 @@ import ( ) func getOrRegisterRuntimeHistogram(name string, scale float64, r Registry) *runtimeHistogram { - constructor := func() Histogram { return newRuntimeHistogram(scale) } - return getOrRegister(name, constructor, r).(*runtimeHistogram) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return newRuntimeHistogram(scale) }).(*runtimeHistogram) } // runtimeHistogram wraps a runtime/metrics histogram. diff --git a/metrics/timer.go b/metrics/timer.go index 894bdfc327..8082b31947 100644 --- a/metrics/timer.go +++ b/metrics/timer.go @@ -10,7 +10,10 @@ import ( // Be sure to unregister the meter from the registry once it is of no use to // allow for garbage collection. func GetOrRegisterTimer(name string, r Registry) *Timer { - return getOrRegister(name, NewTimer, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewTimer() }).(*Timer) } // NewCustomTimer constructs a new Timer from a Histogram and a Meter. diff --git a/miner/miner.go b/miner/miner.go index 90d8116203..59f2f1a18a 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -18,6 +18,7 @@ package miner import ( + "context" "fmt" "math/big" "sync" @@ -48,6 +49,7 @@ type Config struct { GasCeil uint64 // Target gas ceiling for mined blocks. GasPrice *big.Int // Minimum gas price for mining a transaction Recommit time.Duration // The time interval for miner to re-create mining work. + MaxBlobsPerBlock int // Maximum number of blobs per block (0 for unset uses protocol default) } // DefaultConfig contains default settings for miner. @@ -134,8 +136,8 @@ func (miner *Miner) SetGasTip(tip *big.Int) error { } // BuildPayload builds the payload according to the provided parameters. -func (miner *Miner) BuildPayload(args *BuildPayloadArgs, witness bool) (*Payload, error) { - return miner.buildPayload(args, witness) +func (miner *Miner) BuildPayload(ctx context.Context, args *BuildPayloadArgs, witness bool) (*Payload, error) { + return miner.buildPayload(ctx, args, witness) } // getPending retrieves the pending block based on the current head block. @@ -155,16 +157,17 @@ func (miner *Miner) getPending() *newPayloadResult { if miner.chainConfig.IsShanghai(new(big.Int).Add(header.Number, big.NewInt(1)), timestamp, types.DeserializeHeaderExtraInformation(header).ArbOSFormatVersion) { withdrawal = []*types.Withdrawal{} } - ret := miner.generateWork(&generateParams{ - timestamp: timestamp, - forceTime: false, - parentHash: header.Hash(), - coinbase: miner.config.PendingFeeRecipient, - random: common.Hash{}, - withdrawals: withdrawal, - beaconRoot: nil, - noTxs: false, - }, false) // we will never make a witness for a pending block + ret := miner.generateWork(context.Background(), + &generateParams{ + timestamp: timestamp, + forceTime: false, + parentHash: header.Hash(), + coinbase: miner.config.PendingFeeRecipient, + random: common.Hash{}, + withdrawals: withdrawal, + beaconRoot: nil, + noTxs: false, + }, false) // we will never make a witness for a pending block if ret.err != nil { return nil } diff --git a/miner/miner_test.go b/miner/miner_test.go index 3c5e2b45b7..60d57efc7b 100644 --- a/miner/miner_test.go +++ b/miner/miner_test.go @@ -155,7 +155,7 @@ func createMiner(t *testing.T) *Miner { if err != nil { t.Fatalf("can't create new chain %v", err) } - statedb, _ := state.New(bc.Genesis().Root(), bc.StateCache()) + statedb, _ := state.New(bc.Genesis().Root(), state.NewDatabase(bc.TrieDB(), bc.CodeDB())) blockchain := &testBlockChain{bc.Genesis().Root(), chainConfig, statedb, 10000000, new(event.Feed)} pool := legacypool.New(testTxPoolConfig, blockchain) diff --git a/miner/payload_building.go b/miner/payload_building.go index 6b010186bf..ccaabec373 100644 --- a/miner/payload_building.go +++ b/miner/payload_building.go @@ -17,6 +17,7 @@ package miner import ( + "context" "crypto/sha256" "encoding/binary" "math/big" @@ -28,9 +29,11 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/stateless" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/telemetry" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" + "go.opentelemetry.io/otel/trace" ) // BuildPayloadArgs contains the provided parameters for building payload. @@ -43,6 +46,7 @@ type BuildPayloadArgs struct { Random common.Hash // The provided randomness value Withdrawals types.Withdrawals // The provided withdrawals BeaconRoot *common.Hash // The provided beaconRoot (Cancun) + SlotNum *uint64 // The provided slotNumber Version engine.PayloadVersion // Versioning byte for payload id calculation. } @@ -57,6 +61,9 @@ func (args *BuildPayloadArgs) Id() engine.PayloadID { if args.BeaconRoot != nil { hasher.Write(args.BeaconRoot[:]) } + if args.SlotNum != nil { + binary.Write(hasher, binary.BigEndian, args.SlotNum) + } var out engine.PayloadID copy(out[:], hasher.Sum(nil)[:8]) out[0] = byte(args.Version) @@ -97,14 +104,15 @@ func newPayload(empty *types.Block, emptyRequests [][]byte, witness *stateless.W return payload } -// update updates the full-block with latest built version. -func (payload *Payload) update(r *newPayloadResult, elapsed time.Duration) { +// update updates the full-block with latest built version. It returns true if +// the update was accepted (i.e. the new block has higher fees than the previous). +func (payload *Payload) update(r *newPayloadResult, elapsed time.Duration) (result bool) { payload.lock.Lock() defer payload.lock.Unlock() select { case <-payload.stop: - return // reject stale update + return false // reject stale update default: } // Ensure the newly provided full block has a higher transaction fee. @@ -129,8 +137,10 @@ func (payload *Payload) update(r *newPayloadResult, elapsed time.Duration) { "root", r.block.Root(), "elapsed", common.PrettyDuration(elapsed), ) + result = true } payload.cond.Broadcast() // fire signal for notifying full block + return } // Resolve returns the latest built payload and also terminates the background @@ -205,8 +215,33 @@ func (payload *Payload) ResolveFull() *engine.ExecutionPayloadEnvelope { return envelope } +func (miner *Miner) runBuildIteration(ctx context.Context, start time.Time, iteration int, payload *Payload, params *generateParams, witness bool) { + ctx, span, spanEnd := telemetry.StartSpan(ctx, "miner.buildIteration", + telemetry.Int64Attribute("iteration", int64(iteration)), + ) + var err error + defer spanEnd(&err) + + r := miner.generateWork(ctx, params, witness) + err = r.err + if err == nil { + accepted := payload.update(r, time.Since(start)) + span.SetAttributes(telemetry.BoolAttribute("update.accepted", accepted)) + } else { + log.Info("Error while generating work", "id", payload.id, "err", err) + } +} + // buildPayload builds the payload according to the provided parameters. -func (miner *Miner) buildPayload(args *BuildPayloadArgs, witness bool) (*Payload, error) { +func (miner *Miner) buildPayload(ctx context.Context, args *BuildPayloadArgs, witness bool) (result *Payload, err error) { + payloadID := args.Id() + ctx, _, spanEnd := telemetry.StartSpan(ctx, "miner.buildPayload", + telemetry.StringAttribute("payload.id", payloadID.String()), + telemetry.StringAttribute("parent.hash", args.Parent.String()), + telemetry.Int64Attribute("timestamp", int64(args.Timestamp)), + ) + defer spanEnd(&err) + // Build the initial version with no transaction included. It should be fast // enough to run. The empty payload can at least make sure there is something // to deliver for not missing slot. @@ -218,18 +253,28 @@ func (miner *Miner) buildPayload(args *BuildPayloadArgs, witness bool) (*Payload random: args.Random, withdrawals: args.Withdrawals, beaconRoot: args.BeaconRoot, + slotNum: args.SlotNum, noTxs: true, } - empty := miner.generateWork(emptyParams, witness) + empty := miner.generateWork(ctx, emptyParams, witness) if empty.err != nil { return nil, empty.err } // Construct a payload object for return. - payload := newPayload(empty.block, empty.requests, empty.witness, args.Id()) + payload := newPayload(empty.block, empty.requests, empty.witness, payloadID) // Spin up a routine for updating the payload in background. This strategy // can maximum the revenue for including transactions with highest fee. go func() { + var iteration int + bCtx, bSpan, bSpanEnd := telemetry.StartSpan(ctx, "miner.background", + telemetry.Int64Attribute("block.number", int64(empty.block.NumberU64())), + ) + defer func() { + bSpan.SetAttributes(telemetry.Int64Attribute("iterations.total", int64(iteration))) + bSpanEnd(nil) + }() + // Setup the timer for re-building the payload. The initial clock is kept // for triggering process immediately. timer := time.NewTimer(0) @@ -248,24 +293,34 @@ func (miner *Miner) buildPayload(args *BuildPayloadArgs, witness bool) (*Payload random: args.Random, withdrawals: args.Withdrawals, beaconRoot: args.BeaconRoot, + slotNum: args.SlotNum, noTxs: false, } - for { select { case <-timer.C: - start := time.Now() - r := miner.generateWork(fullParams, witness) - if r.err == nil { - payload.update(r, time.Since(start)) - } else { - log.Info("Error while generating work", "id", payload.id, "err", r.err) + // When block building takes close to the full recommit interval, + // the timer fires near-instantly on the next iteration. If the + // payload was resolved during that build, both timer.C and + // payload.stop are ready and Go's select picks one at random. + // Check payload.stop first to avoid an unnecessary generateWork. + select { + case <-payload.stop: + payload.updateSpanForDelivery(bSpan) + log.Info("Stopping work on payload", "id", payload.id, "reason", "delivery") + return + default: } - timer.Reset(miner.config.Recommit) + start := time.Now() + iteration++ + miner.runBuildIteration(bCtx, start, iteration, payload, fullParams, witness) + timer.Reset(max(0, miner.config.Recommit-time.Since(start))) case <-payload.stop: + payload.updateSpanForDelivery(bSpan) log.Info("Stopping work on payload", "id", payload.id, "reason", "delivery") return case <-endTimer.C: + bSpan.SetAttributes(telemetry.StringAttribute("exit.reason", "timeout")) log.Info("Stopping work on payload", "id", payload.id, "reason", "timeout") return } @@ -273,3 +328,37 @@ func (miner *Miner) buildPayload(args *BuildPayloadArgs, witness bool) (*Payload }() return payload, nil } + +func (payload *Payload) updateSpanForDelivery(bSpan trace.Span) { + payload.lock.Lock() + emptyDelivered := payload.full == nil + payload.lock.Unlock() + bSpan.SetAttributes( + telemetry.StringAttribute("exit.reason", "delivery"), + telemetry.BoolAttribute("empty.delivered", emptyDelivered), + ) +} + +// BuildTestingPayload is for testing_buildBlockV*. It creates a block with the exact content given +// by the parameters instead of using the locally available transactions. +func (miner *Miner) BuildTestingPayload(args *BuildPayloadArgs, transactions []*types.Transaction, empty bool, extraData []byte) (*engine.ExecutionPayloadEnvelope, error) { + fullParams := &generateParams{ + timestamp: args.Timestamp, + forceTime: true, + parentHash: args.Parent, + coinbase: args.FeeRecipient, + random: args.Random, + withdrawals: args.Withdrawals, + beaconRoot: args.BeaconRoot, + slotNum: args.SlotNum, + noTxs: empty, + forceOverrides: true, + overrideExtraData: extraData, + overrideTxs: transactions, + } + res := miner.generateWork(context.Background(), fullParams, false) + if res.err != nil { + return nil, res.err + } + return engine.BlockToExecutableData(res.block, new(big.Int), res.sidecars, res.requests), nil +} diff --git a/miner/payload_building_test.go b/miner/payload_building_test.go index 98da02611e..cb55301861 100644 --- a/miner/payload_building_test.go +++ b/miner/payload_building_test.go @@ -17,6 +17,7 @@ package miner import ( + "context" "math/big" "reflect" "testing" @@ -156,7 +157,7 @@ func TestBuildPayload(t *testing.T) { Random: common.Hash{}, FeeRecipient: recipient, } - payload, err := w.buildPayload(args, false) + payload, err := w.buildPayload(context.Background(), args, false) if err != nil { t.Fatalf("Failed to build payload %v", err) } diff --git a/miner/stress/main.go b/miner/stress/main.go new file mode 100644 index 0000000000..aaf0993c37 --- /dev/null +++ b/miner/stress/main.go @@ -0,0 +1,210 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// This file contains a miner stress test based on the Engine API flow. +package main + +import ( + "crypto/ecdsa" + "math/big" + "math/rand" + "os" + "os/signal" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/fdlimit" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/txpool/legacypool" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/catalyst" + "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/params" +) + +var refundContract = common.HexToAddress("0x1000000000000000000000000000000000000001") + +func main() { + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true))) + fdlimit.Raise(2048) + + // Generate a batch of accounts to seal and fund with + faucets := make([]*ecdsa.PrivateKey, 128) + for i := 0; i < len(faucets); i++ { + faucets[i], _ = crypto.GenerateKey() + } + // Create a post-merge network where blocks are built/inserted through + // engine API calls driven by a simulated beacon client. + genesis := makeGenesis(faucets) + + // Handle interrupts. + interruptCh := make(chan os.Signal, 5) + signal.Notify(interruptCh, os.Interrupt) + + // Start one node that accepts transactions and builds/inserts blocks via + // Engine API (through the simulated beacon driver). + stack, backend, beacon, err := makeNode(genesis) + if err != nil { + panic(err) + } + defer stack.Close() + defer beacon.Stop() + + // Start injecting transactions from the faucet like crazy + var ( + sent uint64 + nonces = make([]uint64, len(faucets)) + signer = types.LatestSigner(genesis.Config) + refundSet = true // slot 0 starts as non-zero in genesis + ) + for { + // Stop when interrupted. + select { + case <-interruptCh: + return + default: + } + + var ( + tx *types.Transaction + err error + ) + // Every third tx targets a contract path that alternates set/clear. + // Clearing a previously non-zero slot triggers gas refund. + if sent%3 == 0 { + var data []byte + if refundSet { + data = nil // empty calldata => clear slot to zero (refund path) + } else { + data = []byte{0x01} // non-empty calldata => set slot to one + } + tx, err = types.SignTx(types.NewTransaction(nonces[0], refundContract, new(big.Int), 50000, big.NewInt(100000000000), data), signer, faucets[0]) + if err != nil { + panic(err) + } + nonces[0]++ + refundSet = !refundSet + } else { + index := 1 + rand.Intn(len(faucets)-1) + tx, err = types.SignTx(types.NewTransaction(nonces[index], crypto.PubkeyToAddress(faucets[index].PublicKey), new(big.Int), 21000, big.NewInt(100000000000), nil), signer, faucets[index]) + if err != nil { + panic(err) + } + nonces[index]++ + } + errs := backend.TxPool().Add([]*types.Transaction{tx}, true) + for _, err := range errs { + if err != nil { + panic(err) + } + } + sent++ + + // Create and import blocks through the engine API path. + if sent%256 == 0 { + beacon.Commit() + } + + // Wait if we're too saturated + if pend, _ := backend.TxPool().Stats(); pend > 4096 { + beacon.Commit() + time.Sleep(50 * time.Millisecond) + } + } +} + +// makeGenesis creates a post-merge genesis block. +func makeGenesis(faucets []*ecdsa.PrivateKey) *core.Genesis { + config := *params.AllDevChainProtocolChanges + config.ChainID = big.NewInt(18) + + blockZero := uint64(0) + config.AmsterdamTime = &blockZero + config.BlobScheduleConfig.Amsterdam = ¶ms.BlobConfig{ + Target: 14, + Max: 21, + UpdateFraction: 13739630, + } + + genesis := &core.Genesis{ + Config: &config, + GasLimit: 25000000, + Alloc: types.GenesisAlloc{}, + } + for _, faucet := range faucets { + genesis.Alloc[crypto.PubkeyToAddress(faucet.PublicKey)] = types.Account{ + Balance: new(big.Int).Exp(big.NewInt(2), big.NewInt(128), nil), + } + } + // Runtime code: + // - empty calldata: SSTORE(0,0) + // - non-empty calldata: SSTORE(0,1) + // Slot 0 is initialized to 1 so the first clear includes gas refund. + genesis.Alloc[refundContract] = types.Account{ + Code: common.FromHex("0x3615600b576001600055005b600060005500"), + Storage: map[common.Hash]common.Hash{ + common.Hash{}: common.BigToHash(big.NewInt(1)), + }, + } + return genesis +} + +func makeNode(genesis *core.Genesis) (*node.Node, *eth.Ethereum, *catalyst.SimulatedBeacon, error) { + // Define the basic configurations for the Ethereum node + datadir, _ := os.MkdirTemp("", "") + + config := &node.Config{ + Name: "geth", + DataDir: datadir, + } + // Start the node and configure a full Ethereum node on it + stack, err := node.New(config) + if err != nil { + return nil, nil, nil, err + } + // Create and register the backend + ethBackend, err := eth.New(stack, ðconfig.Config{ + Genesis: genesis, + NetworkId: genesis.Config.ChainID.Uint64(), + SyncMode: downloader.FullSync, + DatabaseCache: 256, + DatabaseHandles: 256, + TxPool: legacypool.DefaultConfig, + GPO: ethconfig.Defaults.GPO, + Miner: ethconfig.Defaults.Miner, + SlowBlockThreshold: time.Second, + }) + if err != nil { + return nil, nil, nil, err + } + + if err := stack.Start(); err != nil { + return nil, nil, nil, err + } + driver, err := catalyst.NewSimulatedBeacon(0, common.Address{}, ethBackend) + if err != nil { + return nil, nil, nil, err + } + if err := driver.Start(); err != nil { + return nil, nil, nil, err + } + return stack, ethBackend, driver, nil +} diff --git a/miner/worker.go b/miner/worker.go index 9e54aecdbe..7e2b0474be 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -17,6 +17,7 @@ package miner import ( + "context" "errors" "fmt" "math/big" @@ -32,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/internal/telemetry" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/holiman/uint256" @@ -43,6 +45,16 @@ var ( errBlockInterruptedByTimeout = errors.New("timeout while building block") ) +// maxBlobsPerBlock returns the maximum number of blobs per block. +// Users can specify the maximum number of blobs per block if necessary. +func (miner *Miner) maxBlobsPerBlock(time uint64) int { + maxBlobs := eip4844.MaxBlobsPerBlock(miner.chainConfig, time) + if miner.config.MaxBlobsPerBlock != 0 { + maxBlobs = miner.config.MaxBlobsPerBlock + } + return maxBlobs +} + // environment is the worker's current environment and holds all // information of the sealing block generation. type environment struct { @@ -63,12 +75,17 @@ type environment struct { witness *stateless.Witness } -// txFits reports whether the transaction fits into the block size limit. -// Always true for Arbitrum (since arbtrium does not need a block size cap) +// txFitsSize reports whether the transaction fits into the block size limit. +// Always true for Arbitrum (since arbitrum does not need a block size cap). func (env *environment) txFitsSize(tx *types.Transaction) bool { return env.evm.ChainConfig().IsArbitrum() || env.size+tx.Size() < params.MaxBlockSize-maxBlockSizeBufferZone } +// discard terminates the background threads before discarding it. +func (env *environment) discard() { + env.state.StopPrefetcher() +} + const ( commitInterruptNone int32 = iota commitInterruptNewHead @@ -102,15 +119,36 @@ type generateParams struct { random common.Hash // The randomness generated by beacon chain, empty before the merge withdrawals types.Withdrawals // List of withdrawals to include in block (shanghai field) beaconRoot *common.Hash // The beacon root (cancun field). + slotNum *uint64 // The slot number (amsterdam field). noTxs bool // Flag whether an empty block without any transaction is expected + + forceOverrides bool // Flag whether we should overwrite extraData and transactions + overrideExtraData []byte + overrideTxs []*types.Transaction } // generateWork generates a sealing block based on the given parameters. -func (miner *Miner) generateWork(genParam *generateParams, witness bool) *newPayloadResult { - work, err := miner.prepareWork(genParam, witness) +func (miner *Miner) generateWork(ctx context.Context, genParam *generateParams, witness bool) (result *newPayloadResult) { + ctx, span, spanEnd := telemetry.StartSpan(ctx, "miner.generateWork") + defer func() { + if result != nil && result.err == nil { + span.SetAttributes( + telemetry.Int64Attribute("txs.count", int64(len(result.block.Transactions()))), + telemetry.Int64Attribute("gas.used", int64(result.block.GasUsed())), + telemetry.StringAttribute("fees", result.fees.String()), + ) + } + if result != nil { + spanEnd(&result.err) + } else { + spanEnd(nil) + } + }() + work, err := miner.prepareWork(ctx, genParam, witness) if err != nil { return &newPayloadResult{err: err} } + defer work.discard() // Check withdrawals fit max block size. // Due to the cap on withdrawal count, this can actually never happen, but we still need to @@ -124,15 +162,27 @@ func (miner *Miner) generateWork(genParam *generateParams, witness bool) *newPay work.size += uint64(genParam.withdrawals.Size()) if !genParam.noTxs { - interrupt := new(atomic.Int32) - timer := time.AfterFunc(miner.config.Recommit, func() { - interrupt.Store(commitInterruptTimeout) - }) - defer timer.Stop() - - err := miner.fillTransactions(interrupt, work) - if errors.Is(err, errBlockInterruptedByTimeout) { - log.Warn("Block building is interrupted", "allowance", common.PrettyDuration(miner.config.Recommit)) + // If forceOverrides is true and overrideTxs is not empty, commit the override transactions + // otherwise, fill the block with the current transactions from the txpool + if genParam.forceOverrides && len(genParam.overrideTxs) > 0 { + for _, tx := range genParam.overrideTxs { + work.state.SetTxContext(tx.Hash(), work.tcount) + if err := miner.commitTransaction(ctx, work, tx); err != nil { + // all passed transactions HAVE to be valid at this point + return &newPayloadResult{err: err} + } + } + } else { + interrupt := new(atomic.Int32) + timer := time.AfterFunc(miner.config.Recommit, func() { + interrupt.Store(commitInterruptTimeout) + }) + defer timer.Stop() + + err := miner.fillTransactions(ctx, interrupt, work) + if errors.Is(err, errBlockInterruptedByTimeout) { + log.Warn("Block building is interrupted", "allowance", common.PrettyDuration(miner.config.Recommit)) + } } } body := types.Body{Transactions: work.txs, Withdrawals: genParam.withdrawals} @@ -166,7 +216,7 @@ func (miner *Miner) generateWork(genParam *generateParams, witness bool) *newPay work.header.RequestsHash = &reqHash } - block, err := miner.engine.FinalizeAndAssemble(miner.chain, work.header, work.state, &body, work.receipts) + block, err := miner.engine.FinalizeAndAssemble(ctx, miner.chain, work.header, work.state, &body, work.receipts) if err != nil { return &newPayloadResult{err: err} } @@ -184,7 +234,9 @@ func (miner *Miner) generateWork(genParam *generateParams, witness bool) *newPay // prepareWork constructs the sealing task according to the given parameters, // either based on the last chain head or specified parent. In this function // the pending transactions are not filled yet, only the empty task returned. -func (miner *Miner) prepareWork(genParams *generateParams, witness bool) (*environment, error) { +func (miner *Miner) prepareWork(ctx context.Context, genParams *generateParams, witness bool) (result *environment, err error) { + _, _, spanEnd := telemetry.StartSpan(ctx, "miner.prepareWork") + defer spanEnd(&err) miner.confMu.RLock() defer miner.confMu.RUnlock() @@ -218,6 +270,9 @@ func (miner *Miner) prepareWork(genParams *generateParams, witness bool) (*envir if len(miner.config.ExtraData) != 0 { header.Extra = miner.config.ExtraData } + if genParams.forceOverrides { + header.Extra = genParams.overrideExtraData + } // Set the randomness field from the beacon chain if it's available. if genParams.random != (common.Hash{}) { header.MixDigest = genParams.random @@ -247,6 +302,13 @@ func (miner *Miner) prepareWork(genParams *generateParams, witness bool) (*envir header.ExcessBlobGas = &excessBlobGas header.ParentBeaconRoot = genParams.beaconRoot } + // Apply EIP-7843. + if miner.chainConfig.IsAmsterdam(header.Number, header.Time) { + if genParams.slotNum == nil { + return nil, errors.New("no slot number set post-amsterdam") + } + header.SlotNumber = genParams.slotNum + } // Could potentially happen if starting to mine in an odd state. // Note genParams.coinbase can be different with header.Coinbase // since clique algorithm can modify the coinbase field in header. @@ -271,26 +333,30 @@ func (miner *Miner) makeEnv(parent *types.Header, header *types.Header, coinbase if err != nil { return nil, err } + var bundle *stateless.Witness if witness { - bundle, err := stateless.NewWitness(header, miner.chain) + bundle, err = stateless.NewWitness(header, miner.chain, false) if err != nil { return nil, err } - state.StartPrefetcher("miner", bundle, nil) } + state.StartPrefetcher("miner", bundle) // Note the passed coinbase may be different with header.Coinbase. return &environment{ signer: types.MakeSigner(miner.chainConfig, header.Number, header.Time, params.MaxArbosVersionSupported), state: state, size: uint64(header.Size()), coinbase: coinbase, + gasPool: core.NewGasPool(header.GasLimit), header: header, witness: state.Witness(), evm: vm.NewEVM(core.NewEVMBlockContext(header, miner.chain, &coinbase), state, miner.chainConfig, vm.Config{}), }, nil } -func (miner *Miner) commitTransaction(env *environment, tx *types.Transaction) error { +func (miner *Miner) commitTransaction(ctx context.Context, env *environment, tx *types.Transaction) (err error) { + _, _, spanEnd := telemetry.StartSpan(ctx, "miner.commitTransaction") + defer spanEnd(&err) if tx.Type() == types.BlobTxType { return miner.commitBlobTransaction(env, tx) } @@ -314,7 +380,7 @@ func (miner *Miner) commitBlobTransaction(env *environment, tx *types.Transactio // isn't really a better place right now. The blob gas limit is checked at block validation time // and not during execution. This means core.ApplyTransaction will not return an error if the // tx has too many blobs. So we have to explicitly check it here. - maxBlobs := eip4844.MaxBlobsPerBlock(miner.chainConfig, env.header.Time) + maxBlobs := miner.maxBlobsPerBlock(env.header.Time) if env.blobs+len(sc.Blobs) > maxBlobs { return errors.New("max data blobs reached") } @@ -337,25 +403,22 @@ func (miner *Miner) commitBlobTransaction(env *environment, tx *types.Transactio func (miner *Miner) applyTransaction(env *environment, tx *types.Transaction) (*types.Receipt, error) { var ( snap = env.state.Snapshot() - gp = env.gasPool.Gas() + gp = env.gasPool.Snapshot() ) - receipt, _, err := core.ApplyTransaction(env.evm, env.gasPool, env.state, env.header, tx, &env.header.GasUsed) + receipt, _, err := core.ApplyTransaction(env.evm, env.gasPool, env.state, env.header, tx) if err != nil { env.state.RevertToSnapshot(snap) - env.gasPool.SetGas(gp) + env.gasPool.Set(gp) + return nil, err } - return receipt, err + env.header.GasUsed = env.gasPool.Used() + return receipt, nil } -func (miner *Miner) commitTransactions(env *environment, plainTxs, blobTxs *transactionsByPriceAndNonce, interrupt *atomic.Int32) error { - arbosVersion := types.DeserializeHeaderExtraInformation(env.header).ArbOSFormatVersion - var ( - isCancun = miner.chainConfig.IsCancun(env.header.Number, env.header.Time, arbosVersion) - gasLimit = env.header.GasLimit - ) - if env.gasPool == nil { - env.gasPool = new(core.GasPool).AddGas(gasLimit) - } +func (miner *Miner) commitTransactions(ctx context.Context, env *environment, plainTxs, blobTxs *transactionsByPriceAndNonce, interrupt *atomic.Int32) error { + ctx, _, spanEnd := telemetry.StartSpan(ctx, "miner.commitTransactions") + defer spanEnd(nil) + isCancun := miner.chainConfig.IsCancun(env.header.Number, env.header.Time, types.DeserializeHeaderExtraInformation(env.header).ArbOSFormatVersion) for { // Check interruption signal and abort building if it's fired. if interrupt != nil { @@ -370,7 +433,7 @@ func (miner *Miner) commitTransactions(env *environment, plainTxs, blobTxs *tran } // If we don't have enough blob space for any further blob transactions, // skip that list altogether - if !blobTxs.Empty() && env.blobs >= eip4844.MaxBlobsPerBlock(miner.chainConfig, env.header.Time) { + if !blobTxs.Empty() && env.blobs >= miner.maxBlobsPerBlock(env.header.Time) { log.Trace("Not enough blob space for further blob transactions") blobTxs.Clear() // Fall though to pick up any plain txs @@ -409,7 +472,7 @@ func (miner *Miner) commitTransactions(env *environment, plainTxs, blobTxs *tran // blobs or not, however the max check panics when called on a chain without // a defined schedule, so we need to verify it's safe to call. if isCancun { - left := eip4844.MaxBlobsPerBlock(miner.chainConfig, env.header.Time) - env.blobs + left := miner.maxBlobsPerBlock(env.header.Time) - env.blobs if left < int(ltx.BlobGas/params.BlobTxBlobGasPerBlob) { log.Trace("Not enough blob space left for transaction", "hash", ltx.Hash, "left", left, "needed", ltx.BlobGas/params.BlobTxBlobGasPerBlob) txs.Pop() @@ -444,7 +507,7 @@ func (miner *Miner) commitTransactions(env *environment, plainTxs, blobTxs *tran // Start executing the transaction env.state.SetTxContext(tx.Hash(), env.tcount) - err := miner.commitTransaction(env, tx) + err := miner.commitTransaction(ctx, env, tx) switch { case errors.Is(err, core.ErrNonceTooLow): // New head notification data race between the transaction pool and miner, shift @@ -468,7 +531,9 @@ func (miner *Miner) commitTransactions(env *environment, plainTxs, blobTxs *tran // fillTransactions retrieves the pending transactions from the txpool and fills them // into the given sealing block. The transaction selection and ordering strategy can // be customized with the plugin in the future. -func (miner *Miner) fillTransactions(interrupt *atomic.Int32, env *environment) error { +func (miner *Miner) fillTransactions(ctx context.Context, interrupt *atomic.Int32, env *environment) (err error) { + ctx, span, spanEnd := telemetry.StartSpan(ctx, "miner.fillTransactions") + defer spanEnd(&err) miner.confMu.RLock() tip := miner.config.GasPrice prio := miner.prio @@ -488,7 +553,7 @@ func (miner *Miner) fillTransactions(interrupt *atomic.Int32, env *environment) filter.GasLimitCap = params.MaxTxGasRenamedForNitroMerges } filter.BlobTxs = false - pendingPlainTxs := miner.txpool.Pending(filter) + pendingPlainTxs, plainTxCount := miner.txpool.Pending(filter) filter.BlobTxs = true if miner.chainConfig.IsOsaka(env.header.Number, env.header.Time, types.DeserializeHeaderExtraInformation(env.header).ArbOSFormatVersion) { @@ -496,7 +561,11 @@ func (miner *Miner) fillTransactions(interrupt *atomic.Int32, env *environment) } else { filter.BlobVersion = types.BlobSidecarVersion0 } - pendingBlobTxs := miner.txpool.Pending(filter) + pendingBlobTxs, blobTxCount := miner.txpool.Pending(filter) + span.SetAttributes( + telemetry.Int64Attribute("pending.plain.count", int64(plainTxCount)), + telemetry.Int64Attribute("pending.blob.count", int64(blobTxCount)), + ) // Split the pending transactions into locals and remotes. prioPlainTxs, normalPlainTxs := make(map[common.Address][]*txpool.LazyTransaction), pendingPlainTxs @@ -517,7 +586,7 @@ func (miner *Miner) fillTransactions(interrupt *atomic.Int32, env *environment) plainTxs := newTransactionsByPriceAndNonce(env.signer, prioPlainTxs, env.header.BaseFee) blobTxs := newTransactionsByPriceAndNonce(env.signer, prioBlobTxs, env.header.BaseFee) - if err := miner.commitTransactions(env, plainTxs, blobTxs, interrupt); err != nil { + if err := miner.commitTransactions(ctx, env, plainTxs, blobTxs, interrupt); err != nil { return err } } @@ -525,7 +594,7 @@ func (miner *Miner) fillTransactions(interrupt *atomic.Int32, env *environment) plainTxs := newTransactionsByPriceAndNonce(env.signer, normalPlainTxs, env.header.BaseFee) blobTxs := newTransactionsByPriceAndNonce(env.signer, normalBlobTxs, env.header.BaseFee) - if err := miner.commitTransactions(env, plainTxs, blobTxs, interrupt); err != nil { + if err := miner.commitTransactions(ctx, env, plainTxs, blobTxs, interrupt); err != nil { return err } } diff --git a/node/config.go b/node/config.go index dcf0fb44da..1cdebb6583 100644 --- a/node/config.go +++ b/node/config.go @@ -201,9 +201,7 @@ type Config struct { GraphQLVirtualHosts []string `toml:",omitempty"` // Logger is a custom logger to use with the p2p.Server. - Logger log.Logger `toml:",omitempty"` - - oldGethResourceWarning bool + Logger log.Logger `toml:"-,omitempty"` // AllowUnprotectedTxs allows non EIP-155 protected transactions to be send over RPC. AllowUnprotectedTxs bool `toml:",omitempty"` @@ -220,6 +218,7 @@ type Config struct { // EnablePersonal enables the deprecated personal namespace. EnablePersonal bool `toml:"-"` + // Configures database engine used by the node. DBEngine string `toml:",omitempty"` // HTTPBodyLimit is the maximum number of bytes allowed in the HTTP request body. @@ -227,6 +226,27 @@ type Config struct { // WSReadLimit is the maximum number of bytes allowed in the websocket request body. WSReadLimit int64 `toml:",omitempty"` + + // Configures OpenTelemetry reporting. + OpenTelemetry OpenTelemetryConfig `toml:",omitempty"` + + oldGethResourceWarning bool +} + +// OpenTelemetryConfig has settings for +type OpenTelemetryConfig struct { + Enabled bool `toml:",omitempty"` + + Tags string `toml:",omitempty"` + InstanceID string `toml:",omitempty"` + + // Exporter endpoint. + Endpoint string `toml:",omitempty"` + AuthUser string `toml:",omitempty"` + AuthPassword string `toml:",omitempty"` + + // Percentage of sampled traces. + SampleRatio float64 `toml:",omitempty"` } // IPCEndpoint resolves an IPC endpoint based on a configured value, taking into diff --git a/node/defaults.go b/node/defaults.go index 917b8bc0a2..3af605baaa 100644 --- a/node/defaults.go +++ b/node/defaults.go @@ -71,9 +71,11 @@ var DefaultConfig = Config{ BatchResponseMaxSize: 25 * 1000 * 1000, GraphQLVirtualHosts: []string{"localhost"}, P2P: p2p.Config{ - ListenAddr: ":30303", - MaxPeers: 50, - NAT: nat.Any(), + ListenAddr: ":30303", + MaxPeers: 50, + NAT: nat.Any(), + DiscoveryV4: true, + DiscoveryV5: true, }, DBEngine: "", // Use whatever exists, will default to Pebble if non-existent and supported } diff --git a/node/node.go b/node/node.go index 34a02f4f19..24139b66cb 100644 --- a/node/node.go +++ b/node/node.go @@ -158,8 +158,10 @@ func New(conf *Config) (*Node, error) { // Configure RPC servers. node.http = newHTTPServer(node.log, conf.HTTPTimeouts) node.httpAuth = newHTTPServer(node.log, conf.HTTPTimeouts) + node.httpAuth.disableHTTP2 = true // Engine API does not need HTTP/2 node.ws = newHTTPServer(node.log, rpc.DefaultHTTPTimeouts) node.wsAuth = newHTTPServer(node.log, rpc.DefaultHTTPTimeouts) + node.wsAuth.disableHTTP2 = true node.ipc = newIPCServer(node.log, conf.IPCEndpoint()) return node, nil diff --git a/node/rpcstack.go b/node/rpcstack.go index f9806c7384..8fad96ebfb 100644 --- a/node/rpcstack.go +++ b/node/rpcstack.go @@ -92,6 +92,9 @@ type httpServer struct { port int handlerNames map[string]string + + // disableHTTP2 disables HTTP/2 support on this server when set to true. + disableHTTP2 bool } const ( @@ -140,6 +143,11 @@ func (h *httpServer) start() error { // Initialize the server. h.server = &http.Server{Handler: h} + h.server.Protocols = new(http.Protocols) + h.server.Protocols.SetHTTP1(true) + if !h.disableHTTP2 { + h.server.Protocols.SetUnencryptedHTTP2(true) + } if h.timeouts != (rpc.HTTPTimeouts{}) { CheckTimeouts(&h.timeouts) h.server.ReadTimeout = h.timeouts.ReadTimeout diff --git a/node/rpcstack_test.go b/node/rpcstack_test.go index 54e58cccb2..bd75dac4eb 100644 --- a/node/rpcstack_test.go +++ b/node/rpcstack_test.go @@ -593,6 +593,38 @@ func TestHTTPWriteTimeout(t *testing.T) { }) } +func TestHTTP2H2C(t *testing.T) { + srv := createAndStartServer(t, &httpConfig{}, false, &wsConfig{}, nil) + defer srv.stop() + + // Create an HTTP/2 cleartext client. + transport := &http.Transport{} + transport.Protocols = new(http.Protocols) + transport.Protocols.SetUnencryptedHTTP2(true) + client := &http.Client{Transport: transport} + + body := strings.NewReader(`{"jsonrpc":"2.0","id":1,"method":"rpc_modules","params":[]}`) + resp, err := client.Post("http://"+srv.listenAddr(), "application/json", body) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + t.Fatalf("expected status 200, got %d", resp.StatusCode) + } + if resp.Proto != "HTTP/2.0" { + t.Fatalf("expected HTTP/2.0, got %s", resp.Proto) + } + result, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(result), "jsonrpc") { + t.Fatalf("unexpected response: %s", result) + } +} + func apis() []rpc.API { return []rpc.API{ { diff --git a/oss-fuzz.sh b/oss-fuzz.sh index 020b6fee27..73209fd8c8 100644 --- a/oss-fuzz.sh +++ b/oss-fuzz.sh @@ -35,7 +35,7 @@ function coverbuild { sed -i -e 's/TestFuzzCorpus/Test'$function'Corpus/' ./"${function,,}"_test.go cat << DOG > $OUT/$fuzzer -#/bin/sh +#!/bin/sh cd $OUT/$path go test -run Test${function}Corpus -v $tags -coverprofile \$1 -coverpkg $coverpkg @@ -64,7 +64,7 @@ function compile_fuzzer() { go get github.com/holiman/gofuzz-shim/testing if [[ $SANITIZER == *coverage* ]]; then - coverbuild $path $function $fuzzer $coverpkg + coverbuild $path $function $fuzzer else gofuzz-shim --func $function --package $package -f $file -o $fuzzer.a $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -o $OUT/$fuzzer diff --git a/p2p/dial.go b/p2p/dial.go index 225709427c..f9463d6d89 100644 --- a/p2p/dial.go +++ b/p2p/dial.go @@ -76,6 +76,7 @@ var ( errSelf = errors.New("is self") errAlreadyDialing = errors.New("already dialing") errAlreadyConnected = errors.New("already connected") + errPendingInbound = errors.New("peer has pending inbound connection") errRecentlyDialed = errors.New("recently dialed") errNetRestrict = errors.New("not contained in netrestrict list") errNoPort = errors.New("node does not provide TCP port") @@ -104,12 +105,15 @@ type dialScheduler struct { remStaticCh chan *enode.Node addPeerCh chan *conn remPeerCh chan *conn + addPendingCh chan enode.ID + remPendingCh chan enode.ID // Everything below here belongs to loop and // should only be accessed by code on the loop goroutine. - dialing map[enode.ID]*dialTask // active tasks - peers map[enode.ID]struct{} // all connected peers - dialPeers int // current number of dialed peers + dialing map[enode.ID]*dialTask // active tasks + peers map[enode.ID]struct{} // all connected peers + pendingInbound map[enode.ID]struct{} // in-progress inbound connections + dialPeers int // current number of dialed peers // The static map tracks all static dial tasks. The subset of usable static dial tasks // (i.e. those passing checkDial) is kept in staticPool. The scheduler prefers @@ -163,19 +167,22 @@ func (cfg dialConfig) withDefaults() dialConfig { func newDialScheduler(config dialConfig, it enode.Iterator, setupFunc dialSetupFunc) *dialScheduler { cfg := config.withDefaults() d := &dialScheduler{ - dialConfig: cfg, - historyTimer: mclock.NewAlarm(cfg.clock), - setupFunc: setupFunc, - dnsLookupFunc: net.DefaultResolver.LookupNetIP, - dialing: make(map[enode.ID]*dialTask), - static: make(map[enode.ID]*dialTask), - peers: make(map[enode.ID]struct{}), - doneCh: make(chan *dialTask), - nodesIn: make(chan *enode.Node), - addStaticCh: make(chan *enode.Node), - remStaticCh: make(chan *enode.Node), - addPeerCh: make(chan *conn), - remPeerCh: make(chan *conn), + dialConfig: cfg, + historyTimer: mclock.NewAlarm(cfg.clock), + setupFunc: setupFunc, + dnsLookupFunc: net.DefaultResolver.LookupNetIP, + dialing: make(map[enode.ID]*dialTask), + static: make(map[enode.ID]*dialTask), + peers: make(map[enode.ID]struct{}), + pendingInbound: make(map[enode.ID]struct{}), + doneCh: make(chan *dialTask), + nodesIn: make(chan *enode.Node), + addStaticCh: make(chan *enode.Node), + remStaticCh: make(chan *enode.Node), + addPeerCh: make(chan *conn), + remPeerCh: make(chan *conn), + addPendingCh: make(chan enode.ID), + remPendingCh: make(chan enode.ID), } d.lastStatsLog = d.clock.Now() d.ctx, d.cancel = context.WithCancel(context.Background()) @@ -223,6 +230,22 @@ func (d *dialScheduler) peerRemoved(c *conn) { } } +// inboundPending notifies the scheduler about a pending inbound connection. +func (d *dialScheduler) inboundPending(id enode.ID) { + select { + case d.addPendingCh <- id: + case <-d.ctx.Done(): + } +} + +// inboundCompleted notifies the scheduler that an inbound connection completed or failed. +func (d *dialScheduler) inboundCompleted(id enode.ID) { + select { + case d.remPendingCh <- id: + case <-d.ctx.Done(): + } +} + // loop is the main loop of the dialer. func (d *dialScheduler) loop(it enode.Iterator) { var ( @@ -276,6 +299,15 @@ loop: delete(d.peers, c.node.ID()) d.updateStaticPool(c.node.ID()) + case id := <-d.addPendingCh: + d.pendingInbound[id] = struct{}{} + d.log.Trace("Marked node as pending inbound", "id", id) + + case id := <-d.remPendingCh: + delete(d.pendingInbound, id) + d.updateStaticPool(id) + d.log.Trace("Unmarked node as pending inbound", "id", id) + case node := <-d.addStaticCh: id := node.ID() _, exists := d.static[id] @@ -390,6 +422,9 @@ func (d *dialScheduler) checkDial(n *enode.Node) error { if _, ok := d.peers[n.ID()]; ok { return errAlreadyConnected } + if _, ok := d.pendingInbound[n.ID()]; ok { + return errPendingInbound + } if d.netRestrict != nil && !d.netRestrict.ContainsAddr(n.IPAddr()) { return errNetRestrict } diff --git a/p2p/dial_test.go b/p2p/dial_test.go index f18dacce2a..9684aa6e91 100644 --- a/p2p/dial_test.go +++ b/p2p/dial_test.go @@ -423,6 +423,82 @@ func TestDialSchedDNSHostname(t *testing.T) { }) } +// This test checks that nodes with pending inbound connections are not dialed. +func TestDialSchedPendingInbound(t *testing.T) { + t.Parallel() + + config := dialConfig{ + maxActiveDials: 5, + maxDialPeers: 4, + } + runDialTest(t, config, []dialTestRound{ + // 2 peers are connected, leaving 2 dial slots. + // Node 0x03 has a pending inbound connection. + // Discovered nodes 0x03, 0x04, 0x05 but only 0x04 and 0x05 should be dialed. + { + peersAdded: []*conn{ + {flags: dynDialedConn, node: newNode(uintID(0x01), "127.0.0.1:30303")}, + {flags: dynDialedConn, node: newNode(uintID(0x02), "127.0.0.2:30303")}, + }, + update: func(d *dialScheduler) { + d.inboundPending(uintID(0x03)) + }, + discovered: []*enode.Node{ + newNode(uintID(0x03), "127.0.0.3:30303"), // not dialed because pending inbound + newNode(uintID(0x04), "127.0.0.4:30303"), + newNode(uintID(0x05), "127.0.0.5:30303"), + }, + wantNewDials: []*enode.Node{ + newNode(uintID(0x04), "127.0.0.4:30303"), + newNode(uintID(0x05), "127.0.0.5:30303"), + }, + }, + // Pending inbound connection for 0x03 completes successfully. + // Node 0x03 becomes a connected peer. + // One dial slot remains, node 0x06 is dialed. + { + update: func(d *dialScheduler) { + // Pending inbound completes + d.inboundCompleted(uintID(0x03)) + }, + peersAdded: []*conn{ + {flags: inboundConn, node: newNode(uintID(0x03), "127.0.0.3:30303")}, + }, + succeeded: []enode.ID{ + uintID(0x04), + }, + failed: []enode.ID{ + uintID(0x05), + }, + discovered: []*enode.Node{ + newNode(uintID(0x03), "127.0.0.3:30303"), // not dialed, now connected + newNode(uintID(0x06), "127.0.0.6:30303"), + }, + wantNewDials: []*enode.Node{ + newNode(uintID(0x06), "127.0.0.6:30303"), + }, + }, + // Inbound peer 0x03 disconnects. + // Another pending inbound starts for 0x07. + // Only 0x03 should be dialed, not 0x07. + { + peersRemoved: []enode.ID{ + uintID(0x03), + }, + update: func(d *dialScheduler) { + d.inboundPending(uintID(0x07)) + }, + discovered: []*enode.Node{ + newNode(uintID(0x03), "127.0.0.3:30303"), + newNode(uintID(0x07), "127.0.0.7:30303"), // not dialed because pending inbound + }, + wantNewDials: []*enode.Node{ + newNode(uintID(0x03), "127.0.0.3:30303"), + }, + }, + }) +} + // ------- // Code below here is the framework for the tests above. diff --git a/p2p/discover/table.go b/p2p/discover/table.go index e5b2c7c8c5..721cd7b589 100644 --- a/p2p/discover/table.go +++ b/p2p/discover/table.go @@ -25,6 +25,7 @@ package discover import ( "context" "fmt" + "net" "net/netip" "slices" "sync" @@ -36,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/p2p/netutil" ) @@ -205,6 +207,13 @@ func (tab *Table) close() { func (tab *Table) setFallbackNodes(nodes []*enode.Node) error { nursery := make([]*enode.Node, 0, len(nodes)) for _, n := range nodes { + if n.Hostname() != "" && !n.IPAddr().IsValid() { + resolved, err := resolveBootnodeHostname(n, tab.log) + if err != nil { + return fmt.Errorf("bad bootstrap node %q: %v", n, err) + } + n = resolved + } if err := n.ValidateComplete(); err != nil { return fmt.Errorf("bad bootstrap node %q: %v", n, err) } @@ -218,6 +227,42 @@ func (tab *Table) setFallbackNodes(nodes []*enode.Node) error { return nil } +// resolveBootnodeHostname resolves the DNS hostname of a bootstrap node to an IP address. +func resolveBootnodeHostname(n *enode.Node, logger log.Logger) (*enode.Node, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + ips, err := net.DefaultResolver.LookupNetIP(ctx, "ip", n.Hostname()) + if err != nil { + return nil, fmt.Errorf("DNS lookup failed for %q: %v", n.Hostname(), err) + } + + var ip4, ip6 netip.Addr + for _, ip := range ips { + if ip.Is4() && !ip4.IsValid() { + ip4 = ip + } + if ip.Is6() && !ip6.IsValid() { + ip6 = ip + } + } + if !ip4.IsValid() && !ip6.IsValid() { + return nil, fmt.Errorf("no IP addresses found for hostname %q", n.Hostname()) + } + + rec := n.Record() + if ip4.IsValid() { + rec.Set(enr.IPv4Addr(ip4)) + } + if ip6.IsValid() { + rec.Set(enr.IPv6Addr(ip6)) + } + rec.SetSeq(n.Seq()) + resolved := enode.SignNull(rec, n.ID()).WithHostname(n.Hostname()) + logger.Debug("Resolved bootstrap node hostname", "name", n.Hostname(), "ip", resolved.IP()) + return resolved, nil +} + // isInitDone returns whether the table's initial seeding procedure has completed. func (tab *Table) isInitDone() bool { select { diff --git a/p2p/discover/table_test.go b/p2p/discover/table_test.go index 8cc4ae33b2..c3b71ea5a6 100644 --- a/p2p/discover/table_test.go +++ b/p2p/discover/table_test.go @@ -490,6 +490,66 @@ func quickcfg() *quick.Config { } } +func TestSetFallbackNodes_DNSHostname(t *testing.T) { + // Create a node with a DNS hostname but no IP, simulating an enode URL + // like enode://@localhost:30303. + key := newkey() + node := enode.NewV4(&key.PublicKey, nil, 30303, 30303).WithHostname("localhost") + + // Verify the node has a hostname but no valid IP. + if node.Hostname() != "localhost" { + t.Fatal("expected hostname to be set") + } + if node.IPAddr().IsValid() { + t.Fatal("expected no IP address") + } + + // Create a table and set the hostname node as a bootnode. + // This should resolve the hostname to an IP address. + db, _ := enode.OpenDB(t.TempDir() + "/node.db") + defer db.Close() + + cfg := Config{Log: testlog.Logger(t, log.LvlTrace)} + cfg = cfg.withDefaults() + tab := &Table{ + cfg: cfg, + log: cfg.Log, + refreshReq: make(chan chan struct{}), + revalResponseCh: make(chan revalidationResponse), + addNodeCh: make(chan addNodeOp), + addNodeHandled: make(chan bool), + trackRequestCh: make(chan trackRequestOp), + initDone: make(chan struct{}), + closeReq: make(chan struct{}), + closed: make(chan struct{}), + ips: netutil.DistinctNetSet{Subnet: tableSubnet, Limit: tableIPLimit}, + } + for i := range tab.buckets { + tab.buckets[i] = &bucket{ + index: i, + ips: netutil.DistinctNetSet{Subnet: bucketSubnet, Limit: bucketIPLimit}, + } + } + + err := tab.setFallbackNodes([]*enode.Node{node}) + if err != nil { + t.Fatalf("setFallbackNodes failed: %v", err) + } + if len(tab.nursery) != 1 { + t.Fatalf("expected 1 nursery node, got %d", len(tab.nursery)) + } + + // The resolved node should have a valid IP and retain the hostname. + resolved := tab.nursery[0] + if !resolved.IPAddr().IsValid() { + t.Fatal("expected resolved node to have a valid IP") + } + if resolved.Hostname() != "localhost" { + t.Errorf("expected hostname to be preserved, got %q", resolved.Hostname()) + } + t.Logf("resolved localhost to %v", resolved.IPAddr()) +} + func newkey() *ecdsa.PrivateKey { key, err := crypto.GenerateKey() if err != nil { diff --git a/p2p/discover/v5_udp_test.go b/p2p/discover/v5_udp_test.go index 6abe20d7a4..9429fbaf0a 100644 --- a/p2p/discover/v5_udp_test.go +++ b/p2p/discover/v5_udp_test.go @@ -856,10 +856,10 @@ type testCodecFrame struct { } func (c *testCodec) Encode(toID enode.ID, addr string, p v5wire.Packet, _ *v5wire.Whoareyou) ([]byte, v5wire.Nonce, error) { - // To match the behavior of v5wire.Codec, we return the cached encoding of - // WHOAREYOU challenges. - if wp, ok := p.(*v5wire.Whoareyou); ok && len(wp.Encoded) > 0 { - return wp.Encoded, wp.Nonce, nil + if wp, ok := p.(*v5wire.Whoareyou); ok && len(wp.ChallengeData) > 0 { + // To match the behavior of v5wire.Codec, we return the cached encoding of + // WHOAREYOU challenges. + return wp.ChallengeData, wp.Nonce, nil } c.ctr++ @@ -874,7 +874,7 @@ func (c *testCodec) Encode(toID enode.ID, addr string, p v5wire.Packet, _ *v5wir // Store recently sent challenges. if w, ok := p.(*v5wire.Whoareyou); ok { w.Nonce = authTag - w.Encoded = frame + w.ChallengeData = frame if c.sentChallenges == nil { c.sentChallenges = make(map[enode.ID]*v5wire.Whoareyou) } @@ -911,6 +911,7 @@ func (c *testCodec) decodeFrame(input []byte) (frame testCodecFrame, p v5wire.Pa case v5wire.WhoareyouPacket: dec := new(v5wire.Whoareyou) err = rlp.DecodeBytes(frame.Packet, &dec) + dec.ChallengeData = bytes.Clone(input) p = dec default: p, err = v5wire.DecodeMessage(frame.Ptype, frame.Packet) diff --git a/p2p/discover/v5wire/encoding.go b/p2p/discover/v5wire/encoding.go index 08292a70ba..9a5bbd4612 100644 --- a/p2p/discover/v5wire/encoding.go +++ b/p2p/discover/v5wire/encoding.go @@ -190,10 +190,16 @@ func (c *Codec) Encode(id enode.ID, addr string, packet Packet, challenge *Whoar ) switch { case packet.Kind() == WhoareyouPacket: - // just send the WHOAREYOU packet raw again, rather than the re-encoded challenge data w := packet.(*Whoareyou) - if len(w.Encoded) > 0 { - return w.Encoded, w.Nonce, nil + if len(w.ChallengeData) > 0 { + // This WHOAREYOU packet was encoded before, so it's a resend. + // The unmasked packet content is stored in w.ChallengeData. + // Just apply the masking again to finish encoding. + c.buf.Reset() + c.buf.Write(w.ChallengeData) + copy(head.IV[:], w.ChallengeData) + enc := applyMasking(id, head.IV, c.buf.Bytes()) + return enc, w.Nonce, nil } head, err = c.encodeWhoareyou(id, packet.(*Whoareyou)) case challenge != nil: @@ -228,7 +234,6 @@ func (c *Codec) Encode(id enode.ID, addr string, packet Packet, challenge *Whoar if err != nil { return nil, Nonce{}, err } - challenge.Encoded = bytes.Clone(enc) c.sc.storeSentHandshake(id, addr, challenge) return enc, head.Nonce, err } @@ -246,14 +251,10 @@ func (c *Codec) Encode(id enode.ID, addr string, packet Packet, challenge *Whoar // EncodeRaw encodes a packet with the given header. func (c *Codec) EncodeRaw(id enode.ID, head Header, msgdata []byte) ([]byte, error) { + // header c.writeHeaders(&head) - - // Apply masking. - masked := c.buf.Bytes()[sizeofMaskingIV:] - mask := head.mask(id) - mask.XORKeyStream(masked[:], masked[:]) - - // Write message data. + applyMasking(id, head.IV, c.buf.Bytes()) + // message data c.buf.Write(msgdata) return c.buf.Bytes(), nil } @@ -326,7 +327,6 @@ func (c *Codec) encodeWhoareyou(toID enode.ID, packet *Whoareyou) (Header, error // Create header. head := c.makeHeader(toID, flagWhoareyou, 0) - head.AuthData = slices.Clone(c.buf.Bytes()) head.Nonce = packet.Nonce // Encode auth data. @@ -464,7 +464,7 @@ func (c *Codec) Decode(inputData []byte, addr string) (src enode.ID, n *enode.No // Unmask the static header. var head Header copy(head.IV[:], input[:sizeofMaskingIV]) - mask := head.mask(c.localnode.ID()) + mask := createMask(c.localnode.ID(), head.IV) staticHeader := input[sizeofMaskingIV:sizeofStaticPacketData] mask.XORKeyStream(staticHeader, staticHeader) @@ -680,10 +680,17 @@ func (h *StaticHeader) checkValid(packetLen int, protocolID [6]byte) error { } // mask returns a cipher for 'masking' / 'unmasking' packet headers. -func (h *Header) mask(destID enode.ID) cipher.Stream { +func createMask(destID enode.ID, iv [16]byte) cipher.Stream { block, err := aes.NewCipher(destID[:16]) if err != nil { panic("can't create cipher") } - return cipher.NewCTR(block, h.IV[:]) + return cipher.NewCTR(block, iv[:]) +} + +func applyMasking(destID enode.ID, iv [16]byte, packet []byte) []byte { + masked := packet[sizeofMaskingIV:] + mask := createMask(destID, iv) + mask.XORKeyStream(masked[:], masked[:]) + return packet } diff --git a/p2p/discover/v5wire/encoding_test.go b/p2p/discover/v5wire/encoding_test.go index 5774cb3d8c..dd7ec6d53c 100644 --- a/p2p/discover/v5wire/encoding_test.go +++ b/p2p/discover/v5wire/encoding_test.go @@ -269,6 +269,35 @@ func TestHandshake_BadHandshakeAttack(t *testing.T) { net.nodeB.expectDecodeErr(t, errUnexpectedHandshake, findnode) } +func TestEncodeWhoareyouResend(t *testing.T) { + t.Parallel() + net := newHandshakeTest() + defer net.close() + + // A -> B WHOAREYOU + challenge := &Whoareyou{ + Nonce: Nonce{1, 2, 3, 4}, + IDNonce: testIDnonce, + RecordSeq: 0, + } + enc, _ := net.nodeA.encode(t, net.nodeB, challenge) + net.nodeB.expectDecode(t, WhoareyouPacket, enc) + whoareyou1 := bytes.Clone(enc) + + if len(challenge.ChallengeData) == 0 { + t.Fatal("ChallengeData not assigned by encode") + } + + // A -> B WHOAREYOU + // Send the same challenge again. This should produce exactly + // the same bytes as the first send. + enc, _ = net.nodeA.encode(t, net.nodeB, challenge) + whoareyou2 := bytes.Clone(enc) + if !bytes.Equal(whoareyou2, whoareyou1) { + t.Fatal("re-encoded challenge not equal to first") + } +} + // This test checks some malformed packets. func TestDecodeErrorsV5(t *testing.T) { t.Parallel() diff --git a/p2p/discover/v5wire/msg.go b/p2p/discover/v5wire/msg.go index 089fd4ebdc..e5c29276ca 100644 --- a/p2p/discover/v5wire/msg.go +++ b/p2p/discover/v5wire/msg.go @@ -63,19 +63,20 @@ type ( // WHOAREYOU contains the handshake challenge. Whoareyou struct { - ChallengeData []byte // Encoded challenge - Nonce Nonce // Nonce of request packet - IDNonce [16]byte // Identity proof data - RecordSeq uint64 // ENR sequence number of recipient + Nonce Nonce // Nonce of request packet + IDNonce [16]byte // Identity proof data + RecordSeq uint64 // ENR sequence number of recipient // Node is the locally known node record of recipient. // This must be set by the caller of Encode. - Node *enode.Node + Node *enode.Node `rlp:"-"` - sent mclock.AbsTime // for handshake GC. + // ChallengeData stores the unmasked encoding of the whole packet. This is the + // input data for verification. It is assigned by both Encode and Decode + // operations. + ChallengeData []byte `rlp:"-"` - // Encoded is packet raw data for sending out, but should not be include in the RLP encoding. - Encoded []byte `rlp:"-"` + sent mclock.AbsTime // for handshake GC. } // PING is sent during liveness checks. diff --git a/p2p/nat/natupnp.go b/p2p/nat/natupnp.go index d79677db55..b9570e561f 100644 --- a/p2p/nat/natupnp.go +++ b/p2p/nat/natupnp.go @@ -107,30 +107,30 @@ func (n *upnp) addAnyPortMapping(protocol string, extport, intport int, ip net.I }) } // For IGDv1 and v1 services we should first try to add with extport. + var lastErr error for i := 0; i < retryCount+1; i++ { - err := n.withRateLimit(func() error { + lastErr = n.withRateLimit(func() error { return n.client.AddPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS) }) - if err == nil { + if lastErr == nil { return uint16(extport), nil } - log.Debug("Failed to add port mapping", "protocol", protocol, "extport", extport, "intport", intport, "err", err) + log.Debug("Failed to add port mapping", "protocol", protocol, "extport", extport, "intport", intport, "err", lastErr) } // If above fails, we retry with a random port. // We retry several times because of possible port conflicts. - var err error for i := 0; i < randomCount; i++ { extport = n.randomPort() - err := n.withRateLimit(func() error { + lastErr = n.withRateLimit(func() error { return n.client.AddPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS) }) - if err == nil { + if lastErr == nil { return uint16(extport), nil } - log.Debug("Failed to add random port mapping", "protocol", protocol, "extport", extport, "intport", intport, "err", err) + log.Debug("Failed to add random port mapping", "protocol", protocol, "extport", extport, "intport", intport, "err", lastErr) } - return 0, err + return 0, lastErr } func (n *upnp) randomPort() int { diff --git a/p2p/rlpx/rlpx.go b/p2p/rlpx/rlpx.go index a36ebc6b59..70658afdf8 100644 --- a/p2p/rlpx/rlpx.go +++ b/p2p/rlpx/rlpx.go @@ -24,6 +24,7 @@ import ( "crypto/ecdsa" "crypto/hmac" "crypto/rand" + "crypto/subtle" "encoding/binary" "errors" "fmt" @@ -34,7 +35,6 @@ import ( "time" "github.com/ethereum/go-ethereum/arbcrypto" - "github.com/ethereum/go-ethereum/common/bitutil" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto/ecies" "github.com/ethereum/go-ethereum/rlp" @@ -677,6 +677,6 @@ func exportPubkey(pub *ecies.PublicKey) []byte { func xor(one, other []byte) (xor []byte) { xor = make([]byte, len(one)) - bitutil.XORBytes(xor, one, other) + subtle.XORBytes(xor, one, other) return xor } diff --git a/p2p/rlpx/rlpx_oracle_poc_test.go b/p2p/rlpx/rlpx_oracle_poc_test.go new file mode 100644 index 0000000000..3497f0026e --- /dev/null +++ b/p2p/rlpx/rlpx_oracle_poc_test.go @@ -0,0 +1,57 @@ +package rlpx + +import ( + "bytes" + "errors" + "testing" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/ecies" +) + +func TestHandshakeECIESInvalidCurveOracle(t *testing.T) { + initKey, err := crypto.GenerateKey() + if err != nil { + t.Fatal(err) + } + respKey, err := crypto.GenerateKey() + if err != nil { + t.Fatal(err) + } + + init := handshakeState{ + initiator: true, + remote: ecies.ImportECDSAPublic(&respKey.PublicKey), + } + authMsg, err := init.makeAuthMsg(initKey) + if err != nil { + t.Fatal(err) + } + packet, err := init.sealEIP8(authMsg) + if err != nil { + t.Fatal(err) + } + + var recv handshakeState + if _, err := recv.readMsg(new(authMsgV4), respKey, bytes.NewReader(packet)); err != nil { + t.Fatalf("expected valid packet to decrypt: %v", err) + } + + tampered := append([]byte(nil), packet...) + if len(tampered) < 2+65 { + t.Fatalf("unexpected packet length %d", len(tampered)) + } + tampered[2] = 0x04 + for i := 1; i < 65; i++ { + tampered[2+i] = 0x00 + } + + var recv2 handshakeState + _, err = recv2.readMsg(new(authMsgV4), respKey, bytes.NewReader(tampered)) + if err == nil { + t.Fatal("expected decryption failure for invalid curve point") + } + if !errors.Is(err, ecies.ErrInvalidPublicKey) { + t.Fatalf("unexpected error: %v", err) + } +} diff --git a/p2p/server.go b/p2p/server.go index 10c855f1c4..6d2323f9ce 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -686,8 +686,11 @@ running: // Ensure that the trusted flag is set before checking against MaxPeers. c.flags |= trustedConn } - // TODO: track in-progress inbound node IDs (pre-Peer) to avoid dialing them. - c.cont <- srv.postHandshakeChecks(peers, inboundCount, c) + err := srv.postHandshakeChecks(peers, inboundCount, c) + if err == nil && c.flags&inboundConn != 0 { + srv.dialsched.inboundPending(c.node.ID()) + } + c.cont <- err case c := <-srv.checkpointAddPeer: // At this point the connection is past the protocol handshake. @@ -870,6 +873,11 @@ func (srv *Server) checkInboundConn(remoteIP netip.Addr) error { // or the handshakes have failed. func (srv *Server) SetupConn(fd net.Conn, flags connFlag, dialDest *enode.Node) error { c := &conn{fd: fd, flags: flags, cont: make(chan error)} + defer func() { + if c.is(inboundConn) && c.node != nil { + srv.dialsched.inboundCompleted(c.node.ID()) + } + }() if dialDest == nil { c.transport = srv.newTransport(fd, nil) } else { diff --git a/p2p/tracker/tracker.go b/p2p/tracker/tracker.go index 99dd42514c..35e1cb1a7c 100644 --- a/p2p/tracker/tracker.go +++ b/p2p/tracker/tracker.go @@ -18,52 +18,63 @@ package tracker import ( "container/list" + "errors" "fmt" "sync" "time" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/p2p" ) const ( - // trackedGaugeName is the prefix of the per-packet request tracking. trackedGaugeName = "p2p/tracked" - - // lostMeterName is the prefix of the per-packet request expirations. - lostMeterName = "p2p/lost" - - // staleMeterName is the prefix of the per-packet stale responses. - staleMeterName = "p2p/stale" - - // waitHistName is the prefix of the per-packet (req only) waiting time histograms. - waitHistName = "p2p/wait" + lostMeterName = "p2p/lost" + staleMeterName = "p2p/stale" + waitHistName = "p2p/wait" // maxTrackedPackets is a huge number to act as a failsafe on the number of // pending requests the node will track. It should never be hit unless an // attacker figures out a way to spin requests. - maxTrackedPackets = 100000 + maxTrackedPackets = 10000 ) -// request tracks sent network requests which have not yet received a response. -type request struct { - peer string - version uint // Protocol version +var ( + ErrNoMatchingRequest = errors.New("no matching request") + ErrTooManyItems = errors.New("response is larger than request allows") + ErrCollision = errors.New("request ID collision") + ErrCodeMismatch = errors.New("wrong response code for request") + ErrLimitReached = errors.New("request limit reached") + ErrStopped = errors.New("tracker stopped") +) - reqCode uint64 // Protocol message code of the request - resCode uint64 // Protocol message code of the expected response +// Request tracks sent network requests which have not yet received a response. +type Request struct { + ID uint64 // Request ID + Size int // Number/size of requested items + ReqCode uint64 // Protocol message code of the request + RespCode uint64 // Protocol message code of the expected response time time.Time // Timestamp when the request was made expire *list.Element // Expiration marker to untrack it } +type Response struct { + ID uint64 // Request ID of the response + MsgCode uint64 // Protocol message code + Size int // number/size of items in response +} + // Tracker is a pending network request tracker to measure how much time it takes // a remote peer to respond. type Tracker struct { - protocol string // Protocol capability identifier for the metrics - timeout time.Duration // Global timeout after which to drop a tracked packet + cap p2p.Cap // Protocol capability identifier for the metrics - pending map[uint64]*request // Currently pending requests + peer string // Peer ID + timeout time.Duration // Global timeout after which to drop a tracked packet + + pending map[uint64]*Request // Currently pending requests expire *list.List // Linked list tracking the expiration order wake *time.Timer // Timer tracking the expiration of the next item @@ -72,52 +83,53 @@ type Tracker struct { // New creates a new network request tracker to monitor how much time it takes to // fill certain requests and how individual peers perform. -func New(protocol string, timeout time.Duration) *Tracker { +func New(cap p2p.Cap, peerID string, timeout time.Duration) *Tracker { return &Tracker{ - protocol: protocol, - timeout: timeout, - pending: make(map[uint64]*request), - expire: list.New(), + cap: cap, + peer: peerID, + timeout: timeout, + pending: make(map[uint64]*Request), + expire: list.New(), } } // Track adds a network request to the tracker to wait for a response to arrive // or until the request it cancelled or times out. -func (t *Tracker) Track(peer string, version uint, reqCode uint64, resCode uint64, id uint64) { - if !metrics.Enabled() { - return - } +func (t *Tracker) Track(req Request) error { t.lock.Lock() defer t.lock.Unlock() + if t.expire == nil { + return ErrStopped + } + // If there's a duplicate request, we've just random-collided (or more probably, // we have a bug), report it. We could also add a metric, but we're not really // expecting ourselves to be buggy, so a noisy warning should be enough. - if _, ok := t.pending[id]; ok { - log.Error("Network request id collision", "protocol", t.protocol, "version", version, "code", reqCode, "id", id) - return + if _, ok := t.pending[req.ID]; ok { + log.Error("Network request id collision", "cap", t.cap, "code", req.ReqCode, "id", req.ID) + return ErrCollision } // If we have too many pending requests, bail out instead of leaking memory if pending := len(t.pending); pending >= maxTrackedPackets { - log.Error("Request tracker exceeded allowance", "pending", pending, "peer", peer, "protocol", t.protocol, "version", version, "code", reqCode) - return + log.Error("Request tracker exceeded allowance", "pending", pending, "peer", t.peer, "cap", t.cap, "code", req.ReqCode) + return ErrLimitReached } + // Id doesn't exist yet, start tracking it - t.pending[id] = &request{ - peer: peer, - version: version, - reqCode: reqCode, - resCode: resCode, - time: time.Now(), - expire: t.expire.PushBack(id), + req.time = time.Now() + req.expire = t.expire.PushBack(req.ID) + t.pending[req.ID] = &req + + if metrics.Enabled() { + t.trackedGauge(req.ReqCode).Inc(1) } - g := fmt.Sprintf("%s/%s/%d/%#02x", trackedGaugeName, t.protocol, version, reqCode) - metrics.GetOrRegisterGauge(g, nil).Inc(1) // If we've just inserted the first item, start the expiration timer if t.wake == nil { t.wake = time.AfterFunc(t.timeout, t.clean) } + return nil } // clean is called automatically when a preset time passes without a response @@ -126,6 +138,10 @@ func (t *Tracker) clean() { t.lock.Lock() defer t.lock.Unlock() + if t.expire == nil { + return // Tracker was stopped. + } + // Expire anything within a certain threshold (might be no items at all if // we raced with the delivery) for t.expire.Len() > 0 { @@ -142,62 +158,111 @@ func (t *Tracker) clean() { t.expire.Remove(head) delete(t.pending, id) - g := fmt.Sprintf("%s/%s/%d/%#02x", trackedGaugeName, t.protocol, req.version, req.reqCode) - metrics.GetOrRegisterGauge(g, nil).Dec(1) - - m := fmt.Sprintf("%s/%s/%d/%#02x", lostMeterName, t.protocol, req.version, req.reqCode) - metrics.GetOrRegisterMeter(m, nil).Mark(1) + if metrics.Enabled() { + t.trackedGauge(req.ReqCode).Dec(1) + t.lostMeter(req.ReqCode).Mark(1) + } } t.schedule() } -// schedule starts a timer to trigger on the expiration of the first network -// packet. +// schedule starts a timer to trigger on the expiration of the first network packet. func (t *Tracker) schedule() { if t.expire.Len() == 0 { t.wake = nil return } - t.wake = time.AfterFunc(time.Until(t.pending[t.expire.Front().Value.(uint64)].time.Add(t.timeout)), t.clean) + nextID := t.expire.Front().Value.(uint64) + nextTime := t.pending[nextID].time + t.wake = time.AfterFunc(time.Until(nextTime.Add(t.timeout)), t.clean) } -// Fulfil fills a pending request, if any is available, reporting on various metrics. -func (t *Tracker) Fulfil(peer string, version uint, code uint64, id uint64) { - if !metrics.Enabled() { - return +// Stop reclaims resources of the tracker. +func (t *Tracker) Stop() { + t.lock.Lock() + defer t.lock.Unlock() + + if t.wake != nil { + t.wake.Stop() + t.wake = nil + } + if metrics.Enabled() { + // Ensure metrics are decremented for pending requests. + counts := make(map[uint64]int64) + for _, req := range t.pending { + counts[req.ReqCode]++ + } + for code, count := range counts { + t.trackedGauge(code).Dec(count) + } } + clear(t.pending) + t.expire = nil +} + +// Fulfil fills a pending request, if any is available, reporting on various metrics. +func (t *Tracker) Fulfil(resp Response) error { t.lock.Lock() defer t.lock.Unlock() // If it's a non existing request, track as stale response - req, ok := t.pending[id] + req, ok := t.pending[resp.ID] if !ok { - m := fmt.Sprintf("%s/%s/%d/%#02x", staleMeterName, t.protocol, version, code) - metrics.GetOrRegisterMeter(m, nil).Mark(1) - return + if metrics.Enabled() { + t.staleMeter(resp.MsgCode).Mark(1) + } + return ErrNoMatchingRequest } + // If the response is funky, it might be some active attack - if req.peer != peer || req.version != version || req.resCode != code { - log.Warn("Network response id collision", - "have", fmt.Sprintf("%s:%s/%d:%d", peer, t.protocol, version, code), - "want", fmt.Sprintf("%s:%s/%d:%d", peer, t.protocol, req.version, req.resCode), + if req.RespCode != resp.MsgCode { + log.Warn("Network response code collision", + "have", fmt.Sprintf("%s:%s/%d:%d", t.peer, t.cap.Name, t.cap.Version, resp.MsgCode), + "want", fmt.Sprintf("%s:%s/%d:%d", t.peer, t.cap.Name, t.cap.Version, req.RespCode), ) - return + return ErrCodeMismatch } + if resp.Size > req.Size { + return ErrTooManyItems + } + // Everything matches, mark the request serviced and meter it + wasHead := req.expire.Prev() == nil t.expire.Remove(req.expire) - delete(t.pending, id) - if req.expire.Prev() == nil { + delete(t.pending, req.ID) + if wasHead { if t.wake.Stop() { t.schedule() } } - g := fmt.Sprintf("%s/%s/%d/%#02x", trackedGaugeName, t.protocol, req.version, req.reqCode) - metrics.GetOrRegisterGauge(g, nil).Dec(1) - h := fmt.Sprintf("%s/%s/%d/%#02x", waitHistName, t.protocol, req.version, req.reqCode) + // Update request metrics. + if metrics.Enabled() { + t.trackedGauge(req.ReqCode).Dec(1) + t.waitHistogram(req.ReqCode).Update(time.Since(req.time).Microseconds()) + } + return nil +} + +func (t *Tracker) trackedGauge(code uint64) *metrics.Gauge { + name := fmt.Sprintf("%s/%s/%d/%#02x", trackedGaugeName, t.cap.Name, t.cap.Version, code) + return metrics.GetOrRegisterGauge(name, nil) +} + +func (t *Tracker) lostMeter(code uint64) *metrics.Meter { + name := fmt.Sprintf("%s/%s/%d/%#02x", lostMeterName, t.cap.Name, t.cap.Version, code) + return metrics.GetOrRegisterMeter(name, nil) +} + +func (t *Tracker) staleMeter(code uint64) *metrics.Meter { + name := fmt.Sprintf("%s/%s/%d/%#02x", staleMeterName, t.cap.Name, t.cap.Version, code) + return metrics.GetOrRegisterMeter(name, nil) +} + +func (t *Tracker) waitHistogram(code uint64) metrics.Histogram { + name := fmt.Sprintf("%s/%s/%d/%#02x", waitHistName, t.cap.Name, t.cap.Version, code) sampler := func() metrics.Sample { return metrics.NewBoundedHistogramSample() } - metrics.GetOrRegisterHistogramLazy(h, nil, sampler).Update(time.Since(req.time).Microseconds()) + return metrics.GetOrRegisterHistogramLazy(name, nil, sampler) } diff --git a/p2p/tracker/tracker_test.go b/p2p/tracker/tracker_test.go new file mode 100644 index 0000000000..95e9629641 --- /dev/null +++ b/p2p/tracker/tracker_test.go @@ -0,0 +1,83 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package tracker + +import ( + "testing" + "time" + + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/p2p" +) + +// TestCleanAfterStop verifies that the clean method does not crash when called +// after Stop. This can happen because clean is scheduled via time.AfterFunc and +// may fire after Stop sets t.expire to nil. +func TestCleanAfterStop(t *testing.T) { + cap := p2p.Cap{Name: "test", Version: 1} + timeout := 50 * time.Millisecond + tr := New(cap, "peer1", timeout) + + // Track a request to start the expiration timer. + tr.Track(Request{ID: 1, ReqCode: 0x01, RespCode: 0x02, Size: 1}) + + // Stop the tracker, then wait for the timer to fire. + tr.Stop() + time.Sleep(timeout + 50*time.Millisecond) + + // Also verify that calling clean directly after stop doesn't panic. + tr.clean() +} + +// This checks that metrics gauges for pending requests are be decremented when a +// Tracker is stopped. +func TestMetricsOnStop(t *testing.T) { + metrics.Enable() + + cap := p2p.Cap{Name: "test", Version: 1} + tr := New(cap, "peer1", time.Minute) + + // Track some requests with different ReqCodes. + var id uint64 + for i := 0; i < 3; i++ { + tr.Track(Request{ID: id, ReqCode: 0x01, RespCode: 0x02, Size: 1}) + id++ + } + for i := 0; i < 5; i++ { + tr.Track(Request{ID: id, ReqCode: 0x03, RespCode: 0x04, Size: 1}) + id++ + } + + gauge1 := tr.trackedGauge(0x01) + gauge2 := tr.trackedGauge(0x03) + + if gauge1.Snapshot().Value() != 3 { + t.Fatalf("gauge1 value mismatch: got %d, want 3", gauge1.Snapshot().Value()) + } + if gauge2.Snapshot().Value() != 5 { + t.Fatalf("gauge2 value mismatch: got %d, want 5", gauge2.Snapshot().Value()) + } + + tr.Stop() + + if gauge1.Snapshot().Value() != 0 { + t.Fatalf("gauge1 value after stop: got %d, want 0", gauge1.Snapshot().Value()) + } + if gauge2.Snapshot().Value() != 0 { + t.Fatalf("gauge2 value after stop: got %d, want 0", gauge2.Snapshot().Value()) + } +} diff --git a/params/config.go b/params/config.go index 794e4e62b1..8189ec9218 100644 --- a/params/config.go +++ b/params/config.go @@ -1436,7 +1436,6 @@ func (err *ConfigCompatError) Error() string { // phases. type Rules struct { IsArbitrum, IsStylus, IsDia bool - ChainID *big.Int ArbOSVersion uint64 IsHomestead, IsEIP150, IsEIP155, IsEIP158 bool IsEIP2929, IsEIP4762 bool @@ -1444,22 +1443,39 @@ type Rules struct { IsBerlin, IsLondon bool IsMerge, IsShanghai, IsCancun, IsPrague, IsOsaka bool IsAmsterdam, IsVerkle bool + + // Arbitrum: per-chain code-size limits, derived from + // ArbitrumChainParams.MaxCodeSize/MaxInitCodeSize when set, or from the + // upstream Amsterdam/default constants otherwise. + MaxCodeSize, MaxInitCodeSize uint64 } // Rules ensures c's ChainID is not nil. func (c *ChainConfig) Rules(num *big.Int, isMerge bool, timestamp uint64, currentArbosVersion uint64) Rules { - chainID := c.ChainID - if chainID == nil { - chainID = new(big.Int) - } // disallow setting Merge out of order isMerge = isMerge && c.IsLondon(num) isVerkle := isMerge && c.IsVerkle(num, timestamp) + isAmsterdam := isMerge && c.IsAmsterdam(num, timestamp) + + // Arbitrum chains may override the protocol code-size limits via + // ArbitrumChainParams; non-Arbitrum chains follow the Amsterdam-aware + // upstream defaults. + var maxCodeSize, maxInitCodeSize uint64 + if c.IsArbitrum() { + maxCodeSize = c.MaxCodeSize() + maxInitCodeSize = c.MaxInitCodeSize() + } else if isAmsterdam { + maxCodeSize = MaxCodeSizeAmsterdam + maxInitCodeSize = MaxInitCodeSizeAmsterdam + } else { + maxCodeSize = MaxCodeSize + maxInitCodeSize = MaxInitCodeSize + } + return Rules{ IsArbitrum: c.IsArbitrum(), IsStylus: c.IsArbitrum() && currentArbosVersion >= ArbosVersion_Stylus, IsDia: c.IsArbitrum() && currentArbosVersion >= ArbosVersion_Dia, - ChainID: new(big.Int).Set(chainID), ArbOSVersion: currentArbosVersion, IsHomestead: c.IsHomestead(num), IsEIP150: c.IsEIP150(num), @@ -1477,8 +1493,10 @@ func (c *ChainConfig) Rules(num *big.Int, isMerge bool, timestamp uint64, curren IsCancun: isMerge && c.IsCancun(num, timestamp, currentArbosVersion), IsPrague: isMerge && c.IsPrague(num, timestamp, currentArbosVersion), IsOsaka: isMerge && c.IsOsaka(num, timestamp, currentArbosVersion), - IsAmsterdam: isMerge && c.IsAmsterdam(num, timestamp), + IsAmsterdam: isAmsterdam, IsVerkle: isVerkle, IsEIP4762: isVerkle, + MaxCodeSize: maxCodeSize, + MaxInitCodeSize: maxInitCodeSize, } } diff --git a/params/protocol_params.go b/params/protocol_params.go index fe35011517..2e0e62987a 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -32,7 +32,7 @@ const ( MaximumExtraDataSize uint64 = 32 // Maximum size extra data may be after Genesis. ExpByteGas uint64 = 10 // Times ceil(log256(exponent)) for the EXP instruction. - SloadGas uint64 = 50 // Multiplied by the number of 32-byte words that are copied (round up) for any *COPY operation and added. + SloadGas uint64 = 50 // CallValueTransferGas uint64 = 9000 // Paid for CALL when the value transfer is non-zero. CallNewAccountGas uint64 = 25000 // Paid for CALL when the destination address didn't exist prior. TxGas uint64 = 21000 // Per transaction not creating a contract. NOTE: Not payable on data of calls between transactions. @@ -82,7 +82,7 @@ const ( CallCreateDepth uint64 = 1024 // Maximum depth of call/create stack. ExpGas uint64 = 10 // Once per EXP instruction LogGas uint64 = 375 // Per LOG* operation. - CopyGas uint64 = 3 // + CopyGas uint64 = 3 // Multiplied by the number of 32-byte words that are copied (round up) for any *COPY operation and added. StackLimit uint64 = 1024 // Maximum size of VM stack allowed. TierStepGas uint64 = 0 // Once per operation, for a selection of them. LogTopicGas uint64 = 375 // Multiplied by the * of the LOG*, per LOG transaction. e.g. LOG0 incurs 0 * c_txLogTopicGas, LOG4 incurs 4 * c_txLogTopicGas. @@ -137,8 +137,14 @@ const ( DefaultElasticityMultiplier = 2 // Bounds the maximum gas limit an EIP-1559 block may have. InitialBaseFee = 1000000000 // Initial base fee for EIP-1559 blocks. - DefaultMaxCodeSize = 24576 // Maximum bytecode to permit for a contract - DefaultMaxInitCodeSize = 2 * DefaultMaxCodeSize // Maximum initcode to permit in a creation transaction and create instructions + MaxCodeSize = 24576 // Maximum bytecode to permit for a contract + MaxInitCodeSize = 2 * MaxCodeSize // Maximum initcode to permit in a creation transaction and create instructions + MaxCodeSizeAmsterdam = 32768 // Maximum bytecode to permit for a contract post Amsterdam + MaxInitCodeSizeAmsterdam = 2 * MaxCodeSizeAmsterdam // Maximum initcode to permit in a creation transaction and create instructions post Amsterdam + + // Arbitrum: per-ChainConfig defaults; overridable via ChainConfig.MaxCodeSize/MaxInitCodeSize. + DefaultMaxCodeSize = MaxCodeSize // Default maximum bytecode to permit for a contract + DefaultMaxInitCodeSize = MaxInitCodeSize // Default maximum initcode to permit in a creation transaction and create instructions // Precompiled contract gas prices @@ -228,3 +234,11 @@ var ( ConsolidationQueueAddress = common.HexToAddress("0x0000BBdDc7CE488642fb579F8B00f3a590007251") ConsolidationQueueCode = common.FromHex("3373fffffffffffffffffffffffffffffffffffffffe1460d35760115f54807fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1461019a57600182026001905f5b5f82111560685781019083028483029004916001019190604d565b9093900492505050366060146088573661019a573461019a575f5260205ff35b341061019a57600154600101600155600354806004026004013381556001015f358155600101602035815560010160403590553360601b5f5260605f60143760745fa0600101600355005b6003546002548082038060021160e7575060025b5f5b8181146101295782810160040260040181607402815460601b815260140181600101548152602001816002015481526020019060030154905260010160e9565b910180921461013b5790600255610146565b90505f6002555f6003555b5f54807fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff141561017357505f5b6001546001828201116101885750505f61018e565b01600190035b5f555f6001556074025ff35b5f5ffd") ) + +// System log events. +var ( + // EIP-7708 - System logs emitted for ETH transfer and burn + EthTransferLogEvent = common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef") // keccak256('Transfer(address,address,uint256)') + EthBurnLogEvent = common.HexToHash("0xcc16f5dbb4873280815c1ee09dbd06736cffcc184412cf7a71a0fdb75d397ca5") // keccak256('Burn(address,uint256)') + +) diff --git a/rlp/encbuffer.go b/rlp/encbuffer.go index 61d8bd059c..ca0aa290fe 100644 --- a/rlp/encbuffer.go +++ b/rlp/encbuffer.go @@ -366,6 +366,16 @@ func (w *EncoderBuffer) AppendToBytes(dst []byte) []byte { return out } +// Size returns the total size of the content that was encoded up to this point. +// Note this does not count the size of any lists which are still 'open' (i.e. for +// which ListEnd has not been called yet). +func (w EncoderBuffer) Size() int { + if w.buf == nil { + return 0 + } + return w.buf.size() +} + // Write appends b directly to the encoder output. func (w EncoderBuffer) Write(b []byte) (int, error) { return w.buf.Write(b) diff --git a/rlp/encode.go b/rlp/encode.go index ed99275739..9d04e6324a 100644 --- a/rlp/encode.go +++ b/rlp/encode.go @@ -102,6 +102,29 @@ func EncodeToReader(val interface{}) (size int, r io.Reader, err error) { return buf.size(), &encReader{buf: buf}, nil } +// EncodeToRawList encodes val as an RLP list and returns it as a RawList. +func EncodeToRawList[T any](val []T) (RawList[T], error) { + if len(val) == 0 { + return RawList[T]{}, nil + } + + // Encode the value to an internal buffer. + buf := getEncBuffer() + defer encBufferPool.Put(buf) + if err := buf.encode(val); err != nil { + return RawList[T]{}, err + } + + // Create the RawList. RawList assumes the initial list header is padded + // 9 bytes, so we have to determine the offset where the value should be + // placed. + contentSize := buf.lheads[0].size + bytes := make([]byte, contentSize+9) + offset := 9 - headsize(uint64(contentSize)) + buf.copyTo(bytes[offset:]) + return RawList[T]{enc: bytes, length: len(val)}, nil +} + type listhead struct { offset int // index of this header in string data size int // total size of encoded data (including list headers) diff --git a/rlp/encode_test.go b/rlp/encode_test.go index e63ea319b4..8dc9fdaf1f 100644 --- a/rlp/encode_test.go +++ b/rlp/encode_test.go @@ -507,6 +507,39 @@ func TestEncodeToReaderReturnToPool(t *testing.T) { wg.Wait() } +func TestEncoderBufferSize(t *testing.T) { + var output bytes.Buffer + eb := NewEncoderBuffer(&output) + + assertSize := func(state string, expectedSize int) { + t.Helper() + if s := eb.Size(); s != expectedSize { + t.Fatalf("wrong size %s: %d", state, s) + } + } + + assertSize("empty buffer", 0) + outerList := eb.List() + assertSize("after outer List()", 0) + eb.WriteString("abc") + assertSize("after string write", 4) + innerList := eb.List() + assertSize("after inner List()", 4) + eb.WriteUint64(1) + eb.WriteUint64(2) + assertSize("after inner list writes", 6) + eb.ListEnd(innerList) + assertSize("after end of inner list", 7) + eb.ListEnd(outerList) + assertSize("after end of outer list", 8) + eb.Flush() + assertSize("after Flush()", 0) + + if output.Len() != 8 { + t.Fatalf("wrong final output size %d", output.Len()) + } +} + var sink interface{} func BenchmarkIntsize(b *testing.B) { diff --git a/rlp/iterator.go b/rlp/iterator.go index 95bd3f2582..9e41cec947 100644 --- a/rlp/iterator.go +++ b/rlp/iterator.go @@ -16,44 +16,74 @@ package rlp -type listIterator struct { - data []byte - next []byte - err error +// Iterator is an iterator over the elements of an encoded container. +type Iterator struct { + data []byte + next []byte + offset int + err error } -// NewListIterator creates an iterator for the (list) represented by data -func NewListIterator(data RawValue) (*listIterator, error) { +// NewListIterator creates an iterator for the (list) represented by data. +func NewListIterator(data RawValue) (Iterator, error) { k, t, c, err := readKind(data) if err != nil { - return nil, err + return Iterator{}, err } if k != List { - return nil, ErrExpectedList - } - it := &listIterator{ - data: data[t : t+c], + return Iterator{}, ErrExpectedList } + it := newIterator(data[t:t+c], int(t)) return it, nil } -// Next forwards the iterator one step, returns true if it was not at end yet -func (it *listIterator) Next() bool { +func newIterator(data []byte, initialOffset int) Iterator { + return Iterator{data: data, offset: initialOffset} +} + +// Next forwards the iterator one step. +// Returns true if there is a next item or an error occurred on this step (check Err()). +// On parse error, the iterator is marked finished and subsequent calls return false. +func (it *Iterator) Next() bool { if len(it.data) == 0 { return false } _, t, c, err := readKind(it.data) - it.next = it.data[:t+c] - it.data = it.data[t+c:] - it.err = err + if err != nil { + it.next = nil + it.err = err + // Mark iteration as finished to avoid potential infinite loops on subsequent Next calls. + it.data = nil + return true + } + length := t + c + it.next = it.data[:length] + it.data = it.data[length:] + it.offset += int(length) + it.err = nil return true } -// Value returns the current value -func (it *listIterator) Value() []byte { +// Value returns the current value. +func (it *Iterator) Value() []byte { return it.next } -func (it *listIterator) Err() error { +// Count returns the remaining number of items. +// Note this is O(n) and the result may be incorrect if the list data is invalid. +// The returned count is always an upper bound on the remaining items +// that will be visited by the iterator. +func (it *Iterator) Count() int { + count, _ := CountValues(it.data) + return count +} + +// Offset returns the offset of the current value into the list data. +func (it *Iterator) Offset() int { + return it.offset - len(it.next) +} + +// Err returns the error that caused Next to return false, if any. +func (it *Iterator) Err() error { return it.err } diff --git a/rlp/iterator_test.go b/rlp/iterator_test.go index a22aaec862..275d4371c7 100644 --- a/rlp/iterator_test.go +++ b/rlp/iterator_test.go @@ -17,6 +17,7 @@ package rlp import ( + "io" "testing" "github.com/ethereum/go-ethereum/common/hexutil" @@ -38,14 +39,25 @@ func TestIterator(t *testing.T) { t.Fatal("expected two elems, got zero") } txs := it.Value() + if offset := it.Offset(); offset != 3 { + t.Fatal("wrong offset", offset, "want 3") + } + // Check that uncles exist if !it.Next() { t.Fatal("expected two elems, got one") } + if offset := it.Offset(); offset != 219 { + t.Fatal("wrong offset", offset, "want 219") + } + txit, err := NewListIterator(txs) if err != nil { t.Fatal(err) } + if c := txit.Count(); c != 2 { + t.Fatal("wrong Count:", c) + } var i = 0 for txit.Next() { if txit.err != nil { @@ -57,3 +69,65 @@ func TestIterator(t *testing.T) { t.Errorf("count wrong, expected %d got %d", i, exp) } } + +func TestIteratorErrors(t *testing.T) { + tests := []struct { + input []byte + wantCount int // expected Count before iterating + wantErr error + }{ + // Second item string header claims 3 bytes content, but only 2 remain. + {unhex("C4 01 83AABB"), 2, ErrValueTooLarge}, + // Second item truncated: B9 requires 2 size bytes, none available. + {unhex("C2 01 B9"), 2, io.ErrUnexpectedEOF}, + // 0x05 should be encoded directly, not as 81 05. + {unhex("C3 01 8105"), 2, ErrCanonSize}, + // Long-form string header B8 used for 1-byte content (< 56). + {unhex("C4 01 B801AA"), 2, ErrCanonSize}, + // Long-form list header F8 used for 1-byte content (< 56). + {unhex("C4 01 F80101"), 2, ErrCanonSize}, + } + for _, tt := range tests { + it, err := NewListIterator(tt.input) + if err != nil { + t.Fatal("NewListIterator error:", err) + } + if c := it.Count(); c != tt.wantCount { + t.Fatalf("%x: Count = %d, want %d", tt.input, c, tt.wantCount) + } + n := 0 + for it.Next() { + if it.Err() != nil { + break + } + n++ + } + if wantN := tt.wantCount - 1; n != wantN { + t.Fatalf("%x: got %d valid items, want %d", tt.input, n, wantN) + } + if it.Err() != tt.wantErr { + t.Fatalf("%x: got error %v, want %v", tt.input, it.Err(), tt.wantErr) + } + if it.Next() { + t.Fatalf("%x: Next returned true after error", tt.input) + } + } +} + +func FuzzIteratorCount(f *testing.F) { + examples := [][]byte{unhex("010203"), unhex("018142"), unhex("01830202")} + for _, e := range examples { + f.Add(e) + } + f.Fuzz(func(t *testing.T, in []byte) { + it := newIterator(in, 0) + count := it.Count() + i := 0 + for it.Next() { + i++ + } + if i != count { + t.Fatalf("%x: count %d not equal to %d iterations", in, count, i) + } + }) +} diff --git a/rlp/raw.go b/rlp/raw.go index cec90346a1..5f41cad5c4 100644 --- a/rlp/raw.go +++ b/rlp/raw.go @@ -17,8 +17,10 @@ package rlp import ( + "fmt" "io" "reflect" + "slices" ) // RawValue represents an encoded RLP value and can be used to delay @@ -28,6 +30,156 @@ type RawValue []byte var rawValueType = reflect.TypeFor[RawValue]() +// RawList represents an encoded RLP list. +type RawList[T any] struct { + // The list is stored in encoded form. + // Note this buffer has some special properties: + // + // - if the buffer is nil, it's the zero value, representing + // an empty list. + // - if the buffer is non-nil, it must have a length of at least + // 9 bytes, which is reserved padding for the encoded list header. + // The remaining bytes, enc[9:], store the content bytes of the list. + // + // The implementation code mostly works with the Content method because it + // returns something valid either way. + enc []byte + + // length holds the number of items in the list. + length int +} + +// Content returns the RLP-encoded data of the list. +// This does not include the list-header. +// The return value is a direct reference to the internal buffer, not a copy. +func (r *RawList[T]) Content() []byte { + if r.enc == nil { + return nil + } else { + return r.enc[9:] + } +} + +// EncodeRLP writes the encoded list to the writer. +func (r RawList[T]) EncodeRLP(w io.Writer) error { + _, err := w.Write(r.Bytes()) + return err +} + +// Bytes returns the RLP encoding of the list. +// Note the return value aliases the internal buffer. +func (r *RawList[T]) Bytes() []byte { + if r == nil || r.enc == nil { + return []byte{0xC0} // zero value encodes as empty list + } + n := puthead(r.enc, 0xC0, 0xF7, uint64(len(r.Content()))) + copy(r.enc[9-n:], r.enc[:n]) + return r.enc[9-n:] +} + +// DecodeRLP decodes the list. This does not perform validation of the items! +func (r *RawList[T]) DecodeRLP(s *Stream) error { + k, size, err := s.Kind() + if err != nil { + return err + } + if k != List { + return fmt.Errorf("%w for %T", ErrExpectedList, r) + } + enc := make([]byte, 9+size) + if err := s.readFull(enc[9:]); err != nil { + return err + } + n, err := CountValues(enc[9:]) + if err != nil { + if err == ErrValueTooLarge { + return ErrElemTooLarge + } + return err + } + *r = RawList[T]{enc: enc, length: n} + return nil +} + +// Items decodes and returns all items in the list. +func (r *RawList[T]) Items() ([]T, error) { + items := make([]T, r.Len()) + it := r.ContentIterator() + for i := 0; it.Next(); i++ { + if err := DecodeBytes(it.Value(), &items[i]); err != nil { + return items[:i], err + } + } + return items, nil +} + +// Len returns the number of items in the list. +func (r *RawList[T]) Len() int { + return r.length +} + +// Size returns the encoded size of the list. +func (r *RawList[T]) Size() uint64 { + return ListSize(uint64(len(r.Content()))) +} + +// ContentIterator returns an iterator over the content of the list. +// Note the offsets returned by iterator.Offset are relative to the +// Content bytes of the list. +func (r *RawList[T]) ContentIterator() Iterator { + return newIterator(r.Content(), 0) +} + +// Append adds an item to the end of the list. +func (r *RawList[T]) Append(item T) error { + if r.enc == nil { + r.enc = make([]byte, 9) + } + + eb := getEncBuffer() + defer encBufferPool.Put(eb) + + if err := eb.encode(item); err != nil { + return err + } + prevEnd := len(r.enc) + end := prevEnd + eb.size() + r.enc = slices.Grow(r.enc, eb.size())[:end] + eb.copyTo(r.enc[prevEnd:end]) + r.length++ + return nil +} + +// AppendRaw adds an encoded item to the list. +// The given byte slice must contain exactly one RLP value. +func (r *RawList[T]) AppendRaw(b []byte) error { + _, tagsize, contentsize, err := readKind(b) + if err != nil { + return err + } + if tagsize+contentsize != uint64(len(b)) { + return fmt.Errorf("rlp: input has trailing bytes in AppendRaw") + } + if r.enc == nil { + r.enc = make([]byte, 9) + } + r.enc = append(r.enc, b...) + r.length++ + return nil +} + +// AppendList appends all items from another RawList to this list. +func (r *RawList[T]) AppendList(other *RawList[T]) { + if other.enc == nil || other.length == 0 { + return + } + if r.enc == nil { + r.enc = make([]byte, 9) + } + r.enc = append(r.enc, other.Content()...) + r.length += other.length +} + // StringSize returns the encoded size of a string. func StringSize(s string) uint64 { switch n := len(s); n { @@ -145,13 +297,52 @@ func CountValues(b []byte) (int, error) { for ; len(b) > 0; i++ { _, tagsize, size, err := readKind(b) if err != nil { - return 0, err + return i + 1, err } b = b[tagsize+size:] } return i, nil } +// SplitListValues extracts the raw elements from the list RLP-encoding blob. +// +// Note: the returned slice must not be modified, as it shares the same +// backing array as the original slice. It's acceptable to deep-copy the elements +// out if necessary, but let's stick with this approach for less allocation +// overhead. +func SplitListValues(b []byte) ([][]byte, error) { + b, _, err := SplitList(b) + if err != nil { + return nil, err + } + n, err := CountValues(b) + if err != nil { + return nil, err + } + var elements = make([][]byte, 0, n) + + for len(b) > 0 { + _, tagsize, size, err := readKind(b) + if err != nil { + return nil, err + } + elements = append(elements, b[:tagsize+size]) + b = b[tagsize+size:] + } + return elements, nil +} + +// MergeListValues takes a list of raw elements and rlp-encodes them as list. +func MergeListValues(elems [][]byte) ([]byte, error) { + w := NewEncoderBuffer(nil) + offset := w.List() + for _, elem := range elems { + w.Write(elem) + } + w.ListEnd(offset) + return w.ToBytes(), nil +} + func readKind(buf []byte) (k Kind, tagsize, contentsize uint64, err error) { if len(buf) == 0 { return 0, 0, 0, io.ErrUnexpectedEOF diff --git a/rlp/raw_test.go b/rlp/raw_test.go index 7b3255eca3..ed7d3524c2 100644 --- a/rlp/raw_test.go +++ b/rlp/raw_test.go @@ -19,11 +19,307 @@ package rlp import ( "bytes" "errors" + "fmt" "io" + "reflect" "testing" "testing/quick" ) +type rawListTest[T any] struct { + input string + content string + items []T + length int +} + +func (test rawListTest[T]) name() string { + return fmt.Sprintf("%T-%d", *new(T), test.length) +} + +func (test rawListTest[T]) run(t *testing.T) { + // check decoding and properties + input := unhex(test.input) + inputSize := len(input) + var rl RawList[T] + if err := DecodeBytes(input, &rl); err != nil { + t.Fatal("decode failed:", err) + } + if l := rl.Len(); l != test.length { + t.Fatalf("wrong Len %d, want %d", l, test.length) + } + if sz := rl.Size(); sz != uint64(inputSize) { + t.Fatalf("wrong Size %d, want %d", sz, inputSize) + } + items, err := rl.Items() + if err != nil { + t.Fatal("Items failed:", err) + } + if !reflect.DeepEqual(items, test.items) { + t.Fatal("wrong items:", items) + } + if !bytes.Equal(rl.Content(), unhex(test.content)) { + t.Fatalf("wrong Content %x, want %s", rl.Content(), test.content) + } + if !bytes.Equal(rl.Bytes(), unhex(test.input)) { + t.Fatalf("wrong Bytes %x, want %s", rl.Bytes(), test.input) + } + + // check iterator + it := rl.ContentIterator() + i := 0 + for it.Next() { + var item T + if err := DecodeBytes(it.Value(), &item); err != nil { + t.Fatalf("item %d decode error: %v", i, err) + } + if !reflect.DeepEqual(item, items[i]) { + t.Fatalf("iterator has wrong item %v at %d", item, i) + } + i++ + } + if i != test.length { + t.Fatalf("iterator produced %d values, want %d", i, test.length) + } + if it.Err() != nil { + t.Fatalf("iterator error: %v", it.Err()) + } + + // check encoding round trip + output, err := EncodeToBytes(&rl) + if err != nil { + t.Fatal("encode error:", err) + } + if !bytes.Equal(output, unhex(test.input)) { + t.Fatalf("encoding does not round trip: %x", output) + } + + // check EncodeToRawList on items produces same bytes + encRL, err := EncodeToRawList(test.items) + if err != nil { + t.Fatal("EncodeToRawList error:", err) + } + encRLOutput, err := EncodeToBytes(&encRL) + if err != nil { + t.Fatal("EncodeToBytes of encoded list failed:", err) + } + if !bytes.Equal(encRLOutput, output) { + t.Fatalf("wrong encoding of EncodeToRawList result: %x", encRLOutput) + } +} + +func TestRawList(t *testing.T) { + tests := []interface { + name() string + run(t *testing.T) + }{ + rawListTest[uint64]{ + input: "C0", + content: "", + items: []uint64{}, + length: 0, + }, + rawListTest[uint64]{ + input: "C3010203", + content: "010203", + items: []uint64{1, 2, 3}, + length: 3, + }, + rawListTest[simplestruct]{ + input: "C6C20102C20304", + content: "C20102C20304", + items: []simplestruct{{1, "\x02"}, {3, "\x04"}}, + length: 2, + }, + rawListTest[string]{ + input: "F83C836161618362626283636363836464648365656583666666836767678368686883696969836A6A6A836B6B6B836C6C6C836D6D6D836E6E6E836F6F6F", + content: "836161618362626283636363836464648365656583666666836767678368686883696969836A6A6A836B6B6B836C6C6C836D6D6D836E6E6E836F6F6F", + items: []string{"aaa", "bbb", "ccc", "ddd", "eee", "fff", "ggg", "hhh", "iii", "jjj", "kkk", "lll", "mmm", "nnn", "ooo"}, + length: 15, + }, + } + + for _, test := range tests { + t.Run(test.name(), test.run) + } +} + +func TestRawListEmpty(t *testing.T) { + // zero value list + var rl RawList[uint64] + b, _ := EncodeToBytes(&rl) + if !bytes.Equal(b, unhex("C0")) { + t.Fatalf("empty RawList has wrong encoding %x", b) + } + if rl.Len() != 0 { + t.Fatalf("empty list has Len %d", rl.Len()) + } + if rl.Size() != 1 { + t.Fatalf("empty list has Size %d", rl.Size()) + } + if len(rl.Content()) > 0 { + t.Fatalf("empty list has non-empty Content") + } + if !bytes.Equal(rl.Bytes(), []byte{0xC0}) { + t.Fatalf("empty list has wrong encoding") + } + + // nil pointer + var nilptr *RawList[uint64] + b, _ = EncodeToBytes(nilptr) + if !bytes.Equal(b, unhex("C0")) { + t.Fatalf("nil pointer to RawList has wrong encoding %x", b) + } +} + +// This checks that *RawList works in an 'optional' context. +func TestRawListOptional(t *testing.T) { + type foo struct { + L *RawList[uint64] `rlp:"optional"` + } + // nil pointer encoding + var empty foo + b, _ := EncodeToBytes(empty) + if !bytes.Equal(b, unhex("C0")) { + t.Fatalf("nil pointer to RawList has wrong encoding %x", b) + } + // decoding + var dec foo + if err := DecodeBytes(unhex("C0"), &dec); err != nil { + t.Fatal(err) + } + if dec.L != nil { + t.Fatal("rawlist was decoded as non-nil") + } +} + +func TestRawListAppend(t *testing.T) { + var rl RawList[simplestruct] + + v1 := simplestruct{1, "one"} + v2 := simplestruct{2, "two"} + if err := rl.Append(v1); err != nil { + t.Fatal("append 1 failed:", err) + } + if err := rl.Append(v2); err != nil { + t.Fatal("append 2 failed:", err) + } + + if rl.Len() != 2 { + t.Fatalf("wrong Len %d", rl.Len()) + } + if rl.Size() != 13 { + t.Fatalf("wrong Size %d", rl.Size()) + } + if !bytes.Equal(rl.Content(), unhex("C501836F6E65 C5028374776F")) { + t.Fatalf("wrong Content %x", rl.Content()) + } + encoded, _ := EncodeToBytes(&rl) + if !bytes.Equal(encoded, unhex("CC C501836F6E65 C5028374776F")) { + t.Fatalf("wrong encoding %x", encoded) + } +} + +func TestRawListAppendRaw(t *testing.T) { + var rl RawList[uint64] + + if err := rl.AppendRaw(unhex("01")); err != nil { + t.Fatal("AppendRaw(01) failed:", err) + } + if err := rl.AppendRaw(unhex("820102")); err != nil { + t.Fatal("AppendRaw(820102) failed:", err) + } + if rl.Len() != 2 { + t.Fatalf("wrong Len %d after valid appends", rl.Len()) + } + + if err := rl.AppendRaw(nil); err == nil { + t.Fatal("AppendRaw(nil) should fail") + } + if err := rl.AppendRaw(unhex("0102")); err == nil { + t.Fatal("AppendRaw(0102) should fail due to trailing bytes") + } + if err := rl.AppendRaw(unhex("8201")); err == nil { + t.Fatal("AppendRaw(8201) should fail due to truncated value") + } + if rl.Len() != 2 { + t.Fatalf("wrong Len %d after invalid appends, want 2", rl.Len()) + } +} +func TestRawListAppendList(t *testing.T) { + var rl1 RawList[uint64] + if err := rl1.Append(uint64(1)); err != nil { + t.Fatal("append 1 failed:", err) + } + if err := rl1.Append(uint64(2)); err != nil { + t.Fatal("append 2 failed:", err) + } + + var rl2 RawList[uint64] + if err := rl2.Append(uint64(3)); err != nil { + t.Fatal("append 3 failed:", err) + } + if err := rl2.Append(uint64(4)); err != nil { + t.Fatal("append 4 failed:", err) + } + + rl1.AppendList(&rl2) + + if rl1.Len() != 4 { + t.Fatalf("wrong Len %d, want 4", rl1.Len()) + } + if rl1.Size() != 5 { + t.Fatalf("wrong Size %d, want 5", rl1.Size()) + } + + items, err := rl1.Items() + if err != nil { + t.Fatal("Items failed:", err) + } + if !reflect.DeepEqual(items, []uint64{1, 2, 3, 4}) { + t.Fatalf("wrong items: %v", items) + } + + var empty RawList[uint64] + prevLen := rl1.Len() + rl1.AppendList(&empty) + + if rl1.Len() != prevLen { + t.Fatalf("appending empty list changed Len: got %d, want %d", rl1.Len(), prevLen) + } + + empty.AppendList(&rl1) + + if empty.Len() != 4 { + t.Fatalf("wrong Len %d, want 4", empty.Len()) + } +} + +func TestRawListDecodeInvalid(t *testing.T) { + tests := []struct { + input string + err error + }{ + // Single item with non-canonical size (0x81 wrapping byte <= 0x7F). + {input: "C28142", err: ErrCanonSize}, + // Single item claiming more bytes than available in the list. + {input: "C484020202", err: ErrElemTooLarge}, + // Two items, second has non-canonical size. + {input: "C3018142", err: ErrCanonSize}, + // Two items, second claims more bytes than remain in the list. + {input: "C401830202", err: ErrElemTooLarge}, + // Item is a sub-list whose declared size exceeds available bytes. + {input: "C3C40102", err: ErrElemTooLarge}, + } + for _, test := range tests { + var rl RawList[RawValue] + err := DecodeBytes(unhex(test.input), &rl) + if !errors.Is(err, test.err) { + t.Errorf("input %s: error mismatch: got %v, want %v", test.input, err, test.err) + } + } +} + func TestCountValues(t *testing.T) { tests := []struct { input string // note: spaces in input are stripped by unhex @@ -40,9 +336,9 @@ func TestCountValues(t *testing.T) { {"820101 820202 8403030303 04", 4, nil}, // size errors - {"8142", 0, ErrCanonSize}, - {"01 01 8142", 0, ErrCanonSize}, - {"02 84020202", 0, ErrValueTooLarge}, + {"8142", 1, ErrCanonSize}, + {"01 01 8142", 3, ErrCanonSize}, + {"02 84020202", 2, ErrValueTooLarge}, { input: "A12000BF49F440A1CD0527E4D06E2765654C0F56452257516D793A9B8D604DCFDF2AB853F851808D10000000000000000000000000A056E81F171BCC55A6FF8345E692C0F86E5B48E01B996CADC001622FB5E363B421A0C5D2460186F7233C927E7DB2DCC703C0E500B653CA82273B7BFAD8045D85A470", @@ -336,3 +632,269 @@ func TestBytesSize(t *testing.T) { } } } + +func TestSplitListValues(t *testing.T) { + tests := []struct { + name string + input string // hex-encoded RLP list + want []string // hex-encoded expected elements + wantErr error + }{ + { + name: "empty list", + input: "C0", + want: []string{}, + }, + { + name: "single byte element", + input: "C101", + want: []string{"01"}, + }, + { + name: "single empty string", + input: "C180", + want: []string{"80"}, + }, + { + name: "two byte elements", + input: "C20102", + want: []string{"01", "02"}, + }, + { + name: "three elements", + input: "C3010203", + want: []string{"01", "02", "03"}, + }, + { + name: "mixed size elements", + input: "C80182020283030303", + want: []string{"01", "820202", "83030303"}, + }, + { + name: "string elements", + input: "C88363617483646F67", + want: []string{"83636174", "83646F67"}, // cat,dog + }, + { + name: "nested list element", + input: "C4C3010203", // [[1,2,3]] + want: []string{"C3010203"}, // [1,2,3] + }, + { + name: "multiple nested lists", + input: "C6C20102C20304", // [[1,2],[3,4]] + want: []string{"C20102", "C20304"}, // [1,2], [3,4] + }, + { + name: "large list", + input: "C6010203040506", + want: []string{"01", "02", "03", "04", "05", "06"}, + }, + { + name: "list with empty strings", + input: "C3808080", + want: []string{"80", "80", "80"}, + }, + // Error cases + { + name: "single byte", + input: "01", + wantErr: ErrExpectedList, + }, + { + name: "string", + input: "83636174", + wantErr: ErrExpectedList, + }, + { + name: "empty input", + input: "", + wantErr: io.ErrUnexpectedEOF, + }, + { + name: "invalid list - value too large", + input: "C60102030405", + wantErr: ErrValueTooLarge, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := SplitListValues(unhex(tt.input)) + if !errors.Is(err, tt.wantErr) { + t.Errorf("SplitListValues() error = %v, wantErr %v", err, tt.wantErr) + return + } + if err != nil { + return + } + if len(got) != len(tt.want) { + t.Errorf("SplitListValues() got %d elements, want %d", len(got), len(tt.want)) + return + } + for i, elem := range got { + want := unhex(tt.want[i]) + if !bytes.Equal(elem, want) { + t.Errorf("SplitListValues() element[%d] = %x, want %x", i, elem, want) + } + } + }) + } +} + +func TestMergeListValues(t *testing.T) { + tests := []struct { + name string + elems []string // hex-encoded RLP elements + want string // hex-encoded expected result + wantErr error + }{ + { + name: "empty list", + elems: []string{}, + want: "C0", + }, + { + name: "single byte element", + elems: []string{"01"}, + want: "C101", + }, + { + name: "single empty string", + elems: []string{"80"}, + want: "C180", + }, + { + name: "two byte elements", + elems: []string{"01", "02"}, + want: "C20102", + }, + { + name: "three elements", + elems: []string{"01", "02", "03"}, + want: "C3010203", + }, + { + name: "mixed size elements", + elems: []string{"01", "820202", "83030303"}, + want: "C80182020283030303", + }, + { + name: "string elements", + elems: []string{"83636174", "83646F67"}, // cat, dog + want: "C88363617483646F67", + }, + { + name: "nested list element", + elems: []string{"C20102", "03"}, // [[1, 2], 3] + want: "C4C2010203", + }, + { + name: "multiple nested lists", + elems: []string{"C20102", "C3030405"}, // [[1,2],[3,4,5]], + want: "C7C20102C3030405", + }, + { + name: "large list", + elems: []string{"01", "02", "03", "04", "05", "06"}, + want: "C6010203040506", + }, + { + name: "list with empty strings", + elems: []string{"80", "80", "80"}, + want: "C3808080", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + elems := make([][]byte, len(tt.elems)) + for i, s := range tt.elems { + elems[i] = unhex(s) + } + got, err := MergeListValues(elems) + if !errors.Is(err, tt.wantErr) { + t.Errorf("MergeListValues() error = %v, wantErr %v", err, tt.wantErr) + return + } + if err != nil { + return + } + want := unhex(tt.want) + if !bytes.Equal(got, want) { + t.Errorf("MergeListValues() = %x, want %x", got, want) + } + }) + } +} + +func TestSplitMergeList(t *testing.T) { + tests := []struct { + name string + input string // hex-encoded RLP list + }{ + { + name: "empty list", + input: "C0", + }, + { + name: "single byte element", + input: "C101", + }, + { + name: "two byte elements", + input: "C20102", + }, + { + name: "three elements", + input: "C3010203", + }, + { + name: "mixed size elements", + input: "C80182020283030303", + }, + { + name: "string elements", + input: "C88363617483646F67", // [cat, dog] + }, + { + name: "nested list element", + input: "C4C2010203", // [[1,2],3] + }, + { + name: "multiple nested lists", + input: "C6C20102C20304", // [[1,2],[3,4]] + }, + { + name: "large list", + input: "C6010203040506", // [1,2,3,4,5,6] + }, + { + name: "list with empty strings", + input: "C3808080", // ["", "", ""] + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + original := unhex(tt.input) + + // Split the list + elements, err := SplitListValues(original) + if err != nil { + t.Fatalf("SplitListValues() error = %v", err) + } + + // Merge back + merged, err := MergeListValues(elements) + if err != nil { + t.Fatalf("MergeListValues() error = %v", err) + } + + // The merged result should match the original + if !bytes.Equal(merged, original) { + t.Errorf("Round trip failed: original = %x, merged = %x", original, merged) + } + }) + } +} diff --git a/rpc/client.go b/rpc/client.go index b0ed124797..fb1ab006fa 100644 --- a/rpc/client.go +++ b/rpc/client.go @@ -126,7 +126,7 @@ func (c *Client) newClientConn(conn ServerCodec) *clientConn { ctx := context.Background() ctx = context.WithValue(ctx, clientContextKey{}, c) ctx = context.WithValue(ctx, peerInfoContextKey{}, conn.peerInfo()) - handler := newHandler(ctx, conn, c.idgen, c.services, c.batchItemLimit, c.batchResponseMaxSize) + handler := newHandler(ctx, conn, c.idgen, c.services, c.batchItemLimit, c.batchResponseMaxSize, nil) return &clientConn{conn, handler} } diff --git a/rpc/handler.go b/rpc/handler.go index 06cc9c28c4..b8342f7ed6 100644 --- a/rpc/handler.go +++ b/rpc/handler.go @@ -28,7 +28,10 @@ import ( "sync" "time" + "github.com/ethereum/go-ethereum/internal/telemetry" "github.com/ethereum/go-ethereum/log" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" ) // handler handles JSON-RPC messages. There is one handler per connection. Note that @@ -65,6 +68,7 @@ type handler struct { allowSubscribe bool batchRequestLimit int batchResponseMaxSize int + tracerProvider trace.TracerProvider subLock sync.Mutex serverSubs map[ID]*Subscription @@ -73,9 +77,10 @@ type handler struct { type callProc struct { ctx context.Context notifiers []*Notifier + isBatch bool } -func newHandler(connCtx context.Context, conn jsonWriter, idgen func() ID, reg *serviceRegistry, batchRequestLimit, batchResponseMaxSize int) *handler { +func newHandler(connCtx context.Context, conn jsonWriter, idgen func() ID, reg *serviceRegistry, batchRequestLimit, batchResponseMaxSize int, tracerProvider trace.TracerProvider) *handler { rootCtx, cancelRoot := context.WithCancel(connCtx) h := &handler{ reg: reg, @@ -90,6 +95,7 @@ func newHandler(connCtx context.Context, conn jsonWriter, idgen func() ID, reg * log: log.Root(), batchRequestLimit: batchRequestLimit, batchResponseMaxSize: batchResponseMaxSize, + tracerProvider: tracerProvider, } if conn.remoteAddr() != "" { h.log = h.log.New("conn", conn.remoteAddr()) @@ -197,6 +203,7 @@ func (h *handler) handleBatch(msgs []*jsonrpcMessage) { // Process calls on a goroutine because they may block indefinitely: h.startCallProc(func(cp *callProc) { + cp.isBatch = true var ( timer *time.Timer cancel context.CancelFunc @@ -497,40 +504,65 @@ func (h *handler) handleCall(cp *callProc, msg *jsonrpcMessage) *jsonrpcMessage if msg.isSubscribe() { return h.handleSubscribe(cp, msg) } - var callb *callback if msg.isUnsubscribe() { - callb = h.unsubscribeCb - } else { - // Check method name length - if len(msg.Method) > maxMethodNameLength { - return msg.errorResponse(&invalidRequestError{fmt.Sprintf("method name too long: %d > %d", len(msg.Method), maxMethodNameLength)}) + args, err := parsePositionalArguments(msg.Params, h.unsubscribeCb.argTypes) + if err != nil { + return msg.errorResponse(&invalidParamsError{err.Error()}) } - callb = h.reg.callback(msg.Method) + return h.runMethod(cp.ctx, msg, h.unsubscribeCb, args) + } + + // Check method name length + if len(msg.Method) > maxMethodNameLength { + return msg.errorResponse(&invalidRequestError{fmt.Sprintf("method name too long: %d > %d", len(msg.Method), maxMethodNameLength)}) } + callb, service, method := h.reg.callback(msg.Method) + + // If the method is not found, return an error. if callb == nil { return msg.errorResponse(&methodNotFoundError{method: msg.Method}) } - args, err := parsePositionalArguments(msg.Params, callb.argTypes) - if err != nil { - return msg.errorResponse(&invalidParamsError{err.Error()}) + // Start root span for the request. + rpcInfo := telemetry.RPCInfo{ + System: "jsonrpc", + Service: service, + Method: method, + RequestID: string(msg.ID), + } + attrib := []telemetry.Attribute{ + telemetry.BoolAttribute("rpc.batch", cp.isBatch), + } + ctx, spanEnd := telemetry.StartServerSpan(cp.ctx, h.tracer(), rpcInfo, attrib...) + defer spanEnd(nil) // don't propagate errors to parent spans + + // Start tracing span before parsing arguments. + _, _, pSpanEnd := telemetry.StartSpanWithTracer(ctx, h.tracer(), "rpc.parsePositionalArguments") + args, pErr := parsePositionalArguments(msg.Params, callb.argTypes) + pSpanEnd(&pErr) + if pErr != nil { + return msg.errorResponse(&invalidParamsError{pErr.Error()}) } start := time.Now() - answer := h.runMethod(cp.ctx, msg, callb, args) - // Collect the statistics for RPC calls if metrics is enabled. - // We only care about pure rpc call. Filter out subscription. - if callb != h.unsubscribeCb { - rpcRequestGauge.Inc(1) - if answer.Error != nil { - failedRequestGauge.Inc(1) - } else { - successfulRequestGauge.Inc(1) - } - rpcServingTimer.UpdateSince(start) - updateServeTimeHistogram(msg.Method, answer.Error == nil, time.Since(start)) + // Start tracing span before running the method. + rctx, _, rSpanEnd := telemetry.StartSpanWithTracer(ctx, h.tracer(), "rpc.runMethod") + answer := h.runMethod(rctx, msg, callb, args) + var rErr error + if answer.Error != nil { + rErr = errors.New(answer.Error.Message) } + rSpanEnd(&rErr) + // Collect the statistics for RPC calls if metrics is enabled. + rpcRequestGauge.Inc(1) + if answer.Error != nil { + failedRequestGauge.Inc(1) + } else { + successfulRequestGauge.Inc(1) + } + rpcServingTimer.UpdateSince(start) + updateServeTimeHistogram(msg.Method, answer.Error == nil, time.Since(start)) return answer } @@ -568,17 +600,33 @@ func (h *handler) handleSubscribe(cp *callProc, msg *jsonrpcMessage) *jsonrpcMes n := &Notifier{h: h, namespace: namespace} cp.notifiers = append(cp.notifiers, n) ctx := context.WithValue(cp.ctx, notifierKey{}, n) - return h.runMethod(ctx, msg, callb, args) } +// tracer returns the OpenTelemetry Tracer for RPC call tracing. +func (h *handler) tracer() trace.Tracer { + if h.tracerProvider == nil { + // Default to global TracerProvider if none is set. + // Note: If no TracerProvider is set, the default is a no-op TracerProvider. + // See https://pkg.go.dev/go.opentelemetry.io/otel#GetTracerProvider + return otel.Tracer("") + } + return h.tracerProvider.Tracer("") +} + // runMethod runs the Go callback for an RPC method. -func (h *handler) runMethod(ctx context.Context, msg *jsonrpcMessage, callb *callback, args []reflect.Value) *jsonrpcMessage { +func (h *handler) runMethod(ctx context.Context, msg *jsonrpcMessage, callb *callback, args []reflect.Value, attributes ...telemetry.Attribute) *jsonrpcMessage { result, err := callb.call(ctx, msg.Method, args) if err != nil { return msg.errorResponse(err) } - return msg.response(result) + _, _, spanEnd := telemetry.StartSpanWithTracer(ctx, h.tracer(), "rpc.encodeJSONResponse", attributes...) + response := msg.response(result) + if response.Error != nil { + err = errors.New(response.Error.Message) + } + spanEnd(&err) + return response } // unsubscribe is the callback function for all *_unsubscribe calls. @@ -612,8 +660,11 @@ type limitedBuffer struct { } func (buf *limitedBuffer) Write(data []byte) (int, error) { - avail := max(buf.limit, len(buf.output)) - if len(data) < avail { + avail := buf.limit - len(buf.output) + if avail <= 0 { + return 0, errTruncatedOutput + } + if len(data) <= avail { buf.output = append(buf.output, data...) return len(data), nil } diff --git a/rpc/http.go b/rpc/http.go index f4b99429ef..55f0abfa72 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -30,6 +30,9 @@ import ( "strconv" "sync" "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" ) const ( @@ -168,13 +171,21 @@ func newClientTransportHTTP(endpoint string, cfg *clientConfig) reconnectFunc { } } +// cleanlyCloseBody avoids sending unnecessary RST_STREAM and PING frames by +// ensuring the whole body is read before being closed. +// See https://blog.cloudflare.com/go-and-enhance-your-calm/#reading-bodies-in-go-can-be-unintuitive +func cleanlyCloseBody(body io.ReadCloser) error { + io.Copy(io.Discard, body) + return body.Close() +} + func (c *Client) sendHTTP(ctx context.Context, op *requestOp, msg interface{}) error { hc := c.writeConn.(*httpConn) respBody, err := hc.doRequest(ctx, msg) if err != nil { return err } - defer respBody.Close() + defer cleanlyCloseBody(respBody) var resp jsonrpcMessage batch := [1]*jsonrpcMessage{&resp} @@ -191,7 +202,7 @@ func (c *Client) sendBatchHTTP(ctx context.Context, op *requestOp, msgs []*jsonr if err != nil { return err } - defer respBody.Close() + defer cleanlyCloseBody(respBody) var respmsgs []*jsonrpcMessage if err := json.NewDecoder(respBody).Decode(&respmsgs); err != nil { @@ -236,7 +247,7 @@ func (hc *httpConn) doRequest(ctx context.Context, msg interface{}) (io.ReadClos if _, err := buf.ReadFrom(resp.Body); err == nil { body = buf.Bytes() } - resp.Body.Close() + cleanlyCloseBody(resp.Body) return nil, HTTPError{ Status: resp.Status, StatusCode: resp.StatusCode, @@ -326,6 +337,9 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { ctx := r.Context() ctx = context.WithValue(ctx, peerInfoContextKey{}, connInfo) + // Extract trace context from incoming headers. + ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.HeaderCarrier(r.Header)) + // All checks passed, create a codec that reads directly from the request body // until EOF, writes the response to w, and orders the server to process a // single request. diff --git a/rpc/http_test.go b/rpc/http_test.go index 6c268b6292..15ddd59bd0 100644 --- a/rpc/http_test.go +++ b/rpc/http_test.go @@ -106,7 +106,7 @@ func confirmHTTPRequestYieldsStatusCode(t *testing.T, method, contentType, body if err != nil { t.Fatalf("request failed: %v", err) } - resp.Body.Close() + cleanlyCloseBody(resp.Body) confirmStatusCode(t, resp.StatusCode, expectedStatusCode) } diff --git a/rpc/server.go b/rpc/server.go index 5c9f54955f..a0a718df44 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -25,6 +25,7 @@ import ( "sync/atomic" "github.com/ethereum/go-ethereum/log" + "go.opentelemetry.io/otel/trace" ) const MetadataApi = "rpc" @@ -55,15 +56,17 @@ type Server struct { batchResponseLimit int httpBodyLimit int wsReadLimit int64 + tracerProvider trace.TracerProvider } // NewServer creates a new server instance with no registered handlers. func NewServer() *Server { server := &Server{ - idgen: randomIDGenerator(), - codecs: make(map[ServerCodec]struct{}), - httpBodyLimit: defaultBodyLimit, - wsReadLimit: wsDefaultReadLimit, + idgen: randomIDGenerator(), + codecs: make(map[ServerCodec]struct{}), + httpBodyLimit: defaultBodyLimit, + wsReadLimit: wsDefaultReadLimit, + tracerProvider: nil, } server.run.Store(true) // Register the default service providing meta information about the RPC service such @@ -133,6 +136,15 @@ func (s *Server) ServeCodec(codec ServerCodec, options CodecOption) { c.Close() } +// setTracerProvider configures the OpenTelemetry TracerProvider for RPC call tracing. +// Note: This method (and the TracerProvider field in the Server/Handler struct) is +// primarily intended for testing. In particular, it allows tests to configure an +// isolated TracerProvider without changing the global provider, avoiding +// interference between tests running in parallel. +func (s *Server) setTracerProvider(tp trace.TracerProvider) { + s.tracerProvider = tp +} + func (s *Server) trackCodec(codec ServerCodec) bool { s.mutex.Lock() defer s.mutex.Unlock() @@ -160,7 +172,7 @@ func (s *Server) serveSingleRequest(ctx context.Context, codec ServerCodec) { return } - h := newHandler(ctx, codec, s.idgen, &s.services, s.batchItemLimit, s.batchResponseLimit) + h := newHandler(ctx, codec, s.idgen, &s.services, s.batchItemLimit, s.batchResponseLimit, s.tracerProvider) h.allowSubscribe = false defer h.close(io.EOF, nil) diff --git a/rpc/service.go b/rpc/service.go index 7cd7ca4297..c3e3bd1d05 100644 --- a/rpc/service.go +++ b/rpc/service.go @@ -100,14 +100,14 @@ func (r *serviceRegistry) registerName(name string, rcvr interface{}) error { } // callback returns the callback corresponding to the given RPC method name. -func (r *serviceRegistry) callback(method string) *callback { +func (r *serviceRegistry) callback(method string) (cb *callback, service, methodName string) { before, after, found := strings.Cut(method, serviceMethodSeparator) if !found { - return nil + return nil, "", "" } r.mu.Lock() defer r.mu.Unlock() - return r.services[before].callbacks[after] + return r.services[before].callbacks[after], before, after } // subscription returns a subscription callback in the given service. diff --git a/rpc/tracing_test.go b/rpc/tracing_test.go new file mode 100644 index 0000000000..5a04c901fd --- /dev/null +++ b/rpc/tracing_test.go @@ -0,0 +1,268 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rpc + +import ( + "context" + "net/http/httptest" + "testing" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/propagation" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" +) + +// attributeMap converts a slice of attributes to a map. +func attributeMap(attrs []attribute.KeyValue) map[string]string { + m := make(map[string]string) + for _, a := range attrs { + switch a.Value.Type() { + case attribute.STRING: + m[string(a.Key)] = a.Value.AsString() + case attribute.BOOL: + if a.Value.AsBool() { + m[string(a.Key)] = "true" + } else { + m[string(a.Key)] = "false" + } + default: + m[string(a.Key)] = a.Value.Emit() + } + } + return m +} + +// newTracingServer creates a new server with tracing enabled. +func newTracingServer(t *testing.T) (*Server, *sdktrace.TracerProvider, *tracetest.InMemoryExporter) { + t.Helper() + exporter := tracetest.NewInMemoryExporter() + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exporter)) + t.Cleanup(func() { _ = tp.Shutdown(context.Background()) }) + server := newTestServer() + server.setTracerProvider(tp) + t.Cleanup(server.Stop) + return server, tp, exporter +} + +// TestTracingHTTP verifies that RPC spans are emitted when processing HTTP requests. +func TestTracingHTTP(t *testing.T) { + // Not parallel: this test modifies the global otel TextMapPropagator. + + // Set up a propagator to extract W3C Trace Context headers. + originalPropagator := otel.GetTextMapPropagator() + otel.SetTextMapPropagator(propagation.TraceContext{}) + t.Cleanup(func() { otel.SetTextMapPropagator(originalPropagator) }) + + server, tracer, exporter := newTracingServer(t) + httpsrv := httptest.NewServer(server) + t.Cleanup(httpsrv.Close) + + // Define the expected trace and span IDs for context propagation. + const ( + traceID = "4bf92f3577b34da6a3ce929d0e0e4736" + parentSpanID = "00f067aa0ba902b7" + traceparent = "00-" + traceID + "-" + parentSpanID + "-01" + ) + + client, err := DialHTTP(httpsrv.URL) + if err != nil { + t.Fatalf("failed to dial: %v", err) + } + t.Cleanup(client.Close) + + // Set trace context headers. + client.SetHeader("traceparent", traceparent) + + // Make a successful RPC call. + var result echoResult + if err := client.Call(&result, "test_echo", "hello", 42, &echoArgs{S: "world"}); err != nil { + t.Fatalf("RPC call failed: %v", err) + } + + // Flush and verify that we emitted the expected span. + if err := tracer.ForceFlush(context.Background()); err != nil { + t.Fatalf("failed to flush: %v", err) + } + spans := exporter.GetSpans() + if len(spans) == 0 { + t.Fatal("no spans were emitted") + } + var rpcSpan *tracetest.SpanStub + for i := range spans { + if spans[i].Name == "jsonrpc.test/echo" { + rpcSpan = &spans[i] + break + } + } + if rpcSpan == nil { + t.Fatalf("jsonrpc.test/echo span not found") + } + + // Verify span attributes. + attrs := attributeMap(rpcSpan.Attributes) + if attrs["rpc.system"] != "jsonrpc" { + t.Errorf("expected rpc.system=jsonrpc, got %v", attrs["rpc.system"]) + } + if attrs["rpc.service"] != "test" { + t.Errorf("expected rpc.service=test, got %v", attrs["rpc.service"]) + } + if attrs["rpc.method"] != "echo" { + t.Errorf("expected rpc.method=echo, got %v", attrs["rpc.method"]) + } + if _, ok := attrs["rpc.jsonrpc.request_id"]; !ok { + t.Errorf("expected rpc.jsonrpc.request_id attribute to be set") + } + + // Verify the span's parent matches the traceparent header values. + if got := rpcSpan.Parent.TraceID().String(); got != traceID { + t.Errorf("parent trace ID mismatch: got %s, want %s", got, traceID) + } + if got := rpcSpan.Parent.SpanID().String(); got != parentSpanID { + t.Errorf("parent span ID mismatch: got %s, want %s", got, parentSpanID) + } + if !rpcSpan.Parent.IsRemote() { + t.Error("expected parent span context to be marked as remote") + } +} + +// TestTracingErrorRecording verifies that errors are recorded on spans. +func TestTracingHTTPErrorRecording(t *testing.T) { + t.Parallel() + server, tracer, exporter := newTracingServer(t) + httpsrv := httptest.NewServer(server) + t.Cleanup(httpsrv.Close) + client, err := DialHTTP(httpsrv.URL) + if err != nil { + t.Fatalf("failed to dial: %v", err) + } + t.Cleanup(client.Close) + + // Call a method that returns an error. + var result any + err = client.Call(&result, "test_returnError") + if err == nil { + t.Fatal("expected error from test_returnError") + } + + // Flush and verify spans recorded the error. + if err := tracer.ForceFlush(context.Background()); err != nil { + t.Fatalf("failed to flush: %v", err) + } + spans := exporter.GetSpans() + + // Only the runMethod span should have error status. + if len(spans) == 0 { + t.Fatal("no spans were emitted") + } + for _, span := range spans { + switch span.Name { + case "rpc.runMethod": + if span.Status.Code != codes.Error { + t.Errorf("expected %s span status Error, got %v", span.Name, span.Status.Code) + } + default: + if span.Status.Code == codes.Error { + t.Errorf("unexpected error status on span %s", span.Name) + } + } + } +} + +// TestTracingBatchHTTP verifies that RPC spans are emitted for batched JSON-RPC calls over HTTP. +func TestTracingBatchHTTP(t *testing.T) { + t.Parallel() + server, tracer, exporter := newTracingServer(t) + httpsrv := httptest.NewServer(server) + t.Cleanup(httpsrv.Close) + client, err := DialHTTP(httpsrv.URL) + if err != nil { + t.Fatalf("failed to dial: %v", err) + } + t.Cleanup(client.Close) + + // Make a successful batch RPC call. + batch := []BatchElem{ + { + Method: "test_echo", + Args: []any{"hello", 42, &echoArgs{S: "world"}}, + Result: new(echoResult), + }, + { + Method: "test_echo", + Args: []any{"your", 7, &echoArgs{S: "mom"}}, + Result: new(echoResult), + }, + } + if err := client.BatchCall(batch); err != nil { + t.Fatalf("batch RPC call failed: %v", err) + } + + // Flush and verify we emitted spans for each batch element. + if err := tracer.ForceFlush(context.Background()); err != nil { + t.Fatalf("failed to flush: %v", err) + } + spans := exporter.GetSpans() + if len(spans) == 0 { + t.Fatal("no spans were emitted") + } + var found int + for i := range spans { + if spans[i].Name == "jsonrpc.test/echo" { + attrs := attributeMap(spans[i].Attributes) + if attrs["rpc.system"] == "jsonrpc" && + attrs["rpc.service"] == "test" && + attrs["rpc.method"] == "echo" && + attrs["rpc.batch"] == "true" { + found++ + } + } + } + if found != len(batch) { + t.Fatalf("expected %d matching batch spans, got %d", len(batch), found) + } +} + +// TestTracingSubscribeUnsubscribe verifies that subscribe and unsubscribe calls +// do not emit any spans. +// Note: This works because client.newClientConn() passes nil as the tracer provider. +func TestTracingSubscribeUnsubscribe(t *testing.T) { + t.Parallel() + server, tracer, exporter := newTracingServer(t) + client := DialInProc(server) + t.Cleanup(client.Close) + + // Subscribe to notifications. + sub, err := client.Subscribe(context.Background(), "nftest", make(chan int), "someSubscription", 1, 1) + if err != nil { + t.Fatalf("subscribe failed: %v", err) + } + + // Unsubscribe. + sub.Unsubscribe() + + // Flush and check that no spans were emitted. + if err := tracer.ForceFlush(context.Background()); err != nil { + t.Fatalf("failed to flush: %v", err) + } + spans := exporter.GetSpans() + if len(spans) != 0 { + t.Errorf("expected no spans for subscribe/unsubscribe, got %d", len(spans)) + } +} diff --git a/signer/core/apitypes/types.go b/signer/core/apitypes/types.go index 19323246a7..401f4fba07 100644 --- a/signer/core/apitypes/types.go +++ b/signer/core/apitypes/types.go @@ -242,8 +242,8 @@ func (args *SendTxArgs) validateTxSidecar() error { if args.Proofs != nil { if len(args.Proofs) == n { // v1 transaction - for i, b := range args.Blobs { - if err := kzg4844.VerifyBlobProof(&b, args.Commitments[i], args.Proofs[i]); err != nil { + for i := range args.Blobs { + if err := kzg4844.VerifyBlobProof(&args.Blobs[i], args.Commitments[i], args.Proofs[i]); err != nil { return fmt.Errorf("failed to verify blob proof: %v", err) } } @@ -259,25 +259,27 @@ func (args *SendTxArgs) validateTxSidecar() error { if args.Commitments == nil { // Generate commitment and proof. commitments := make([]kzg4844.Commitment, n) - proofs := make([]kzg4844.Proof, 0, n) - for i, b := range args.Blobs { - c, err := kzg4844.BlobToCommitment(&b) + for i := range args.Blobs { + c, err := kzg4844.BlobToCommitment(&args.Blobs[i]) if err != nil { return fmt.Errorf("blobs[%d]: error computing commitment: %v", i, err) } commitments[i] = c } - if args.BlobVersion == 1 { - for i, b := range args.Blobs { - p, err := kzg4844.ComputeCellProofs(&b) + var proofs []kzg4844.Proof + if args.BlobVersion == types.BlobSidecarVersion1 { + proofs = make([]kzg4844.Proof, 0, n*kzg4844.CellProofsPerBlob) + for i := range args.Blobs { + p, err := kzg4844.ComputeCellProofs(&args.Blobs[i]) if err != nil { return fmt.Errorf("blobs[%d]: error computing cell proof: %v", i, err) } proofs = append(proofs, p...) } } else { - for i, b := range args.Blobs { - p, err := kzg4844.ComputeBlobProof(&b, commitments[i]) + proofs = make([]kzg4844.Proof, 0, n) + for i := range args.Blobs { + p, err := kzg4844.ComputeBlobProof(&args.Blobs[i], commitments[i]) if err != nil { return fmt.Errorf("blobs[%d]: error computing proof: %v", i, err) } diff --git a/signer/core/apitypes/types_test.go b/signer/core/apitypes/types_test.go index 3e8f3bd43b..f010cef207 100644 --- a/signer/core/apitypes/types_test.go +++ b/signer/core/apitypes/types_test.go @@ -20,7 +20,6 @@ import ( "crypto/rand" "crypto/sha256" "encoding/json" - "fmt" "testing" "github.com/ethereum/go-ethereum/common" @@ -308,7 +307,7 @@ func TestValidateTxSidecar(t *testing.T) { name: "blobs with v1 version flag - should generate cell proofs", args: SendTxArgs{ Blobs: []kzg4844.Blob{blob1}, - BlobVersion: 1, + BlobVersion: types.BlobSidecarVersion1, }, wantErr: false, }, @@ -366,6 +365,17 @@ func TestValidateTxSidecar(t *testing.T) { }, wantErr: true, }, + { + name: "invalid proof", + args: SendTxArgs{ + BlobVersion: types.BlobSidecarVersion1, + Blobs: []kzg4844.Blob{blob1}, + Commitments: []kzg4844.Commitment{commitment1}, + Proofs: []kzg4844.Proof{proof1, proof2}, // wrong proof + BlobHashes: []common.Hash{hash1}, + }, + wantErr: true, + }, } for _, tt := range tests { @@ -390,7 +400,6 @@ func TestValidateTxSidecar(t *testing.T) { t.Errorf("validateTxSidecar() should have generated commitments") } if args.Proofs == nil || (len(args.Proofs) != len(args.Blobs) && len(args.Proofs) != len(args.Blobs)*kzg4844.CellProofsPerBlob) { - fmt.Println("proofs", args.Proofs) t.Errorf("validateTxSidecar() should have generated proofs") } if args.BlobHashes == nil || len(args.BlobHashes) != len(args.Blobs) { diff --git a/signer/core/uiapi.go b/signer/core/uiapi.go index 2f511c7e19..09ee4b492f 100644 --- a/signer/core/uiapi.go +++ b/signer/core/uiapi.go @@ -73,8 +73,9 @@ type rawWallet struct { // Example call // {"jsonrpc":"2.0","method":"clef_listWallets","params":[], "id":5} func (api *UIServerAPI) ListWallets() []rawWallet { - wallets := make([]rawWallet, 0) // return [] instead of nil if empty - for _, wallet := range api.am.Wallets() { + allWallets := api.am.Wallets() + wallets := make([]rawWallet, 0, len(allWallets)) // return [] instead of nil if empty + for _, wallet := range allWallets { status, failure := wallet.Status() raw := rawWallet{ @@ -130,8 +131,12 @@ func (api *UIServerAPI) ImportRawKey(privkey string, password string) (accounts. if err := ValidatePasswordFormat(password); err != nil { return accounts.Account{}, fmt.Errorf("password requirements not met: %v", err) } + ks := fetchKeystore(api.am) + if ks == nil { + return accounts.Account{}, errors.New("password based accounts not supported") + } // No error - return fetchKeystore(api.am).ImportECDSA(key, password) + return ks.ImportECDSA(key, password) } // OpenWallet initiates a hardware wallet opening procedure, establishing a USB diff --git a/tests/block_test_util.go b/tests/block_test_util.go index 1a70bcce46..1841383e9d 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -97,6 +97,7 @@ type btHeader struct { BlobGasUsed *uint64 ExcessBlobGas *uint64 ParentBeaconBlockRoot *common.Hash + SlotNumber *uint64 } type btHeaderMarshaling struct { @@ -109,6 +110,7 @@ type btHeaderMarshaling struct { BaseFeePerGas *math.HexOrDecimal256 BlobGasUsed *math.HexOrDecimal64 ExcessBlobGas *math.HexOrDecimal64 + SlotNumber *math.HexOrDecimal64 } func (t *BlockTest) Run(snapshotter bool, scheme string, witness bool, tracer *tracing.Hooks, postCheck func(error, *core.BlockChain)) (result error) { @@ -116,27 +118,29 @@ func (t *BlockTest) Run(snapshotter bool, scheme string, witness bool, tracer *t if !ok { return UnsupportedForkError{t.json.Network} } + // import pre accounts & construct test genesis block & state root + // Commit genesis state var ( + gspec = t.genesis(config) db = rawdb.NewMemoryDatabase() tconf = &triedb.Config{ Preimages: true, + IsVerkle: gspec.Config.VerkleTime != nil && *gspec.Config.VerkleTime <= gspec.Timestamp, } ) - if scheme == rawdb.PathScheme { + if scheme == rawdb.PathScheme || tconf.IsVerkle { tconf.PathDB = pathdb.Defaults } else { tconf.HashDB = hashdb.Defaults } - // Commit genesis state - gspec := t.genesis(config) // if ttd is not specified, set an arbitrary huge value if gspec.Config.TerminalTotalDifficulty == nil { gspec.Config.TerminalTotalDifficulty = big.NewInt(stdmath.MaxInt64) } triedb := triedb.NewDatabase(db, tconf) - gblock, err := gspec.Commit(db, triedb) + gblock, err := gspec.Commit(db, triedb, nil) if err != nil { return err } @@ -157,9 +161,9 @@ func (t *BlockTest) Run(snapshotter bool, scheme string, witness bool, tracer *t Preimages: true, TriesInMemory: 128, VmConfig: vm.Config{ - Tracer: tracer, - StatelessSelfValidation: witness, + Tracer: tracer, }, + StatelessSelfValidation: witness, } if snapshotter { options.SnapshotLimit = 1 @@ -202,6 +206,11 @@ func (t *BlockTest) Run(snapshotter bool, scheme string, witness bool, tracer *t return t.validateImportedHeaders(chain, validBlocks) } +// Network returns the network/fork name for this test. +func (t *BlockTest) Network() string { + return t.json.Network +} + func (t *BlockTest) genesis(config *params.ChainConfig) *core.Genesis { return &core.Genesis{ Config: config, @@ -259,7 +268,7 @@ func (t *BlockTest) insertBlocks(blockchain *core.BlockChain) ([]btBlock, error) } if b.BlockHeader == nil { if data, err := json.MarshalIndent(cb.Header(), "", " "); err == nil { - fmt.Fprintf(os.Stderr, "block (index %d) insertion should have failed due to: %v:\n%v\n", + fmt.Fprintf(os.Stdout, "block (index %d) insertion should have failed due to: %v:\n%v\n", bi, b.ExpectException, string(data)) } return nil, fmt.Errorf("block (index %d) insertion should have failed due to: %v", @@ -336,6 +345,9 @@ func validateHeader(h *btHeader, h2 *types.Header) error { if !reflect.DeepEqual(h.ParentBeaconBlockRoot, h2.ParentBeaconRoot) { return fmt.Errorf("parentBeaconBlockRoot: want: %v have: %v", h.ParentBeaconBlockRoot, h2.ParentBeaconRoot) } + if !reflect.DeepEqual(h.SlotNumber, h2.SlotNumber) { + return fmt.Errorf("slotNumber: want: %v have: %v", h.SlotNumber, h2.SlotNumber) + } return nil } diff --git a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go index 4d94d31c0c..c60c9cb6e6 100644 --- a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go +++ b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go @@ -32,7 +32,6 @@ import ( type kv struct { k, v []byte - t bool } type fuzzer struct { @@ -62,8 +61,8 @@ func (f *fuzzer) randomTrie(n int) (*trie.Trie, map[string]*kv) { size := f.readInt() // Fill it with some fluff for i := byte(0); i < byte(size); i++ { - value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} - value2 := &kv{common.LeftPadBytes([]byte{i + 10}, 32), []byte{i}, false} + value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}} + value2 := &kv{common.LeftPadBytes([]byte{i + 10}, 32), []byte{i}} trie.MustUpdate(value.k, value.v) trie.MustUpdate(value2.k, value2.v) vals[string(value.k)] = value @@ -76,7 +75,7 @@ func (f *fuzzer) randomTrie(n int) (*trie.Trie, map[string]*kv) { for i := 0; i < n; i++ { k := f.randBytes(32) v := f.randBytes(20) - value := &kv{k, v, false} + value := &kv{k, v} trie.MustUpdate(k, v) vals[string(k)] = value if f.exhausted { diff --git a/tests/fuzzers/txfetcher/txfetcher_fuzzer.go b/tests/fuzzers/txfetcher/txfetcher_fuzzer.go index c136253a62..bcceaff383 100644 --- a/tests/fuzzers/txfetcher/txfetcher_fuzzer.go +++ b/tests/fuzzers/txfetcher/txfetcher_fuzzer.go @@ -78,7 +78,8 @@ func fuzz(input []byte) int { rand := rand.New(rand.NewSource(0x3a29)) // Same used in package tests!!! f := fetcher.NewTxFetcherForTests( - func(common.Hash) bool { return false }, + nil, + func(common.Hash, byte) error { return nil }, func(txs []*types.Transaction) []error { return make([]error, len(txs)) }, diff --git a/tests/gen_btheader.go b/tests/gen_btheader.go index 80ad89e03b..eb6d9a8271 100644 --- a/tests/gen_btheader.go +++ b/tests/gen_btheader.go @@ -38,6 +38,7 @@ func (b btHeader) MarshalJSON() ([]byte, error) { BlobGasUsed *math.HexOrDecimal64 ExcessBlobGas *math.HexOrDecimal64 ParentBeaconBlockRoot *common.Hash + SlotNumber *math.HexOrDecimal64 } var enc btHeader enc.Bloom = b.Bloom @@ -61,6 +62,7 @@ func (b btHeader) MarshalJSON() ([]byte, error) { enc.BlobGasUsed = (*math.HexOrDecimal64)(b.BlobGasUsed) enc.ExcessBlobGas = (*math.HexOrDecimal64)(b.ExcessBlobGas) enc.ParentBeaconBlockRoot = b.ParentBeaconBlockRoot + enc.SlotNumber = (*math.HexOrDecimal64)(b.SlotNumber) return json.Marshal(&enc) } @@ -88,6 +90,7 @@ func (b *btHeader) UnmarshalJSON(input []byte) error { BlobGasUsed *math.HexOrDecimal64 ExcessBlobGas *math.HexOrDecimal64 ParentBeaconBlockRoot *common.Hash + SlotNumber *math.HexOrDecimal64 } var dec btHeader if err := json.Unmarshal(input, &dec); err != nil { @@ -156,5 +159,8 @@ func (b *btHeader) UnmarshalJSON(input []byte) error { if dec.ParentBeaconBlockRoot != nil { b.ParentBeaconBlockRoot = dec.ParentBeaconBlockRoot } + if dec.SlotNumber != nil { + b.SlotNumber = (*uint64)(dec.SlotNumber) + } return nil } diff --git a/tests/init.go b/tests/init.go index 705e929ae9..d10b47986c 100644 --- a/tests/init.go +++ b/tests/init.go @@ -720,6 +720,25 @@ var Forks = map[string]*params.ChainConfig{ BPO4: params.DefaultBPO4BlobConfig, }, }, + "Verkle": { + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), + ArrowGlacierBlock: big.NewInt(0), + MergeNetsplitBlock: big.NewInt(0), + TerminalTotalDifficulty: big.NewInt(0), + ShanghaiTime: u64(0), + VerkleTime: u64(0), + }, } var bpo1BlobConfig = ¶ms.BlobConfig{ diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 4f881fcd41..f19e41df2d 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -239,6 +239,20 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bo if err != nil { // Here, an error exists but it was expected. // We do not check the post state or logs. + // However, if the test defines a post state root, we should check it. + // In case of an error, the state is reverted to the snapshot, so we need to + // recalculate the root. + post := t.json.Post[subtest.Fork][subtest.Index] + if post.Root != (common.UnprefixedHash{}) { + config, _, err := GetChainConfig(subtest.Fork) + if err != nil { + return fmt.Errorf("failed to get chain config: %w", err) + } + root = st.StateDB.IntermediateRoot(config.IsEIP158(new(big.Int).SetUint64(t.json.Env.Number))) + if root != common.Hash(post.Root) { + return fmt.Errorf("post-state root does not match the pre-state root, indicates an error in the test: got %x, want %x", root, post.Root) + } + } return nil } post := t.json.Post[subtest.Fork][subtest.Index] @@ -334,9 +348,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh } // Execute the message. snapshot := st.StateDB.Snapshot() - gaspool := new(core.GasPool) - gaspool.AddGas(block.GasLimit()) - vmRet, err := core.ApplyMessage(evm, msg, gaspool) + vmRet, err := core.ApplyMessage(evm, msg, core.NewGasPool(block.GasLimit())) if err != nil { st.StateDB.RevertToSnapshot(snapshot) if tracer := evm.Config.Tracer; tracer != nil && tracer.OnTxEnd != nil { @@ -538,7 +550,7 @@ func MakePreState(db ethdb.Database, accounts types.GenesisAlloc, snapshotter bo } snaps, _ = snapshot.New(snapconfig, db, triedb, root) } - sdb = state.NewDatabase(triedb, snaps) + sdb = state.NewDatabase(triedb, nil).WithSnapshot(snaps) statedb, _ = state.New(root, sdb) return StateTestState{statedb, triedb, snaps} } diff --git a/trie/bintrie/binary_node.go b/trie/bintrie/binary_node.go index 1c003a6c8f..a7392ec958 100644 --- a/trie/bintrie/binary_node.go +++ b/trie/bintrie/binary_node.go @@ -31,8 +31,11 @@ type ( var zero [32]byte const ( - NodeWidth = 256 // Number of child per leaf node - StemSize = 31 // Number of bytes to travel before reaching a group of leaves + StemNodeWidth = 256 // Number of child per leaf node + StemSize = 31 // Number of bytes to travel before reaching a group of leaves + NodeTypeBytes = 1 // Size of node type prefix in serialization + HashSize = 32 // Size of a hash in bytes + BitmapSize = 32 // Size of the bitmap in a stem node ) const ( @@ -58,25 +61,28 @@ type BinaryNode interface { func SerializeNode(node BinaryNode) []byte { switch n := (node).(type) { case *InternalNode: - var serialized [65]byte + // InternalNode: 1 byte type + 32 bytes left hash + 32 bytes right hash + var serialized [NodeTypeBytes + HashSize + HashSize]byte serialized[0] = nodeTypeInternal copy(serialized[1:33], n.left.Hash().Bytes()) copy(serialized[33:65], n.right.Hash().Bytes()) return serialized[:] case *StemNode: - var serialized [32 + 32 + 256*32]byte + // StemNode: 1 byte type + 31 bytes stem + 32 bytes bitmap + 256*32 bytes values + var serialized [NodeTypeBytes + StemSize + BitmapSize + StemNodeWidth*HashSize]byte serialized[0] = nodeTypeStem - copy(serialized[1:32], node.(*StemNode).Stem) - bitmap := serialized[32:64] - offset := 64 - for i, v := range node.(*StemNode).Values { + copy(serialized[NodeTypeBytes:NodeTypeBytes+StemSize], n.Stem) + bitmap := serialized[NodeTypeBytes+StemSize : NodeTypeBytes+StemSize+BitmapSize] + offset := NodeTypeBytes + StemSize + BitmapSize + for i, v := range n.Values { if v != nil { bitmap[i/8] |= 1 << (7 - (i % 8)) - copy(serialized[offset:offset+32], v) - offset += 32 + copy(serialized[offset:offset+HashSize], v) + offset += HashSize } } - return serialized[:] + // Only return the actual data, not the entire array + return serialized[:offset] default: panic("invalid node type") } @@ -84,8 +90,18 @@ func SerializeNode(node BinaryNode) []byte { var invalidSerializedLength = errors.New("invalid serialized node length") -// DeserializeNode deserializes a binary trie node from a byte slice. +// DeserializeNode deserializes a binary trie node from a byte slice. The +// hash will be recomputed from the deserialized data. func DeserializeNode(serialized []byte, depth int) (BinaryNode, error) { + return deserializeNode(serialized, depth, common.Hash{}, true) +} + +// DeserializeNodeWithHash deserializes a binary trie node from a byte slice, using the provided hash. +func DeserializeNodeWithHash(serialized []byte, depth int, hn common.Hash) (BinaryNode, error) { + return deserializeNode(serialized, depth, hn, false) +} + +func deserializeNode(serialized []byte, depth int, hn common.Hash, mustRecompute bool) (BinaryNode, error) { if len(serialized) == 0 { return Empty{}, nil } @@ -96,31 +112,35 @@ func DeserializeNode(serialized []byte, depth int) (BinaryNode, error) { return nil, invalidSerializedLength } return &InternalNode{ - depth: depth, - left: HashedNode(common.BytesToHash(serialized[1:33])), - right: HashedNode(common.BytesToHash(serialized[33:65])), + depth: depth, + left: HashedNode(common.BytesToHash(serialized[1:33])), + right: HashedNode(common.BytesToHash(serialized[33:65])), + hash: hn, + mustRecompute: mustRecompute, }, nil case nodeTypeStem: if len(serialized) < 64 { return nil, invalidSerializedLength } - var values [256][]byte - bitmap := serialized[32:64] - offset := 64 + var values [StemNodeWidth][]byte + bitmap := serialized[NodeTypeBytes+StemSize : NodeTypeBytes+StemSize+BitmapSize] + offset := NodeTypeBytes + StemSize + BitmapSize - for i := range 256 { + for i := range StemNodeWidth { if bitmap[i/8]>>(7-(i%8))&1 == 1 { - if len(serialized) < offset+32 { + if len(serialized) < offset+HashSize { return nil, invalidSerializedLength } - values[i] = serialized[offset : offset+32] - offset += 32 + values[i] = serialized[offset : offset+HashSize] + offset += HashSize } } return &StemNode{ - Stem: serialized[1:32], - Values: values[:], - depth: depth, + Stem: serialized[NodeTypeBytes : NodeTypeBytes+StemSize], + Values: values[:], + depth: depth, + hash: hn, + mustRecompute: mustRecompute, }, nil default: return nil, errors.New("invalid node type") diff --git a/trie/bintrie/binary_node_test.go b/trie/bintrie/binary_node_test.go index b21daaab69..242743ba53 100644 --- a/trie/bintrie/binary_node_test.go +++ b/trie/bintrie/binary_node_test.go @@ -77,12 +77,12 @@ func TestSerializeDeserializeInternalNode(t *testing.T) { // TestSerializeDeserializeStemNode tests serialization and deserialization of StemNode func TestSerializeDeserializeStemNode(t *testing.T) { // Create a stem node with some values - stem := make([]byte, 31) + stem := make([]byte, StemSize) for i := range stem { stem[i] = byte(i) } - var values [256][]byte + var values [StemNodeWidth][]byte // Add some values at different indices values[0] = common.HexToHash("0x0101010101010101010101010101010101010101010101010101010101010101").Bytes() values[10] = common.HexToHash("0x0202020202020202020202020202020202020202020202020202020202020202").Bytes() @@ -103,7 +103,7 @@ func TestSerializeDeserializeStemNode(t *testing.T) { } // Check the stem is correctly serialized - if !bytes.Equal(serialized[1:32], stem) { + if !bytes.Equal(serialized[1:1+StemSize], stem) { t.Errorf("Stem mismatch in serialized data") } @@ -136,7 +136,7 @@ func TestSerializeDeserializeStemNode(t *testing.T) { } // Check that other values are nil - for i := range NodeWidth { + for i := range StemNodeWidth { if i == 0 || i == 10 || i == 255 { continue } @@ -218,15 +218,15 @@ func TestKeyToPath(t *testing.T) { }, { name: "max valid depth", - depth: 31 * 8, - key: make([]byte, 32), - expected: make([]byte, 31*8+1), + depth: StemSize * 8, + key: make([]byte, HashSize), + expected: make([]byte, StemSize*8+1), wantErr: false, }, { name: "depth too large", - depth: 31*8 + 1, - key: make([]byte, 32), + depth: StemSize*8 + 1, + key: make([]byte, HashSize), wantErr: true, }, } diff --git a/trie/bintrie/empty.go b/trie/bintrie/empty.go index 7cfe373b35..252146a4a7 100644 --- a/trie/bintrie/empty.go +++ b/trie/bintrie/empty.go @@ -32,9 +32,10 @@ func (e Empty) Insert(key []byte, value []byte, _ NodeResolverFn, depth int) (Bi var values [256][]byte values[key[31]] = value return &StemNode{ - Stem: slices.Clone(key[:31]), - Values: values[:], - depth: depth, + Stem: slices.Clone(key[:31]), + Values: values[:], + depth: depth, + mustRecompute: true, }, nil } @@ -53,9 +54,10 @@ func (e Empty) GetValuesAtStem(_ []byte, _ NodeResolverFn) ([][]byte, error) { func (e Empty) InsertValuesAtStem(key []byte, values [][]byte, _ NodeResolverFn, depth int) (BinaryNode, error) { return &StemNode{ - Stem: slices.Clone(key[:31]), - Values: values, - depth: depth, + Stem: slices.Clone(key[:31]), + Values: values, + depth: depth, + mustRecompute: true, }, nil } diff --git a/trie/bintrie/hashed_node.go b/trie/bintrie/hashed_node.go index 8f9fd66a59..e44c6d1e8a 100644 --- a/trie/bintrie/hashed_node.go +++ b/trie/bintrie/hashed_node.go @@ -46,8 +46,31 @@ func (h HashedNode) GetValuesAtStem(_ []byte, _ NodeResolverFn) ([][]byte, error return nil, errors.New("attempted to get values from an unresolved node") } -func (h HashedNode) InsertValuesAtStem(key []byte, values [][]byte, resolver NodeResolverFn, depth int) (BinaryNode, error) { - return nil, errors.New("insertValuesAtStem not implemented for hashed node") +func (h HashedNode) InsertValuesAtStem(stem []byte, values [][]byte, resolver NodeResolverFn, depth int) (BinaryNode, error) { + // Step 1: Generate the path for this node's position in the tree + path, err := keyToPath(depth, stem) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem path generation error: %w", err) + } + + if resolver == nil { + return nil, errors.New("InsertValuesAtStem resolve error: resolver is nil") + } + + // Step 2: Resolve the hashed node to get the actual node data + data, err := resolver(path, common.Hash(h)) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem resolve error: %w", err) + } + + // Step 3: Deserialize the resolved data into a concrete node + node, err := DeserializeNodeWithHash(data, depth, common.Hash(h)) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem node deserialization error: %w", err) + } + + // Step 4: Call InsertValuesAtStem on the resolved concrete node + return node.InsertValuesAtStem(stem, values, resolver, depth) } func (h HashedNode) toDot(parent string, path string) string { @@ -58,7 +81,8 @@ func (h HashedNode) toDot(parent string, path string) string { } func (h HashedNode) CollectNodes([]byte, NodeFlushFn) error { - return errors.New("collectNodes not implemented for hashed node") + // HashedNodes are already persisted in the database and don't need to be collected. + return nil } func (h HashedNode) GetHeight() int { diff --git a/trie/bintrie/hashed_node_test.go b/trie/bintrie/hashed_node_test.go index 0c19ae0c57..f9e6984888 100644 --- a/trie/bintrie/hashed_node_test.go +++ b/trie/bintrie/hashed_node_test.go @@ -17,6 +17,7 @@ package bintrie import ( + "bytes" "testing" "github.com/ethereum/go-ethereum/common" @@ -59,8 +60,8 @@ func TestHashedNodeCopy(t *testing.T) { func TestHashedNodeInsert(t *testing.T) { node := HashedNode(common.HexToHash("0x1234")) - key := make([]byte, 32) - value := make([]byte, 32) + key := make([]byte, HashSize) + value := make([]byte, HashSize) _, err := node.Insert(key, value, nil, 0) if err == nil { @@ -76,7 +77,7 @@ func TestHashedNodeInsert(t *testing.T) { func TestHashedNodeGetValuesAtStem(t *testing.T) { node := HashedNode(common.HexToHash("0x1234")) - stem := make([]byte, 31) + stem := make([]byte, StemSize) _, err := node.GetValuesAtStem(stem, nil) if err == nil { t.Fatal("Expected error for GetValuesAtStem on HashedNode") @@ -91,17 +92,85 @@ func TestHashedNodeGetValuesAtStem(t *testing.T) { func TestHashedNodeInsertValuesAtStem(t *testing.T) { node := HashedNode(common.HexToHash("0x1234")) - stem := make([]byte, 31) - values := make([][]byte, 256) + stem := make([]byte, StemSize) + values := make([][]byte, StemNodeWidth) + // Test 1: nil resolver should return an error _, err := node.InsertValuesAtStem(stem, values, nil, 0) if err == nil { - t.Fatal("Expected error for InsertValuesAtStem on HashedNode") + t.Fatal("Expected error for InsertValuesAtStem on HashedNode with nil resolver") } - if err.Error() != "insertValuesAtStem not implemented for hashed node" { + if err.Error() != "InsertValuesAtStem resolve error: resolver is nil" { t.Errorf("Unexpected error message: %v", err) } + + // Test 2: mock resolver returning invalid data should return deserialization error + mockResolver := func(path []byte, hash common.Hash) ([]byte, error) { + // Return invalid/nonsense data that cannot be deserialized + return []byte{0xff, 0xff, 0xff}, nil + } + + _, err = node.InsertValuesAtStem(stem, values, mockResolver, 0) + if err == nil { + t.Fatal("Expected error for InsertValuesAtStem on HashedNode with invalid resolver data") + } + + expectedPrefix := "InsertValuesAtStem node deserialization error:" + if len(err.Error()) < len(expectedPrefix) || err.Error()[:len(expectedPrefix)] != expectedPrefix { + t.Errorf("Expected deserialization error, got: %v", err) + } + + // Test 3: mock resolver returning valid serialized node should succeed + stem = make([]byte, StemSize) + stem[0] = 0xaa + var originalValues [StemNodeWidth][]byte + originalValues[0] = common.HexToHash("0x1111111111111111111111111111111111111111111111111111111111111111").Bytes() + originalValues[1] = common.HexToHash("0x2222222222222222222222222222222222222222222222222222222222222222").Bytes() + + originalNode := &StemNode{ + Stem: stem, + Values: originalValues[:], + depth: 0, + } + + // Serialize the node + serialized := SerializeNode(originalNode) + + // Create a mock resolver that returns the serialized node + validResolver := func(path []byte, hash common.Hash) ([]byte, error) { + return serialized, nil + } + + var newValues [StemNodeWidth][]byte + newValues[2] = common.HexToHash("0x3333333333333333333333333333333333333333333333333333333333333333").Bytes() + + resolvedNode, err := node.InsertValuesAtStem(stem, newValues[:], validResolver, 0) + if err != nil { + t.Fatalf("Expected successful resolution and insertion, got error: %v", err) + } + + resultStem, ok := resolvedNode.(*StemNode) + if !ok { + t.Fatalf("Expected resolved node to be *StemNode, got %T", resolvedNode) + } + + if !bytes.Equal(resultStem.Stem, stem) { + t.Errorf("Stem mismatch: expected %x, got %x", stem, resultStem.Stem) + } + + // Verify the original values are preserved + if !bytes.Equal(resultStem.Values[0], originalValues[0]) { + t.Errorf("Original value at index 0 not preserved: expected %x, got %x", originalValues[0], resultStem.Values[0]) + } + if !bytes.Equal(resultStem.Values[1], originalValues[1]) { + t.Errorf("Original value at index 1 not preserved: expected %x, got %x", originalValues[1], resultStem.Values[1]) + } + + // Verify the new value was inserted + if !bytes.Equal(resultStem.Values[2], newValues[2]) { + t.Errorf("New value at index 2 not inserted correctly: expected %x, got %x", newValues[2], resultStem.Values[2]) + } } // TestHashedNodeToDot tests the toDot method for visualization diff --git a/core/rawdb/database_tablewriter_unix.go b/trie/bintrie/hasher.go similarity index 71% rename from core/rawdb/database_tablewriter_unix.go rename to trie/bintrie/hasher.go index 8bec5396e8..b81c145723 100644 --- a/core/rawdb/database_tablewriter_unix.go +++ b/trie/bintrie/hasher.go @@ -1,4 +1,4 @@ -// Copyright 2025 The go-ethereum Authors +// Copyright 2026 go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -14,20 +14,26 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//go:build !tinygo -// +build !tinygo - -package rawdb +package bintrie import ( - "io" - - "github.com/olekukonko/tablewriter" + "crypto/sha256" + "hash" + "sync" ) -// Re-export the real tablewriter types and functions -type Table = tablewriter.Table +var sha256Pool = sync.Pool{ + New: func() any { + return sha256.New() + }, +} + +func newSha256() hash.Hash { + h := sha256Pool.Get().(hash.Hash) + h.Reset() + return h +} -func newTableWriter(w io.Writer) *Table { - return tablewriter.NewWriter(w) +func returnSha256(h hash.Hash) { + sha256Pool.Put(h) } diff --git a/trie/bintrie/internal_node.go b/trie/bintrie/internal_node.go index f3ddd1aab0..946203bcfb 100644 --- a/trie/bintrie/internal_node.go +++ b/trie/bintrie/internal_node.go @@ -20,10 +20,30 @@ import ( "crypto/sha256" "errors" "fmt" + "math/bits" + "runtime" + "sync" "github.com/ethereum/go-ethereum/common" ) +// parallelDepth returns the tree depth below which Hash() spawns goroutines. +func parallelDepth() int { + return min(bits.Len(uint(runtime.NumCPU())), 8) +} + +// isDirty reports whether a BinaryNode child needs rehashing. +func isDirty(n BinaryNode) bool { + switch v := n.(type) { + case *InternalNode: + return v.mustRecompute + case *StemNode: + return v.mustRecompute + default: + return false + } +} + func keyToPath(depth int, key []byte) ([]byte, error) { if depth > 31*8 { return nil, errors.New("node too deep") @@ -40,6 +60,9 @@ func keyToPath(depth int, key []byte) ([]byte, error) { type InternalNode struct { left, right BinaryNode depth int + + mustRecompute bool // true if the hash needs to be recomputed + hash common.Hash // cached hash when mustRecompute == false } // GetValuesAtStem retrieves the group of values located at the given stem key. @@ -49,14 +72,26 @@ func (bt *InternalNode) GetValuesAtStem(stem []byte, resolver NodeResolverFn) ([ } bit := stem[bt.depth/8] >> (7 - (bt.depth % 8)) & 1 - var child *BinaryNode if bit == 0 { - child = &bt.left - } else { - child = &bt.right + if hn, ok := bt.left.(HashedNode); ok { + path, err := keyToPath(bt.depth, stem) + if err != nil { + return nil, fmt.Errorf("GetValuesAtStem resolve error: %w", err) + } + data, err := resolver(path, common.Hash(hn)) + if err != nil { + return nil, fmt.Errorf("GetValuesAtStem resolve error: %w", err) + } + node, err := DeserializeNodeWithHash(data, bt.depth+1, common.Hash(hn)) + if err != nil { + return nil, fmt.Errorf("GetValuesAtStem node deserialization error: %w", err) + } + bt.left = node + } + return bt.left.GetValuesAtStem(stem, resolver) } - if hn, ok := (*child).(HashedNode); ok { + if hn, ok := bt.right.(HashedNode); ok { path, err := keyToPath(bt.depth, stem) if err != nil { return nil, fmt.Errorf("GetValuesAtStem resolve error: %w", err) @@ -65,13 +100,13 @@ func (bt *InternalNode) GetValuesAtStem(stem []byte, resolver NodeResolverFn) ([ if err != nil { return nil, fmt.Errorf("GetValuesAtStem resolve error: %w", err) } - node, err := DeserializeNode(data, bt.depth+1) + node, err := DeserializeNodeWithHash(data, bt.depth+1, common.Hash(hn)) if err != nil { return nil, fmt.Errorf("GetValuesAtStem node deserialization error: %w", err) } - *child = node + bt.right = node } - return (*child).GetValuesAtStem(stem, resolver) + return bt.right.GetValuesAtStem(stem, resolver) } // Get retrieves the value for the given key. @@ -80,6 +115,9 @@ func (bt *InternalNode) Get(key []byte, resolver NodeResolverFn) ([]byte, error) if err != nil { return nil, fmt.Errorf("get error: %w", err) } + if values == nil { + return nil, nil + } return values[key[31]], nil } @@ -93,15 +131,45 @@ func (bt *InternalNode) Insert(key []byte, value []byte, resolver NodeResolverFn // Copy creates a deep copy of the node. func (bt *InternalNode) Copy() BinaryNode { return &InternalNode{ - left: bt.left.Copy(), - right: bt.right.Copy(), - depth: bt.depth, + left: bt.left.Copy(), + right: bt.right.Copy(), + depth: bt.depth, + mustRecompute: bt.mustRecompute, + hash: bt.hash, } } // Hash returns the hash of the node. func (bt *InternalNode) Hash() common.Hash { - h := sha256.New() + if !bt.mustRecompute { + return bt.hash + } + + // At shallow depths, parallelize when both children need rehashing: + // hash left subtree in a goroutine, right subtree inline, then combine. + // Skip goroutine overhead when only one child is dirty (common case + // for narrow state updates that touch a single path through the trie). + if bt.depth < parallelDepth() && isDirty(bt.left) && isDirty(bt.right) { + var input [64]byte + var lh common.Hash + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + lh = bt.left.Hash() + }() + rh := bt.right.Hash() + copy(input[32:], rh[:]) + wg.Wait() + copy(input[:32], lh[:]) + bt.hash = sha256.Sum256(input[:]) + bt.mustRecompute = false + return bt.hash + } + + // Deeper nodes: sequential using pooled hasher (goroutine overhead > hash cost) + h := newSha256() + defer returnSha256(h) if bt.left != nil { h.Write(bt.left.Hash().Bytes()) } else { @@ -112,23 +180,64 @@ func (bt *InternalNode) Hash() common.Hash { } else { h.Write(zero[:]) } - return common.BytesToHash(h.Sum(nil)) + bt.hash = common.BytesToHash(h.Sum(nil)) + bt.mustRecompute = false + return bt.hash } // InsertValuesAtStem inserts a full value group at the given stem in the internal node. // Already-existing values will be overwritten. func (bt *InternalNode) InsertValuesAtStem(stem []byte, values [][]byte, resolver NodeResolverFn, depth int) (BinaryNode, error) { - var ( - child *BinaryNode - err error - ) + var err error bit := stem[bt.depth/8] >> (7 - (bt.depth % 8)) & 1 if bit == 0 { - child = &bt.left - } else { - child = &bt.right + if bt.left == nil { + bt.left = Empty{} + } + + if hn, ok := bt.left.(HashedNode); ok { + path, err := keyToPath(bt.depth, stem) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem resolve error: %w", err) + } + data, err := resolver(path, common.Hash(hn)) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem resolve error: %w", err) + } + node, err := DeserializeNodeWithHash(data, bt.depth+1, common.Hash(hn)) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem node deserialization error: %w", err) + } + bt.left = node + } + + bt.left, err = bt.left.InsertValuesAtStem(stem, values, resolver, depth+1) + bt.mustRecompute = true + return bt, err + } + + if bt.right == nil { + bt.right = Empty{} + } + + if hn, ok := bt.right.(HashedNode); ok { + path, err := keyToPath(bt.depth, stem) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem resolve error: %w", err) + } + data, err := resolver(path, common.Hash(hn)) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem resolve error: %w", err) + } + node, err := DeserializeNodeWithHash(data, bt.depth+1, common.Hash(hn)) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem node deserialization error: %w", err) + } + bt.right = node } - *child, err = (*child).InsertValuesAtStem(stem, values, resolver, depth+1) + + bt.right, err = bt.right.InsertValuesAtStem(stem, values, resolver, depth+1) + bt.mustRecompute = true return bt, err } diff --git a/trie/bintrie/internal_node_test.go b/trie/bintrie/internal_node_test.go index 158d8b7147..69097483fd 100644 --- a/trie/bintrie/internal_node_test.go +++ b/trie/bintrie/internal_node_test.go @@ -239,6 +239,7 @@ func TestInternalNodeHash(t *testing.T) { // Changing a child should change the hash node.left = HashedNode(common.HexToHash("0x3333")) + node.mustRecompute = true hash3 := node.Hash() if hash1 == hash3 { t.Error("Hash didn't change after modifying left child") @@ -246,9 +247,10 @@ func TestInternalNodeHash(t *testing.T) { // Test with nil children (should use zero hash) nodeWithNil := &InternalNode{ - depth: 0, - left: nil, - right: HashedNode(common.HexToHash("0x4444")), + depth: 0, + left: nil, + right: HashedNode(common.HexToHash("0x4444")), + mustRecompute: true, } hashWithNil := nodeWithNil.Hash() if hashWithNil == (common.Hash{}) { diff --git a/trie/bintrie/iterator.go b/trie/bintrie/iterator.go index a6bab2bcfa..048d37f766 100644 --- a/trie/bintrie/iterator.go +++ b/trie/bintrie/iterator.go @@ -108,33 +108,54 @@ func (it *binaryNodeIterator) Next(descend bool) bool { } // go back to parent to get the next leaf + // Check if we're at the root before popping + if len(it.stack) == 1 { + it.lastErr = errIteratorEnd + return false + } it.stack = it.stack[:len(it.stack)-1] it.current = it.stack[len(it.stack)-1].Node it.stack[len(it.stack)-1].Index++ return it.Next(descend) case HashedNode: // resolve the node - data, err := it.trie.nodeResolver(it.Path(), common.Hash(node)) + resolverPath := it.Path() + data, err := it.trie.nodeResolver(resolverPath, common.Hash(node)) if err != nil { panic(err) } - it.current, err = DeserializeNode(data, len(it.stack)-1) + if data == nil { + // Empty/nil node — treat as Empty, backtrack + it.current = Empty{} + it.stack[len(it.stack)-1].Node = it.current + return it.Next(descend) + } + it.current, err = DeserializeNodeWithHash(data, len(it.stack)-1, common.Hash(node)) if err != nil { panic(err) } // update the stack and parent with the resolved node it.stack[len(it.stack)-1].Node = it.current - parent := &it.stack[len(it.stack)-2] - if parent.Index == 0 { - parent.Node.(*InternalNode).left = it.current - } else { - parent.Node.(*InternalNode).right = it.current + if len(it.stack) >= 2 { + parent := &it.stack[len(it.stack)-2] + if parent.Index == 0 { + parent.Node.(*InternalNode).left = it.current + } else { + parent.Node.(*InternalNode).right = it.current + } } return it.Next(descend) case Empty: - // do nothing - return false + // Empty node - go back to parent and continue + if len(it.stack) <= 1 { + it.lastErr = errIteratorEnd + return false + } + it.stack = it.stack[:len(it.stack)-1] + it.current = it.stack[len(it.stack)-1].Node + it.stack[len(it.stack)-1].Index++ + return it.Next(descend) default: panic("invalid node type") } @@ -183,9 +204,31 @@ func (it *binaryNodeIterator) NodeBlob() []byte { } // Leaf returns true iff the current node is a leaf node. +// In a Binary Trie, a StemNode contains up to 256 leaf values. +// The iterator is only considered to be "at a leaf" when it's positioned +// at a specific non-nil value within the StemNode, not just at the StemNode itself. func (it *binaryNodeIterator) Leaf() bool { - _, ok := it.current.(*StemNode) - return ok + sn, ok := it.current.(*StemNode) + if !ok { + return false + } + + // Check if we have a valid stack position + if len(it.stack) == 0 { + return false + } + + // The Index in the stack state points to the NEXT position after the current value. + // So if Index is 0, we haven't started iterating through the values yet. + // If Index is 5, we're currently at value[4] (the 5th value, 0-indexed). + idx := it.stack[len(it.stack)-1].Index + if idx == 0 || idx > 256 { + return false + } + + // Check if there's actually a value at the current position + currentValueIndex := idx - 1 + return sn.Values[currentValueIndex] != nil } // LeafKey returns the key of the leaf. The method panics if the iterator is not @@ -219,7 +262,7 @@ func (it *binaryNodeIterator) LeafProof() [][]byte { panic("LeafProof() called on an binary node iterator not at a leaf location") } - proof := make([][]byte, 0, len(it.stack)+NodeWidth) + proof := make([][]byte, 0, len(it.stack)+StemNodeWidth) // Build proof by walking up the stack and collecting sibling hashes for i := range it.stack[:len(it.stack)-2] { diff --git a/trie/bintrie/iterator_test.go b/trie/bintrie/iterator_test.go index 8773e9e0c5..3e717c07ba 100644 --- a/trie/bintrie/iterator_test.go +++ b/trie/bintrie/iterator_test.go @@ -1,4 +1,4 @@ -// Copyright 2025 The go-ethereum Authors +// Copyright 2026 go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -17,67 +17,223 @@ package bintrie import ( + "bytes" "testing" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/triedb" - "github.com/ethereum/go-ethereum/triedb/hashdb" - "github.com/ethereum/go-ethereum/triedb/pathdb" - "github.com/holiman/uint256" + "github.com/ethereum/go-ethereum/trie" ) -func newTestDatabase(diskdb ethdb.Database, scheme string) *triedb.Database { - config := &triedb.Config{Preimages: true} - if scheme == rawdb.HashScheme { - config.HashDB = &hashdb.Config{CleanCacheSize: 0} - } else { - config.PathDB = &pathdb.Config{TrieCleanSize: 0, StateCleanSize: 0} +// makeTrie creates a BinaryTrie populated with the given key-value pairs. +func makeTrie(t *testing.T, entries [][2]common.Hash) *BinaryTrie { + t.Helper() + tr := &BinaryTrie{ + root: NewBinaryNode(), + tracer: trie.NewPrevalueTracer(), } - return triedb.NewDatabase(diskdb, config) + for _, kv := range entries { + var err error + tr.root, err = tr.root.Insert(kv[0][:], kv[1][:], nil, 0) + if err != nil { + t.Fatal(err) + } + } + return tr } -func TestBinaryIterator(t *testing.T) { - trie, err := NewBinaryTrie(types.EmptyVerkleHash, newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme)) +// countLeaves iterates the trie and returns the number of leaves visited. +func countLeaves(t *testing.T, tr *BinaryTrie) int { + t.Helper() + it, err := newBinaryNodeIterator(tr, nil) if err != nil { t.Fatal(err) } - account0 := &types.StateAccount{ - Nonce: 1, - Balance: uint256.NewInt(2), - Root: types.EmptyRootHash, - CodeHash: nil, - } - // NOTE: the code size isn't written to the trie via TryUpdateAccount - // so it will be missing from the test nodes. - trie.UpdateAccount(common.Address{}, account0, 0) - account1 := &types.StateAccount{ - Nonce: 1337, - Balance: uint256.NewInt(2000), - Root: types.EmptyRootHash, - CodeHash: nil, - } - // This address is meant to hash to a value that has the same first byte as 0xbf - var clash = common.HexToAddress("69fd8034cdb20934dedffa7dccb4fb3b8062a8be") - trie.UpdateAccount(clash, account1, 0) - - // Manually go over every node to check that we get all - // the correct nodes. - it, err := trie.NodeIterator(nil) + leaves := 0 + for it.Next(true) { + if it.Leaf() { + leaves++ + } + } + if it.Error() != nil { + t.Fatalf("iterator error: %v", it.Error()) + } + return leaves +} + +// TestIteratorEmptyTrie verifies that iterating over an empty trie returns +// no nodes and reports no error. +func TestIteratorEmptyTrie(t *testing.T) { + tr := &BinaryTrie{ + root: Empty{}, + tracer: trie.NewPrevalueTracer(), + } + it, err := newBinaryNodeIterator(tr, nil) if err != nil { t.Fatal(err) } - var leafcount int + if it.Next(true) { + t.Fatal("expected no iteration over empty trie") + } + if it.Error() != nil { + t.Fatalf("unexpected error: %v", it.Error()) + } +} + +// TestIteratorSingleStem verifies iteration over a trie with a single stem +// node containing multiple values. +func TestIteratorSingleStem(t *testing.T) { + tr := makeTrie(t, [][2]common.Hash{ + {common.HexToHash("0000000000000000000000000000000000000000000000000000000000000003"), oneKey}, + {common.HexToHash("0000000000000000000000000000000000000000000000000000000000000007"), oneKey}, + {common.HexToHash("00000000000000000000000000000000000000000000000000000000000000FF"), oneKey}, + }) + if leaves := countLeaves(t, tr); leaves != 3 { + t.Fatalf("expected 3 leaves, got %d", leaves) + } +} + +// TestIteratorTwoStems verifies iteration over a trie with two stems +// separated by internal nodes, ensuring all leaves from both stems are visited. +func TestIteratorTwoStems(t *testing.T) { + tr := makeTrie(t, [][2]common.Hash{ + {common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001"), oneKey}, + {common.HexToHash("0000000000000000000000000000000000000000000000000000000000000002"), oneKey}, + {common.HexToHash("8000000000000000000000000000000000000000000000000000000000000001"), oneKey}, + {common.HexToHash("8000000000000000000000000000000000000000000000000000000000000002"), oneKey}, + }) + if leaves := countLeaves(t, tr); leaves != 4 { + t.Fatalf("expected 4 leaves, got %d", leaves) + } +} + +// TestIteratorLeafKeyAndBlob verifies that the iterator returns correct +// leaf keys and values. +func TestIteratorLeafKeyAndBlob(t *testing.T) { + key := common.HexToHash("0000000000000000000000000000000000000000000000000000000000000005") + val := common.HexToHash("00000000000000000000000000000000000000000000000000000000deadbeef") + tr := makeTrie(t, [][2]common.Hash{{key, val}}) + + it, err := newBinaryNodeIterator(tr, nil) + if err != nil { + t.Fatal(err) + } + + found := false for it.Next(true) { - t.Logf("Node: %x", it.Path()) if it.Leaf() { - leafcount++ - t.Logf("\tLeaf: %x", it.LeafKey()) + found = true + if !bytes.Equal(it.LeafKey(), key[:]) { + t.Fatalf("leaf key mismatch: got %x, want %x", it.LeafKey(), key) + } + if !bytes.Equal(it.LeafBlob(), val[:]) { + t.Fatalf("leaf blob mismatch: got %x, want %x", it.LeafBlob(), val) + } } } - if leafcount != 2 { - t.Fatalf("invalid leaf count: %d != 6", leafcount) + if !found { + t.Fatal("expected to find a leaf") + } +} + +// TestIteratorEmptyNodeBacktrack is a regression test for the Empty node +// backtracking bug. Before the fix, encountering an Empty child during +// iteration would terminate the walk prematurely instead of backtracking +// to the parent and continuing with the next sibling. +func TestIteratorEmptyNodeBacktrack(t *testing.T) { + tr := makeTrie(t, [][2]common.Hash{ + {common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001"), oneKey}, + {common.HexToHash("8000000000000000000000000000000000000000000000000000000000000001"), oneKey}, + }) + + if _, ok := tr.root.(*InternalNode); !ok { + t.Fatalf("expected InternalNode root, got %T", tr.root) + } + if leaves := countLeaves(t, tr); leaves != 2 { + t.Fatalf("expected 2 leaves, got %d (Empty backtrack bug?)", leaves) + } +} + +// TestIteratorHashedNodeNilData is a regression test for the nil-data guard. +// When nodeResolver encounters a zero-hash HashedNode, it returns (nil, nil). +// The iterator should treat this as Empty and continue rather than panicking. +func TestIteratorHashedNodeNilData(t *testing.T) { + tr := makeTrie(t, [][2]common.Hash{ + {common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001"), oneKey}, + {common.HexToHash("8000000000000000000000000000000000000000000000000000000000000001"), oneKey}, + }) + + root, ok := tr.root.(*InternalNode) + if !ok { + t.Fatalf("expected InternalNode root, got %T", tr.root) + } + + // Replace right child with a zero-hash HashedNode. nodeResolver + // short-circuits on common.Hash{} and returns (nil, nil), which + // triggers the nil-data guard in the iterator. + root.right = HashedNode(common.Hash{}) + + // Should not panic; the zero-hash right child should be treated as Empty. + if leaves := countLeaves(t, tr); leaves != 1 { + t.Fatalf("expected 1 leaf (zero-hash right node skipped), got %d", leaves) + } +} + +// TestIteratorManyStems verifies iteration correctness with many stems, +// producing a deep tree structure. +func TestIteratorManyStems(t *testing.T) { + entries := make([][2]common.Hash, 16) + for i := range entries { + var key common.Hash + key[0] = byte(i << 4) + key[31] = 1 + entries[i] = [2]common.Hash{key, oneKey} + } + tr := makeTrie(t, entries) + if leaves := countLeaves(t, tr); leaves != 16 { + t.Fatalf("expected 16 leaves, got %d", leaves) + } +} + +// TestIteratorDeepTree verifies iteration over a trie with stems that share +// a long common prefix, producing many intermediate InternalNodes. +func TestIteratorDeepTree(t *testing.T) { + tr := makeTrie(t, [][2]common.Hash{ + {common.HexToHash("0000000000C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0"), oneKey}, + {common.HexToHash("0000000000E00000000000000000000000000000000000000000000000000000"), twoKey}, + }) + if leaves := countLeaves(t, tr); leaves != 2 { + t.Fatalf("expected 2 leaves in deep tree, got %d", leaves) + } +} + +// TestIteratorNodeCount verifies the total number of Next(true) calls +// for a known tree structure. +func TestIteratorNodeCount(t *testing.T) { + tr := makeTrie(t, [][2]common.Hash{ + {common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001"), oneKey}, + {common.HexToHash("8000000000000000000000000000000000000000000000000000000000000001"), oneKey}, + }) + + it, err := newBinaryNodeIterator(tr, nil) + if err != nil { + t.Fatal(err) + } + + total := 0 + leaves := 0 + for it.Next(true) { + total++ + if it.Leaf() { + leaves++ + } + } + if leaves != 2 { + t.Fatalf("expected 2 leaves, got %d", leaves) + } + // Root(InternalNode) + leaf1 (from left StemNode) + leaf2 (from right StemNode) = 3 + // StemNodes are not returned as separate steps; the iterator advances + // directly to the first non-nil value within the stem. + if total != 3 { + t.Fatalf("expected 3 total nodes, got %d", total) } } diff --git a/trie/bintrie/key_encoding.go b/trie/bintrie/key_encoding.go index 13c2057371..c009f1529f 100644 --- a/trie/bintrie/key_encoding.go +++ b/trie/bintrie/key_encoding.go @@ -18,7 +18,6 @@ package bintrie import ( "bytes" - "crypto/sha256" "github.com/ethereum/go-ethereum/common" "github.com/holiman/uint256" @@ -33,47 +32,117 @@ const ( ) var ( - zeroHash = common.Hash{} - codeOffset = uint256.NewInt(128) + zeroInt = uint256.NewInt(0) + zeroHash = common.Hash{} + verkleNodeWidthLog2 = 8 + headerStorageOffset = uint256.NewInt(64) + codeOffset = uint256.NewInt(128) + codeStorageDelta = uint256.NewInt(0).Sub(codeOffset, headerStorageOffset) + mainStorageOffsetLshVerkleNodeWidth = new(uint256.Int).Lsh(uint256.NewInt(1), 248-uint(verkleNodeWidthLog2)) + CodeOffset = uint256.NewInt(128) + VerkleNodeWidth = uint256.NewInt(256) + HeaderStorageOffset = uint256.NewInt(64) + VerkleNodeWidthLog2 = 8 ) func GetBinaryTreeKey(addr common.Address, key []byte) []byte { - hasher := sha256.New() + return getBinaryTreeKey(addr, key, false) +} + +func getBinaryTreeKey(addr common.Address, offset []byte, overflow bool) []byte { + hasher := newSha256() + defer returnSha256(hasher) hasher.Write(zeroHash[:12]) hasher.Write(addr[:]) - hasher.Write(key[:31]) + var buf [32]byte + // key is big endian, hashed value is little endian + for i := range offset[:31] { + buf[i] = offset[30-i] + } + if overflow { + // Overflow detected when adding MAIN_STORAGE_OFFSET, + // reporting it in the shifter 32 byte value. + buf[31] = 1 + } + hasher.Write(buf[:]) k := hasher.Sum(nil) - k[31] = key[31] + k[31] = offset[31] return k } +func GetBinaryTreeKeyBasicData(addr common.Address) []byte { + var k [32]byte + k[31] = BasicDataLeafKey + return GetBinaryTreeKey(addr, k[:]) +} + func GetBinaryTreeKeyCodeHash(addr common.Address) []byte { var k [32]byte k[31] = CodeHashLeafKey return GetBinaryTreeKey(addr, k[:]) } -func GetBinaryTreeKeyStorageSlot(address common.Address, key []byte) []byte { - var k [32]byte +func GetBinaryTreeKeyStorageSlot(address common.Address, slotnum []byte) []byte { + var offset [32]byte // Case when the key belongs to the account header - if bytes.Equal(key[:31], zeroHash[:31]) && key[31] < 64 { - k[31] = 64 + key[31] - return GetBinaryTreeKey(address, k[:]) + if bytes.Equal(slotnum[:31], zeroHash[:31]) && slotnum[31] < 64 { + offset[31] = 64 + slotnum[31] + return GetBinaryTreeKey(address, offset[:]) } - // Set the main storage offset - // note that the first 64 bytes of the main offset storage - // are unreachable, which is consistent with the spec and - // what verkle does. - k[0] = 1 // 1 << 248 - copy(k[1:], key[:31]) - k[31] = key[31] + // Set the main storage offset offset = MAIN_STORAGE_OFFSET + slotnum + // * Note that MAIN_STORAGE_OFFSET is 1 << 248, so the number + // can overflow into a 33rd byte, but since the value is + // shifted by one byte in getBinaryTreeKey, this only takes + // note of the overflow, and the value will be added after + // the shift, in order to avoid allocating an extra byte. + // * Note that the first 64 bytes of the main offset storage + // are unreachable, which is consistent with the spec. + // * Note that `slotnum` is big-endian + overflow := slotnum[0] == 255 + copy(offset[:], slotnum) + offset[0] += 1 // 1 << 248, handle overflow out of band - return GetBinaryTreeKey(address, k[:]) + return getBinaryTreeKey(address, offset[:], overflow) } func GetBinaryTreeKeyCodeChunk(address common.Address, chunknr *uint256.Int) []byte { chunkOffset := new(uint256.Int).Add(codeOffset, chunknr).Bytes() return GetBinaryTreeKey(address, chunkOffset) } + +func StorageIndex(storageKey []byte) (*uint256.Int, byte) { + // If the storage slot is in the header, we need to add the header offset. + var key uint256.Int + key.SetBytes(storageKey) + if key.Cmp(codeStorageDelta) < 0 { + // This addition is always safe; it can't ever overflow since pos> (7 - (bt.depth % 8)) & 1 - n := &InternalNode{depth: bt.depth} + n := &InternalNode{depth: bt.depth, mustRecompute: true} bt.depth++ var child, other *BinaryNode if bitStem == 0 { @@ -65,33 +67,37 @@ func (bt *StemNode) Insert(key []byte, value []byte, _ NodeResolverFn, depth int } *other = Empty{} } else { - var values [256][]byte - values[key[31]] = value + var values [StemNodeWidth][]byte + values[key[StemSize]] = value *other = &StemNode{ - Stem: slices.Clone(key[:31]), - Values: values[:], - depth: depth + 1, + Stem: slices.Clone(key[:StemSize]), + Values: values[:], + depth: depth + 1, + mustRecompute: true, } } return n, nil } - if len(value) != 32 { + if len(value) != HashSize { return bt, errors.New("invalid insertion: value length") } - bt.Values[key[31]] = value + bt.Values[key[StemSize]] = value + bt.mustRecompute = true return bt, nil } // Copy creates a deep copy of the node. func (bt *StemNode) Copy() BinaryNode { - var values [256][]byte + var values [StemNodeWidth][]byte for i, v := range bt.Values { values[i] = slices.Clone(v) } return &StemNode{ - Stem: slices.Clone(bt.Stem), - Values: values[:], - depth: bt.depth, + Stem: slices.Clone(bt.Stem), + Values: values[:], + depth: bt.depth, + hash: bt.hash, + mustRecompute: bt.mustRecompute, } } @@ -102,17 +108,24 @@ func (bt *StemNode) GetHeight() int { // Hash returns the hash of the node. func (bt *StemNode) Hash() common.Hash { - var data [NodeWidth]common.Hash + if !bt.mustRecompute { + return bt.hash + } + + var data [StemNodeWidth]common.Hash + h := newSha256() + defer returnSha256(h) for i, v := range bt.Values { if v != nil { - h := sha256.Sum256(v) - data[i] = common.BytesToHash(h[:]) + h.Reset() + h.Write(v) + h.Sum(data[i][:0]) } } + h.Reset() - h := sha256.New() for level := 1; level <= 8; level++ { - for i := range NodeWidth / (1 << level) { + for i := range StemNodeWidth / (1 << level) { h.Reset() if data[i*2] == (common.Hash{}) && data[i*2+1] == (common.Hash{}) { @@ -130,7 +143,9 @@ func (bt *StemNode) Hash() common.Hash { h.Write(bt.Stem) h.Write([]byte{0}) h.Write(data[0][:]) - return common.BytesToHash(h.Sum(nil)) + bt.hash = common.BytesToHash(h.Sum(nil)) + bt.mustRecompute = false + return bt.hash } // CollectNodes collects all child nodes at a given path, and flushes it @@ -141,17 +156,20 @@ func (bt *StemNode) CollectNodes(path []byte, flush NodeFlushFn) error { } // GetValuesAtStem retrieves the group of values located at the given stem key. -func (bt *StemNode) GetValuesAtStem(_ []byte, _ NodeResolverFn) ([][]byte, error) { +func (bt *StemNode) GetValuesAtStem(stem []byte, _ NodeResolverFn) ([][]byte, error) { + if !bytes.Equal(bt.Stem, stem) { + return nil, nil + } return bt.Values[:], nil } // InsertValuesAtStem inserts a full value group at the given stem in the internal node. // Already-existing values will be overwritten. func (bt *StemNode) InsertValuesAtStem(key []byte, values [][]byte, _ NodeResolverFn, depth int) (BinaryNode, error) { - if !bytes.Equal(bt.Stem, key[:31]) { + if !bytes.Equal(bt.Stem, key[:StemSize]) { bitStem := bt.Stem[bt.depth/8] >> (7 - (bt.depth % 8)) & 1 - n := &InternalNode{depth: bt.depth} + n := &InternalNode{depth: bt.depth, mustRecompute: true} bt.depth++ var child, other *BinaryNode if bitStem == 0 { @@ -174,9 +192,10 @@ func (bt *StemNode) InsertValuesAtStem(key []byte, values [][]byte, _ NodeResolv *other = Empty{} } else { *other = &StemNode{ - Stem: slices.Clone(key[:31]), - Values: values, - depth: n.depth + 1, + Stem: slices.Clone(key[:StemSize]), + Values: values, + depth: n.depth + 1, + mustRecompute: true, } } return n, nil @@ -186,6 +205,7 @@ func (bt *StemNode) InsertValuesAtStem(key []byte, values [][]byte, _ NodeResolv for i, v := range values { if v != nil { bt.Values[i] = v + bt.mustRecompute = true } } return bt, nil @@ -206,7 +226,7 @@ func (bt *StemNode) toDot(parent, path string) string { // Key returns the full key for the given index. func (bt *StemNode) Key(i int) []byte { - var ret [32]byte + var ret [HashSize]byte copy(ret[:], bt.Stem) ret[StemSize] = byte(i) return ret[:] diff --git a/trie/bintrie/stem_node_test.go b/trie/bintrie/stem_node_test.go index e0ffd5c3c8..92c1b49e02 100644 --- a/trie/bintrie/stem_node_test.go +++ b/trie/bintrie/stem_node_test.go @@ -220,6 +220,7 @@ func TestStemNodeHash(t *testing.T) { // Changing a value should change the hash node.Values[1] = common.HexToHash("0x0202").Bytes() + node.mustRecompute = true hash3 := node.Hash() if hash1 == hash3 { t.Error("Hash didn't change after modifying values") @@ -251,27 +252,23 @@ func TestStemNodeGetValuesAtStem(t *testing.T) { } // Check that all values match - for i := 0; i < 256; i++ { + for i := range 256 { if !bytes.Equal(retrievedValues[i], values[i]) { t.Errorf("Value mismatch at index %d", i) } } - // GetValuesAtStem with different stem also returns the same values - // (implementation ignores the stem parameter) + // GetValuesAtStem with different stem should return nil differentStem := make([]byte, 31) differentStem[0] = 0xFF - retrievedValues2, err := node.GetValuesAtStem(differentStem, nil) + shouldBeNil, err := node.GetValuesAtStem(differentStem, nil) if err != nil { t.Fatalf("Failed to get values with different stem: %v", err) } - // Should still return the same values (stem is ignored) - for i := 0; i < 256; i++ { - if !bytes.Equal(retrievedValues2[i], values[i]) { - t.Errorf("Value mismatch at index %d with different stem", i) - } + if shouldBeNil != nil { + t.Error("Expected nil for different stem, got non-nil") } } diff --git a/trie/bintrie/trie.go b/trie/bintrie/trie.go index 0a8bd325f5..6c29239a87 100644 --- a/trie/bintrie/trie.go +++ b/trie/bintrie/trie.go @@ -33,6 +33,81 @@ import ( var errInvalidRootType = errors.New("invalid root type") +// ChunkedCode represents a sequence of HashSize-byte chunks of code (StemSize bytes of which +// are actual code, and NodeTypeBytes byte is the pushdata offset). +type ChunkedCode []byte + +// Copy the values here so as to avoid an import cycle +const ( + PUSH1 = byte(0x60) + PUSH32 = byte(0x7f) +) + +// ChunkifyCode generates the chunked version of an array representing EVM bytecode +// according to EIP-7864 specification. +// +// The code is divided into HashSize-byte chunks, where each chunk contains: +// - Byte 0: Metadata byte indicating the number of leading bytes that are PUSHDATA (0-StemSize) +// - Bytes 1-StemSize: Actual code bytes +// +// This format enables stateless clients to validate jump destinations within a chunk +// without requiring additional context. When a PUSH instruction's data spans multiple +// chunks, the metadata byte tells us how many bytes at the start of the chunk are +// part of the previous chunk's PUSH instruction data. +// +// For example: +// - If a chunk starts with regular code: metadata byte = 0 +// - If a PUSH32 instruction starts at byte 30 of chunk N: +// - Chunk N: normal, contains PUSH32 opcode + 1 byte of data +// - Chunk N+1: metadata = StemSize (entire chunk is PUSH data) +// - Chunk N+2: metadata = 1 (first byte is PUSH data, then normal code resumes) +// +// This chunking approach ensures that jump destination validity can be determined +// by examining only the chunk containing the potential JUMPDEST, making it ideal +// for stateless execution and verkle/binary tries. +// +// Reference: https://eips.ethereum.org/EIPS/eip-7864 +func ChunkifyCode(code []byte) ChunkedCode { + var ( + chunkOffset = 0 // offset in the chunk + chunkCount = len(code) / StemSize + codeOffset = 0 // offset in the code + ) + if len(code)%StemSize != 0 { + chunkCount++ + } + chunks := make([]byte, chunkCount*HashSize) + for i := 0; i < chunkCount; i++ { + // number of bytes to copy, StemSize unless the end of the code has been reached. + end := min(len(code), StemSize*(i+1)) + copy(chunks[i*HashSize+1:], code[StemSize*i:end]) // copy the code itself + + // chunk offset = taken from the last chunk. + if chunkOffset > StemSize { + // skip offset calculation if push data covers the whole chunk + chunks[i*HashSize] = StemSize + chunkOffset = 1 + continue + } + chunks[HashSize*i] = byte(chunkOffset) + chunkOffset = 0 + + // Check each instruction and update the offset it should be 0 unless + // a PUSH-N overflows. + for ; codeOffset < end; codeOffset++ { + if code[codeOffset] >= PUSH1 && code[codeOffset] <= PUSH32 { + codeOffset += int(code[codeOffset] - PUSH1 + 1) + if codeOffset+1 >= StemSize*(i+1) { + codeOffset++ + chunkOffset = codeOffset - StemSize*(i+1) + break + } + } + } + } + return chunks +} + // NewBinaryNode creates a new empty binary trie func NewBinaryNode() BinaryNode { return Empty{} @@ -68,7 +143,7 @@ func NewBinaryTrie(root common.Hash, db database.NodeDatabase) (*BinaryTrie, err if err != nil { return nil, err } - node, err := DeserializeNode(blob, 0) + node, err := DeserializeNodeWithHash(blob, 0, root) if err != nil { return nil, err } @@ -114,7 +189,7 @@ func (t *BinaryTrie) GetAccount(addr common.Address) (*types.StateAccount, error ) switch r := t.root.(type) { case *InternalNode: - values, err = r.GetValuesAtStem(key[:31], t.nodeResolver) + values, err = r.GetValuesAtStem(key[:StemSize], t.nodeResolver) case *StemNode: values = r.Values case Empty: @@ -161,15 +236,15 @@ func (t *BinaryTrie) GetAccount(addr common.Address) (*types.StateAccount, error // not be modified by the caller. If a node was not found in the database, a // trie.MissingNodeError is returned. func (t *BinaryTrie) GetStorage(addr common.Address, key []byte) ([]byte, error) { - return t.root.Get(GetBinaryTreeKey(addr, key), t.nodeResolver) + return t.root.Get(GetBinaryTreeKeyStorageSlot(addr, key), t.nodeResolver) } // UpdateAccount updates the account information for the given address. func (t *BinaryTrie) UpdateAccount(addr common.Address, acc *types.StateAccount, codeLen int) error { var ( err error - basicData [32]byte - values = make([][]byte, NodeWidth) + basicData [HashSize]byte + values = make([][]byte, StemNodeWidth) stem = GetBinaryTreeKey(addr, zero[:]) ) binary.BigEndian.PutUint32(basicData[BasicDataCodeSizeOffset-1:], uint32(codeLen)) @@ -177,14 +252,14 @@ func (t *BinaryTrie) UpdateAccount(addr common.Address, acc *types.StateAccount, // Because the balance is a max of 16 bytes, truncate // the extra values. This happens in devmode, where - // 0xff**32 is allocated to the developer account. + // 0xff**HashSize is allocated to the developer account. balanceBytes := acc.Balance.Bytes() // TODO: reduce the size of the allocation in devmode, then panic instead // of truncating. if len(balanceBytes) > 16 { balanceBytes = balanceBytes[16:] } - copy(basicData[32-len(balanceBytes):], balanceBytes[:]) + copy(basicData[HashSize-len(balanceBytes):], balanceBytes[:]) values[BasicDataLeafKey] = basicData[:] values[CodeHashLeafKey] = acc.CodeHash[:] @@ -205,11 +280,11 @@ func (t *BinaryTrie) UpdateStem(key []byte, values [][]byte) error { // database, a trie.MissingNodeError is returned. func (t *BinaryTrie) UpdateStorage(address common.Address, key, value []byte) error { k := GetBinaryTreeKeyStorageSlot(address, key) - var v [32]byte - if len(value) >= 32 { - copy(v[:], value[:32]) + var v [HashSize]byte + if len(value) >= HashSize { + copy(v[:], value[:HashSize]) } else { - copy(v[32-len(value):], value[:]) + copy(v[HashSize-len(value):], value[:]) } root, err := t.root.Insert(k, v[:], t.nodeResolver, 0) if err != nil { @@ -227,8 +302,8 @@ func (t *BinaryTrie) DeleteAccount(addr common.Address) error { // DeleteStorage removes any existing value for key from the trie. If a node was not // found in the database, a trie.MissingNodeError is returned. func (t *BinaryTrie) DeleteStorage(addr common.Address, key []byte) error { - k := GetBinaryTreeKey(addr, key) - var zero [32]byte + k := GetBinaryTreeKeyStorageSlot(addr, key) + var zero [HashSize]byte root, err := t.root.Insert(k, zero[:], t.nodeResolver, 0) if err != nil { return fmt.Errorf("DeleteStorage (%x) error: %v", addr, err) @@ -246,12 +321,12 @@ func (t *BinaryTrie) Hash() common.Hash { // Commit writes all nodes to the trie's memory database, tracking the internal // and external (for account tries) references. func (t *BinaryTrie) Commit(_ bool) (common.Hash, *trienode.NodeSet) { - root := t.root.(*InternalNode) nodeset := trienode.NewNodeSet(common.Hash{}) - err := root.CollectNodes(nil, func(path []byte, node BinaryNode) { + // The root can be any type of BinaryNode (InternalNode, StemNode, etc.) + err := t.root.CollectNodes(nil, func(path []byte, node BinaryNode) { serialized := SerializeNode(node) - nodeset.AddNode(path, trienode.NewNodeWithPrev(common.Hash{}, serialized, t.tracer.Get(path))) + nodeset.AddNode(path, trienode.NewNodeWithPrev(node.Hash(), serialized, t.tracer.Get(path))) }) if err != nil { panic(fmt.Errorf("CollectNodes failed: %v", err)) @@ -299,23 +374,23 @@ func (t *BinaryTrie) IsVerkle() bool { // Note: the basic data leaf needs to have been previously created for this to work func (t *BinaryTrie) UpdateContractCode(addr common.Address, codeHash common.Hash, code []byte) error { var ( - chunks = trie.ChunkifyCode(code) + chunks = ChunkifyCode(code) values [][]byte key []byte err error ) - for i, chunknr := 0, uint64(0); i < len(chunks); i, chunknr = i+32, chunknr+1 { - groupOffset := (chunknr + 128) % 256 + for i, chunknr := 0, uint64(0); i < len(chunks); i, chunknr = i+HashSize, chunknr+1 { + groupOffset := (chunknr + 128) % StemNodeWidth if groupOffset == 0 /* start of new group */ || chunknr == 0 /* first chunk in header group */ { - values = make([][]byte, NodeWidth) - var offset [32]byte - binary.LittleEndian.PutUint64(offset[24:], chunknr+128) + values = make([][]byte, StemNodeWidth) + var offset [HashSize]byte + binary.BigEndian.PutUint64(offset[24:], chunknr+128) key = GetBinaryTreeKey(addr, offset[:]) } - values[groupOffset] = chunks[i : i+32] + values[groupOffset] = chunks[i : i+HashSize] - if groupOffset == 255 || len(chunks)-i <= 32 { - err = t.UpdateStem(key[:31], values) + if groupOffset == StemNodeWidth-1 || len(chunks)-i <= HashSize { + err = t.UpdateStem(key[:StemSize], values) if err != nil { return fmt.Errorf("UpdateContractCode (addr=%x) error: %w", addr[:], err) @@ -349,5 +424,5 @@ func (t *BinaryTrie) PrefetchStorage(addr common.Address, keys [][]byte) error { // Witness returns a set containing all trie nodes that have been accessed. func (t *BinaryTrie) Witness() map[string][]byte { - panic("not implemented") + return t.tracer.Values() } diff --git a/trie/bintrie/trie_test.go b/trie/bintrie/trie_test.go index 84f7689549..256fd218e2 100644 --- a/trie/bintrie/trie_test.go +++ b/trie/bintrie/trie_test.go @@ -22,10 +22,13 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/trie" + "github.com/holiman/uint256" ) var ( - zeroKey = [32]byte{} + zeroKey = [HashSize]byte{} oneKey = common.HexToHash("0101010101010101010101010101010101010101010101010101010101010101") twoKey = common.HexToHash("0202020202020202020202020202020202020202020202020202020202020202") threeKey = common.HexToHash("0303030303030303030303030303030303030303030303030303030303030303") @@ -158,8 +161,8 @@ func TestInsertDuplicateKey(t *testing.T) { func TestLargeNumberOfEntries(t *testing.T) { var err error tree := NewBinaryNode() - for i := range 256 { - var key [32]byte + for i := range StemNodeWidth { + var key [HashSize]byte key[0] = byte(i) tree, err = tree.Insert(key[:], ffKey[:], nil, 0) if err != nil { @@ -182,7 +185,7 @@ func TestMerkleizeMultipleEntries(t *testing.T) { common.HexToHash("8100000000000000000000000000000000000000000000000000000000000000").Bytes(), } for i, key := range keys { - var v [32]byte + var v [HashSize]byte binary.LittleEndian.PutUint64(v[:8], uint64(i)) tree, err = tree.Insert(key, v[:], nil, 0) if err != nil { @@ -195,3 +198,97 @@ func TestMerkleizeMultipleEntries(t *testing.T) { t.Fatalf("invalid root, expected=%x, got = %x", expected, got) } } + +// TestStorageRoundTrip verifies that GetStorage and DeleteStorage use the same +// key mapping as UpdateStorage (GetBinaryTreeKeyStorageSlot). This is a regression +// test: previously GetStorage and DeleteStorage used GetBinaryTreeKey directly, +// which produced different tree keys and broke the read/delete path. +func TestStorageRoundTrip(t *testing.T) { + tracer := trie.NewPrevalueTracer() + tr := &BinaryTrie{ + root: NewBinaryNode(), + tracer: tracer, + } + addr := common.HexToAddress("0x1234567890abcdef1234567890abcdef12345678") + + // Create an account first so the root becomes an InternalNode, + // which is the realistic state when storage operations happen. + acc := &types.StateAccount{ + Nonce: 1, + Balance: uint256.NewInt(1000), + CodeHash: common.HexToHash("c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + } + if err := tr.UpdateAccount(addr, acc, 0); err != nil { + t.Fatalf("UpdateAccount error: %v", err) + } + + // Test main storage slots (key[31] >= 64 or key[:31] != 0). + // These produce a different stem than the account data, so after + // UpdateAccount + UpdateStorage the root is an InternalNode. + // Note: header slots (key[31] < 64, key[:31] == 0) share the same + // stem as account data and are covered by GetAccount/UpdateAccount path. + slots := []common.Hash{ + common.HexToHash("00000000000000000000000000000000000000000000000000000000000000FF"), // main storage (slot 255) + common.HexToHash("0100000000000000000000000000000000000000000000000000000000000001"), // main storage (non-zero prefix) + } + val := common.TrimLeftZeroes(common.HexToHash("00000000000000000000000000000000000000000000000000000000deadbeef").Bytes()) + + for _, slot := range slots { + // Write + if err := tr.UpdateStorage(addr, slot[:], val); err != nil { + t.Fatalf("UpdateStorage(%x) error: %v", slot, err) + } + // Read back + got, err := tr.GetStorage(addr, slot[:]) + if err != nil { + t.Fatalf("GetStorage(%x) error: %v", slot, err) + } + if len(got) == 0 { + t.Fatalf("GetStorage(%x) returned empty, expected value", slot) + } + // Verify value (right-justified in 32 bytes) + var expected [HashSize]byte + copy(expected[HashSize-len(val):], val) + if !bytes.Equal(got, expected[:]) { + t.Fatalf("GetStorage(%x) = %x, want %x", slot, got, expected) + } + // Delete + if err := tr.DeleteStorage(addr, slot[:]); err != nil { + t.Fatalf("DeleteStorage(%x) error: %v", slot, err) + } + // Verify deleted (should read as zero, not the old value) + got, err = tr.GetStorage(addr, slot[:]) + if err != nil { + t.Fatalf("GetStorage(%x) after delete error: %v", slot, err) + } + if len(got) > 0 && !bytes.Equal(got, zero[:]) { + t.Fatalf("GetStorage(%x) after delete = %x, expected zero", slot, got) + } + } +} + +func TestBinaryTrieWitness(t *testing.T) { + tracer := trie.NewPrevalueTracer() + + tr := &BinaryTrie{ + root: NewBinaryNode(), + tracer: tracer, + } + if w := tr.Witness(); len(w) != 0 { + t.Fatal("expected empty witness for fresh trie") + } + + tracer.Put([]byte("path1"), []byte("blob1")) + tracer.Put([]byte("path2"), []byte("blob2")) + + witness := tr.Witness() + if len(witness) != 2 { + t.Fatalf("expected 2 witness entries, got %d", len(witness)) + } + if !bytes.Equal(witness[string([]byte("path1"))], []byte("blob1")) { + t.Fatal("unexpected witness value for path1") + } + if !bytes.Equal(witness[string([]byte("path2"))], []byte("blob2")) { + t.Fatal("unexpected witness value for path2") + } +} diff --git a/trie/inspect.go b/trie/inspect.go new file mode 100644 index 0000000000..5e781e7900 --- /dev/null +++ b/trie/inspect.go @@ -0,0 +1,933 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "bufio" + "bytes" + "cmp" + "container/heap" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "slices" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/internal/tablewriter" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/triedb/database" + "golang.org/x/sync/errgroup" + "golang.org/x/sync/semaphore" +) + +const ( + inspectDumpRecordSize = 32 + trieStatLevels*(3*4+8) + inspectDefaultTopN = 10 + inspectParallelism = int64(16) +) + +// inspector is used by the inner inspect function to coordinate across threads. +type inspector struct { + triedb database.NodeDatabase + root common.Hash + + config *InspectConfig + accountStat *LevelStats + + sem *semaphore.Weighted + + // Pass 1: dump file writer. + dumpMu sync.Mutex + dumpBuf *bufio.Writer + dumpFile *os.File + storageRecordsWritten atomic.Uint64 + + errMu sync.Mutex + err error +} + +// InspectConfig is a set of options to control inspection and format the output. +// TopN determines the maximum number of entries retained for each top-list. +// Path controls optional JSON output. DumpPath controls the pass-1 dump location. +type InspectConfig struct { + NoStorage bool + TopN int + Path string + DumpPath string +} + +// Inspect walks the trie with the given root and records the number and type of +// nodes at each depth. Storage trie stats are streamed to disk in fixed-size +// records, then summarized in a second pass. +func Inspect(triedb database.NodeDatabase, root common.Hash, config *InspectConfig) error { + trie, err := New(TrieID(root), triedb) + if err != nil { + return fmt.Errorf("fail to open trie %s: %w", root, err) + } + config = normalizeInspectConfig(config) + + dumpFile, err := os.OpenFile(config.DumpPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return fmt.Errorf("failed to create trie dump %s: %w", config.DumpPath, err) + } + in := inspector{ + triedb: triedb, + root: root, + config: config, + accountStat: NewLevelStats(), + sem: semaphore.NewWeighted(inspectParallelism), + dumpBuf: bufio.NewWriterSize(dumpFile, 1<<20), + dumpFile: dumpFile, + } + + // Start progress reporter + start := time.Now() + done := make(chan struct{}) + go func() { + ticker := time.NewTicker(8 * time.Second) + defer ticker.Stop() + for { + select { + case <-ticker.C: + accountNodes := in.accountStat.TotalNodes() + storageRecords := in.storageRecordsWritten.Load() + log.Info("Inspecting trie", + "accountNodes", accountNodes, + "storageRecords", storageRecords, + "elapsed", common.PrettyDuration(time.Since(start))) + case <-done: + return + } + } + }() + + in.recordRootSize(trie, root, in.accountStat) + in.inspect(trie, trie.root, 0, []byte{}, in.accountStat) + + // inspect is synchronous: it waits for all spawned goroutines in its + // subtree before returning, so no additional wait is needed here. + + // Persist account trie stats as the sentinel record. + in.writeDumpRecord(common.Hash{}, in.accountStat) + if err := in.closeDump(); err != nil { + in.setError(err) + } + + // Stop progress reporter + close(done) + + if err := in.getError(); err != nil { + return err + } + return Summarize(config.DumpPath, config) +} + +// InspectContract inspects the on-disk footprint of a single contract. +// It reports snapshot storage (slot count + size) and storage trie node +// statistics (node type breakdown and per-depth distribution). +func InspectContract(triedb database.NodeDatabase, db ethdb.Database, stateRoot common.Hash, address common.Address) error { + // Resolve account from the state trie. + accountHash := crypto.Keccak256Hash(address.Bytes()) + accountTrie, err := New(TrieID(stateRoot), triedb) + if err != nil { + return fmt.Errorf("failed to open account trie: %w", err) + } + accountRLP, err := accountTrie.Get(crypto.Keccak256(address.Bytes())) + if err != nil { + return fmt.Errorf("failed to read account: %w", err) + } + if accountRLP == nil { + return fmt.Errorf("account not found: %s", address) + } + var account types.StateAccount + if err := rlp.DecodeBytes(accountRLP, &account); err != nil { + return fmt.Errorf("failed to decode account: %w", err) + } + if account.Root == (common.Hash{}) || account.Root == types.EmptyRootHash { + return fmt.Errorf("account %s has no storage", address) + } + + // Look up account snapshot. + accountData, err := rawdb.ReadAccountSnapshot(db, accountHash) + if err != nil { + return fmt.Errorf("failed to read account snapshot: %w", err) + } + + // Run trie walk + snap iteration in parallel. + var ( + snapSlots atomic.Uint64 + snapSize atomic.Uint64 + g errgroup.Group + start = time.Now() + ) + + // Goroutine 1: Snapshot storage iteration. + g.Go(func() error { + prefix := append(rawdb.SnapshotStoragePrefix, accountHash.Bytes()...) + it := db.NewIterator(prefix, nil) + defer it.Release() + + for it.Next() { + if !bytes.HasPrefix(it.Key(), prefix) { + break + } + snapSlots.Add(1) + snapSize.Add(uint64(len(it.Key()) + len(it.Value()))) + } + return it.Error() + }) + + // Goroutine 2: Storage trie walk using the existing inspector. + storageStat := NewLevelStats() + g.Go(func() error { + owner := accountHash + storage, err := New(StorageTrieID(stateRoot, owner, account.Root), triedb) + if err != nil { + return fmt.Errorf("failed to open storage trie: %w", err) + } + in := &inspector{ + triedb: triedb, + root: stateRoot, + config: &InspectConfig{NoStorage: true}, + accountStat: NewLevelStats(), // unused, but needed by inspector + sem: semaphore.NewWeighted(inspectParallelism), + dumpBuf: bufio.NewWriter(io.Discard), + } + + in.recordRootSize(storage, account.Root, storageStat) + in.inspect(storage, storage.root, 0, []byte{}, storageStat) + + if err := in.getError(); err != nil { + return err + } + return nil + }) + + // Progress reporter. + done := make(chan struct{}) + go func() { + ticker := time.NewTicker(8 * time.Second) + defer ticker.Stop() + for { + select { + case <-ticker.C: + log.Info("Inspecting contract", + "snapSlots", snapSlots.Load(), + "trieNodes", storageStat.TotalNodes(), + "elapsed", common.PrettyDuration(time.Since(start))) + case <-done: + return + } + } + }() + + if err := g.Wait(); err != nil { + close(done) + return err + } + close(done) + + // Display results. + fmt.Printf("\n=== Contract Inspection: %s ===\n", address) + fmt.Printf("Account hash: %s\n\n", accountHash) + + if len(accountData) == 0 { + fmt.Println("Account snapshot: not found") + } else { + fmt.Printf("Account snapshot: %s\n", common.StorageSize(len(accountData))) + } + + fmt.Printf("Snapshot storage: %d slots (%s)\n", + snapSlots.Load(), common.StorageSize(snapSize.Load())) + + // Compute trie totals from LevelStats. + var trieTotal, trieSize uint64 + for i := 0; i < trieStatLevels; i++ { + short, full, value, size := storageStat.level[i].load() + trieTotal += short + full + value + trieSize += size + } + fmt.Printf("Storage trie: %d nodes (%s)\n", trieTotal, common.StorageSize(trieSize)) + + // Depth distribution table with node type columns. + fmt.Println("\nStorage Trie Depth Distribution:") + b := new(strings.Builder) + table := tablewriter.NewWriter(b) + table.SetHeader([]string{"Depth", "Short", "Full", "Value", "Nodes", "Size"}) + for i := 0; i < trieStatLevels; i++ { + short, full, value, size := storageStat.level[i].load() + total := short + full + value + if total == 0 && size == 0 { + continue + } + table.AppendBulk([][]string{{ + fmt.Sprint(i), + fmt.Sprint(short), + fmt.Sprint(full), + fmt.Sprint(value), + fmt.Sprint(total), + common.StorageSize(size).String(), + }}) + } + table.Render() + fmt.Print(b.String()) + + return nil +} + +func normalizeInspectConfig(config *InspectConfig) *InspectConfig { + if config == nil { + config = &InspectConfig{} + } + if config.TopN <= 0 { + config.TopN = inspectDefaultTopN + } + if config.DumpPath == "" { + config.DumpPath = "trie-dump.bin" + } + return config +} + +func (in *inspector) recordRootSize(trie *Trie, root common.Hash, stat *LevelStats) { + if root == (common.Hash{}) || root == types.EmptyRootHash { + return + } + blob := trie.prevalueTracer.Get(nil) + if len(blob) == 0 { + resolved, err := trie.reader.Node(nil, root) + if err != nil { + log.Error("Failed to read trie root for size accounting", "trie", trie.Hash(), "root", root, "err", err) + return + } + blob = resolved + } + stat.addSize(0, uint64(len(blob))) +} + +func (in *inspector) closeDump() error { + var ret error + if in.dumpBuf != nil { + if err := in.dumpBuf.Flush(); err != nil { + ret = errors.Join(ret, fmt.Errorf("failed to flush trie dump %s: %w", in.config.DumpPath, err)) + } + } + if in.dumpFile != nil { + if err := in.dumpFile.Close(); err != nil { + ret = errors.Join(ret, fmt.Errorf("failed to close trie dump %s: %w", in.config.DumpPath, err)) + } + } + return ret +} + +func (in *inspector) setError(err error) { + if err == nil { + return + } + in.errMu.Lock() + defer in.errMu.Unlock() + in.err = errors.Join(in.err, err) +} + +func (in *inspector) getError() error { + in.errMu.Lock() + defer in.errMu.Unlock() + return in.err +} + +func (in *inspector) hasError() bool { + return in.getError() != nil +} + +// trySpawn attempts to run fn in a new goroutine, bounded by the semaphore. +// If a slot is available the goroutine is started and tracked via wg; the +// caller must call wg.Wait() before reading any state written by fn. +// Returns false (and does not start a goroutine) when no slot is available. +func (in *inspector) trySpawn(wg *sync.WaitGroup, fn func()) bool { + if !in.sem.TryAcquire(1) { + return false + } + wg.Add(1) + go func() { + defer in.sem.Release(1) + defer wg.Done() + fn() + }() + return true +} + +func (in *inspector) writeDumpRecord(owner common.Hash, s *LevelStats) { + if in.hasError() { + return + } + var buf [inspectDumpRecordSize]byte + copy(buf[:32], owner[:]) + + off := 32 + for i := 0; i < trieStatLevels; i++ { + binary.LittleEndian.PutUint32(buf[off:], uint32(s.level[i].short.Load())) + off += 4 + binary.LittleEndian.PutUint32(buf[off:], uint32(s.level[i].full.Load())) + off += 4 + binary.LittleEndian.PutUint32(buf[off:], uint32(s.level[i].value.Load())) + off += 4 + binary.LittleEndian.PutUint64(buf[off:], s.level[i].size.Load()) + off += 8 + } + in.dumpMu.Lock() + _, err := in.dumpBuf.Write(buf[:]) + in.dumpMu.Unlock() + if err != nil { + in.setError(fmt.Errorf("failed writing trie dump record: %w", err)) + } + + // Increment counter for storage tries only (not for account trie) + if owner != (common.Hash{}) { + in.storageRecordsWritten.Add(1) + } +} + +// inspect walks the trie rooted at n and records node statistics into stat. +// It may spawn goroutines for subtrees, but always waits for them before +// returning — the caller sees a fully-populated stat when inspect returns. +func (in *inspector) inspect(trie *Trie, n node, height uint32, path []byte, stat *LevelStats) { + if n == nil { + return + } + + // wg tracks goroutines spawned by this call so we can wait for them + // before returning. This guarantees stat is complete when we return, + // which is critical for storage tries that write their dump record + // immediately after inspect returns. + var wg sync.WaitGroup + + // Four types of nodes can be encountered: + // - short: extend path with key, inspect single value. + // - full: inspect all 17 children, spin up new threads when possible. + // - hash: need to resolve node from disk, retry inspect on result. + // - value: if account, begin inspecting storage trie. + switch n := (n).(type) { + case *shortNode: + nextPath := slices.Concat(path, n.Key) + in.inspect(trie, n.Val, height+1, nextPath, stat) + case *fullNode: + for idx, child := range n.Children { + if child == nil { + continue + } + childPath := slices.Concat(path, []byte{byte(idx)}) + childNode := child + if in.trySpawn(&wg, func() { + in.inspect(trie, childNode, height+1, childPath, stat) + }) { + continue + } + in.inspect(trie, childNode, height+1, childPath, stat) + } + case hashNode: + blob, err := trie.reader.Node(path, common.BytesToHash(n)) + if err != nil { + log.Error("Failed to resolve HashNode", "err", err, "trie", trie.Hash(), "height", height+1, "path", path) + return + } + stat.addSize(height, uint64(len(blob))) + resolved := mustDecodeNode(n, blob) + in.inspect(trie, resolved, height, path, stat) + + // Return early here so this level isn't recorded twice. + return + case valueNode: + if !hasTerm(path) { + break + } + var account types.StateAccount + if err := rlp.Decode(bytes.NewReader(n), &account); err != nil { + // Not an account value. + break + } + if account.Root == (common.Hash{}) || account.Root == types.EmptyRootHash { + // Account is empty, nothing further to inspect. + break + } + + if !in.config.NoStorage { + owner := common.BytesToHash(hexToCompact(path)) + storage, err := New(StorageTrieID(in.root, owner, account.Root), in.triedb) + if err != nil { + log.Error("Failed to open account storage trie", "node", n, "error", err, "height", height, "path", common.Bytes2Hex(path)) + break + } + storageStat := NewLevelStats() + run := func() { + in.recordRootSize(storage, account.Root, storageStat) + in.inspect(storage, storage.root, 0, []byte{}, storageStat) + in.writeDumpRecord(owner, storageStat) + } + if in.trySpawn(&wg, run) { + break + } + run() + } + default: + panic(fmt.Sprintf("%T: invalid node: %v", n, n)) + } + + // Wait for all goroutines spawned at this level before recording + // the current node. This ensures the entire subtree is counted + // before this call returns. + wg.Wait() + + // Record stats for current height. + stat.add(n, height) +} + +// Summarize performs pass 2 over a trie dump and reports account stats, +// aggregate storage statistics, and top-N rankings. +func Summarize(dumpPath string, config *InspectConfig) error { + config = normalizeInspectConfig(config) + if dumpPath == "" { + dumpPath = config.DumpPath + } + if dumpPath == "" { + return errors.New("missing dump path") + } + file, err := os.Open(dumpPath) + if err != nil { + return fmt.Errorf("failed to open trie dump %s: %w", dumpPath, err) + } + defer file.Close() + + if info, err := file.Stat(); err == nil { + if info.Size()%inspectDumpRecordSize != 0 { + return fmt.Errorf("invalid trie dump size %d (not a multiple of %d)", info.Size(), inspectDumpRecordSize) + } + } + + depthTop := newStorageStatsTopN(config.TopN, compareStorageStatsByDepth) + totalTop := newStorageStatsTopN(config.TopN, compareStorageStatsByTotal) + valueTop := newStorageStatsTopN(config.TopN, compareStorageStatsByValue) + + summary := &inspectSummary{} + reader := bufio.NewReaderSize(file, 1<<20) + var buf [inspectDumpRecordSize]byte + + for { + _, err := io.ReadFull(reader, buf[:]) + if errors.Is(err, io.EOF) { + break + } + if errors.Is(err, io.ErrUnexpectedEOF) { + return fmt.Errorf("truncated trie dump %s", dumpPath) + } + if err != nil { + return fmt.Errorf("failed reading trie dump %s: %w", dumpPath, err) + } + + record := decodeDumpRecord(buf[:]) + snapshot := newStorageStats(record.Owner, record.Levels) + if record.Owner == (common.Hash{}) { + summary.Account = snapshot + continue + } + summary.StorageCount++ + summary.DepthHistogram[snapshot.MaxDepth]++ + for i := 0; i < trieStatLevels; i++ { + summary.StorageLevels[i].Short += record.Levels[i].Short + summary.StorageLevels[i].Full += record.Levels[i].Full + summary.StorageLevels[i].Value += record.Levels[i].Value + summary.StorageLevels[i].Size += record.Levels[i].Size + } + + depthTop.TryInsert(snapshot) + totalTop.TryInsert(snapshot) + valueTop.TryInsert(snapshot) + } + if summary.Account == nil { + return fmt.Errorf("dump file %s does not contain the account trie sentinel record", dumpPath) + } + for i := 0; i < trieStatLevels; i++ { + summary.StorageTotals.Short += summary.StorageLevels[i].Short + summary.StorageTotals.Full += summary.StorageLevels[i].Full + summary.StorageTotals.Value += summary.StorageLevels[i].Value + summary.StorageTotals.Size += summary.StorageLevels[i].Size + } + summary.TopByDepth = depthTop.Sorted() + summary.TopByTotalNodes = totalTop.Sorted() + summary.TopByValueNodes = valueTop.Sorted() + + if config.Path != "" { + return summary.writeJSON(config.Path) + } + summary.display() + return nil +} + +type dumpRecord struct { + Owner common.Hash + Levels [trieStatLevels]jsonLevel +} + +func decodeDumpRecord(raw []byte) dumpRecord { + var ( + record dumpRecord + off = 32 + ) + copy(record.Owner[:], raw[:32]) + for i := 0; i < trieStatLevels; i++ { + record.Levels[i] = jsonLevel{ + Short: uint64(binary.LittleEndian.Uint32(raw[off:])), + Full: uint64(binary.LittleEndian.Uint32(raw[off+4:])), + Value: uint64(binary.LittleEndian.Uint32(raw[off+8:])), + Size: binary.LittleEndian.Uint64(raw[off+12:]), + } + off += 20 + } + return record +} + +type storageStats struct { + Owner common.Hash + Levels [trieStatLevels]jsonLevel + Summary jsonLevel + MaxDepth int + TotalNodes uint64 + TotalSize uint64 +} + +func newStorageStats(owner common.Hash, levels [trieStatLevels]jsonLevel) *storageStats { + snapshot := &storageStats{Owner: owner, Levels: levels} + for i := 0; i < trieStatLevels; i++ { + level := levels[i] + if level.Short != 0 || level.Full != 0 || level.Value != 0 { + snapshot.MaxDepth = i + } + snapshot.Summary.Short += level.Short + snapshot.Summary.Full += level.Full + snapshot.Summary.Value += level.Value + snapshot.Summary.Size += level.Size + } + snapshot.TotalNodes = snapshot.Summary.Short + snapshot.Summary.Full + snapshot.Summary.Value + snapshot.TotalSize = snapshot.Summary.Size + return snapshot +} + +func trimLevels(levels [trieStatLevels]jsonLevel) []jsonLevel { + n := len(levels) + for n > 0 && levels[n-1] == (jsonLevel{}) { + n-- + } + return levels[:n] +} + +func (s *storageStats) MarshalJSON() ([]byte, error) { + type jsonStorageSnapshot struct { + Owner common.Hash `json:"Owner"` + MaxDepth int `json:"MaxDepth"` + TotalNodes uint64 `json:"TotalNodes"` + TotalSize uint64 `json:"TotalSize"` + ValueNodes uint64 `json:"ValueNodes"` + Levels []jsonLevel `json:"Levels"` + Summary jsonLevel `json:"Summary"` + } + return json.Marshal(jsonStorageSnapshot{ + Owner: s.Owner, + MaxDepth: s.MaxDepth, + TotalNodes: s.TotalNodes, + TotalSize: s.TotalSize, + ValueNodes: s.Summary.Value, + Levels: trimLevels(s.Levels), + Summary: s.Summary, + }) +} + +func (s *storageStats) toLevelStats() *LevelStats { + stats := NewLevelStats() + for i := 0; i < trieStatLevels; i++ { + stats.level[i].short.Store(s.Levels[i].Short) + stats.level[i].full.Store(s.Levels[i].Full) + stats.level[i].value.Store(s.Levels[i].Value) + stats.level[i].size.Store(s.Levels[i].Size) + } + return stats +} + +type storageStatsCompare func(a, b *storageStats) int + +type storageStatsTopN struct { + limit int + cmp storageStatsCompare + heap storageStatsHeap +} + +type storageStatsHeap struct { + items []*storageStats + cmp storageStatsCompare +} + +func (h storageStatsHeap) Len() int { return len(h.items) } + +func (h storageStatsHeap) Less(i, j int) bool { + // Keep the weakest entry at the root (min-heap semantics). + return h.cmp(h.items[i], h.items[j]) < 0 +} + +func (h storageStatsHeap) Swap(i, j int) { h.items[i], h.items[j] = h.items[j], h.items[i] } + +func (h *storageStatsHeap) Push(x any) { + h.items = append(h.items, x.(*storageStats)) +} + +func (h *storageStatsHeap) Pop() any { + item := h.items[len(h.items)-1] + h.items = h.items[:len(h.items)-1] + return item +} + +func newStorageStatsTopN(limit int, cmp storageStatsCompare) *storageStatsTopN { + h := storageStatsHeap{cmp: cmp} + heap.Init(&h) + return &storageStatsTopN{limit: limit, cmp: cmp, heap: h} +} + +func (t *storageStatsTopN) TryInsert(item *storageStats) { + if t.limit <= 0 { + return + } + if t.heap.Len() < t.limit { + heap.Push(&t.heap, item) + return + } + if t.cmp(item, t.heap.items[0]) <= 0 { + return + } + heap.Pop(&t.heap) + heap.Push(&t.heap, item) +} + +func (t *storageStatsTopN) Sorted() []*storageStats { + out := append([]*storageStats(nil), t.heap.items...) + sort.Slice(out, func(i, j int) bool { return t.cmp(out[i], out[j]) > 0 }) + return out +} + +func compareStorageStatsByDepth(a, b *storageStats) int { + return cmp.Or( + cmp.Compare(a.MaxDepth, b.MaxDepth), + cmp.Compare(a.TotalNodes, b.TotalNodes), + cmp.Compare(a.Summary.Value, b.Summary.Value), + bytes.Compare(a.Owner[:], b.Owner[:]), + ) +} + +func compareStorageStatsByTotal(a, b *storageStats) int { + return cmp.Or( + cmp.Compare(a.TotalNodes, b.TotalNodes), + cmp.Compare(a.MaxDepth, b.MaxDepth), + cmp.Compare(a.Summary.Value, b.Summary.Value), + bytes.Compare(a.Owner[:], b.Owner[:]), + ) +} + +func compareStorageStatsByValue(a, b *storageStats) int { + return cmp.Or( + cmp.Compare(a.Summary.Value, b.Summary.Value), + cmp.Compare(a.MaxDepth, b.MaxDepth), + cmp.Compare(a.TotalNodes, b.TotalNodes), + bytes.Compare(a.Owner[:], b.Owner[:]), + ) +} + +type inspectSummary struct { + Account *storageStats + StorageCount uint64 + StorageTotals jsonLevel + StorageLevels [trieStatLevels]jsonLevel + DepthHistogram [trieStatLevels]uint64 + TopByDepth []*storageStats + TopByTotalNodes []*storageStats + TopByValueNodes []*storageStats +} + +func (s *inspectSummary) display() { + s.displayCombinedDepthTable() + s.Account.toLevelStats().display("Accounts trie") + fmt.Println("Storage trie aggregate summary") + fmt.Printf("Total storage tries: %d\n", s.StorageCount) + totalNodes := s.StorageTotals.Short + s.StorageTotals.Full + s.StorageTotals.Value + fmt.Printf("Total nodes: %d\n", totalNodes) + fmt.Printf("Total size: %s\n", common.StorageSize(s.StorageTotals.Size)) + fmt.Printf(" Short nodes: %d\n", s.StorageTotals.Short) + fmt.Printf(" Full nodes: %d\n", s.StorageTotals.Full) + fmt.Printf(" Value nodes: %d\n", s.StorageTotals.Value) + + b := new(strings.Builder) + table := tablewriter.NewWriter(b) + table.SetHeader([]string{"Max Depth", "Storage Tries"}) + for i, count := range s.DepthHistogram { + table.AppendBulk([][]string{{fmt.Sprint(i), fmt.Sprint(count)}}) + } + table.Render() + fmt.Print(b.String()) + fmt.Println() + + s.displayTop("Top storage tries by max depth", s.TopByDepth) + s.displayTop("Top storage tries by total node count", s.TopByTotalNodes) + s.displayTop("Top storage tries by value (slot) count", s.TopByValueNodes) +} + +func (s *inspectSummary) displayCombinedDepthTable() { + accountTotal := s.Account.Summary.Short + s.Account.Summary.Full + s.Account.Summary.Value + storageTotal := s.StorageTotals.Short + s.StorageTotals.Full + s.StorageTotals.Value + accountTotalSize := s.Account.Summary.Size + storageTotalSize := s.StorageTotals.Size + + fmt.Println("Trie Depth Distribution") + fmt.Printf("Account Trie: %d nodes (%s)\n", accountTotal, common.StorageSize(accountTotalSize)) + fmt.Printf("Storage Tries: %d nodes (%s) across %d tries\n", storageTotal, common.StorageSize(storageTotalSize), s.StorageCount) + + b := new(strings.Builder) + table := tablewriter.NewWriter(b) + table.SetHeader([]string{"Depth", "Account Nodes", "Account Size", "Storage Nodes", "Storage Size"}) + for i := 0; i < trieStatLevels; i++ { + accountNodes := s.Account.Levels[i].Short + s.Account.Levels[i].Full + s.Account.Levels[i].Value + accountSize := s.Account.Levels[i].Size + storageNodes := s.StorageLevels[i].Short + s.StorageLevels[i].Full + s.StorageLevels[i].Value + storageSize := s.StorageLevels[i].Size + if accountNodes == 0 && storageNodes == 0 { + continue + } + table.AppendBulk([][]string{{ + fmt.Sprint(i), + fmt.Sprint(accountNodes), + common.StorageSize(accountSize).String(), + fmt.Sprint(storageNodes), + common.StorageSize(storageSize).String(), + }}) + } + table.Render() + fmt.Print(b.String()) + fmt.Println() +} + +func (s *inspectSummary) displayTop(title string, list []*storageStats) { + fmt.Println(title) + if len(list) == 0 { + fmt.Println("No storage tries found") + fmt.Println() + return + } + for i, item := range list { + fmt.Printf("%d: %s\n", i+1, item.Owner) + item.toLevelStats().display("storage trie") + } +} + +func (s *inspectSummary) MarshalJSON() ([]byte, error) { + type jsonAccountTrie struct { + Name string `json:"Name"` + Levels []jsonLevel `json:"Levels"` + Summary jsonLevel `json:"Summary"` + } + type jsonStorageSummary struct { + TotalStorageTries uint64 `json:"TotalStorageTries"` + Totals jsonLevel `json:"Totals"` + Levels []jsonLevel `json:"Levels"` + DepthHistogram [trieStatLevels]uint64 `json:"DepthHistogram"` + } + type jsonInspectSummary struct { + AccountTrie jsonAccountTrie `json:"AccountTrie"` + StorageSummary jsonStorageSummary `json:"StorageSummary"` + TopByDepth []*storageStats `json:"TopByDepth"` + TopByTotalNodes []*storageStats `json:"TopByTotalNodes"` + TopByValueNodes []*storageStats `json:"TopByValueNodes"` + } + return json.Marshal(jsonInspectSummary{ + AccountTrie: jsonAccountTrie{ + Name: "account trie", + Levels: trimLevels(s.Account.Levels), + Summary: s.Account.Summary, + }, + StorageSummary: jsonStorageSummary{ + TotalStorageTries: s.StorageCount, + Totals: s.StorageTotals, + Levels: trimLevels(s.StorageLevels), + DepthHistogram: s.DepthHistogram, + }, + TopByDepth: s.TopByDepth, + TopByTotalNodes: s.TopByTotalNodes, + TopByValueNodes: s.TopByValueNodes, + }) +} + +func (s *inspectSummary) writeJSON(path string) error { + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return err + } + defer file.Close() + + enc := json.NewEncoder(file) + enc.SetIndent("", " ") + return enc.Encode(s) +} + +// display will print a table displaying the trie's node statistics. +func (s *LevelStats) display(title string) { + // Shorten title if too long. + if len(title) > 32 { + title = title[0:8] + "..." + title[len(title)-8:] + } + + b := new(strings.Builder) + table := tablewriter.NewWriter(b) + table.SetHeader([]string{title, "Level", "Short Nodes", "Full Node", "Value Node"}) + + stat := &stat{} + for i := range s.level { + if s.level[i].empty() { + continue + } + short, full, value, _ := s.level[i].load() + table.AppendBulk([][]string{{"-", fmt.Sprint(i), fmt.Sprint(short), fmt.Sprint(full), fmt.Sprint(value)}}) + stat.add(&s.level[i]) + } + short, full, value, _ := stat.load() + table.SetFooter([]string{"Total", "", fmt.Sprint(short), fmt.Sprint(full), fmt.Sprint(value)}) + table.Render() + fmt.Print(b.String()) + fmt.Println("Max depth", s.MaxDepth()) + fmt.Println() +} + +type jsonLevel struct { + Short uint64 + Full uint64 + Value uint64 + Size uint64 +} diff --git a/trie/inspect_test.go b/trie/inspect_test.go new file mode 100644 index 0000000000..c07904c52d --- /dev/null +++ b/trie/inspect_test.go @@ -0,0 +1,256 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "encoding/json" + "math/rand" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/holiman/uint256" +) + +// TestInspect inspects a randomly generated account trie. It's useful for +// quickly verifying changes to the results display. +func TestInspect(t *testing.T) { + db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) + trie, err := NewStateTrie(TrieID(types.EmptyRootHash), db) + if err != nil { + t.Fatalf("failed to create state trie: %v", err) + } + // Create a realistic looking account trie with storage. + addresses, accounts := makeAccountsWithStorage(db, 11, true) + for i := 0; i < len(addresses); i++ { + trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) + } + // Insert the accounts into the trie and hash it + root, nodes := trie.Commit(true) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Commit(root) + + tempDir := t.TempDir() + dumpPath := filepath.Join(tempDir, "trie-dump.bin") + if err := Inspect(db, root, &InspectConfig{ + TopN: 1, + DumpPath: dumpPath, + Path: filepath.Join(tempDir, "trie-summary.json"), + }); err != nil { + t.Fatalf("inspect failed: %v", err) + } + reanalysisPath := filepath.Join(tempDir, "trie-summary-reanalysis.json") + if err := Summarize(dumpPath, &InspectConfig{ + TopN: 1, + Path: reanalysisPath, + }); err != nil { + t.Fatalf("summarize failed: %v", err) + } + + inspectSummaryPath := filepath.Join(tempDir, "trie-summary.json") + inspectOut := loadInspectJSON(t, inspectSummaryPath) + reanalysisOut := loadInspectJSON(t, reanalysisPath) + + if len(inspectOut.StorageSummary.Levels) == 0 { + t.Fatal("expected StorageSummary.Levels to be populated") + } + if inspectOut.AccountTrie.Summary.Size == 0 { + t.Fatal("expected account trie size summary to be populated") + } + if inspectOut.StorageSummary.Totals.Size == 0 { + t.Fatal("expected storage trie size summary to be populated") + } + if !reflect.DeepEqual(inspectOut.AccountTrie, reanalysisOut.AccountTrie) { + t.Fatal("account trie summary mismatch between inspect and summarize") + } + if !reflect.DeepEqual(inspectOut.StorageSummary, reanalysisOut.StorageSummary) { + t.Fatal("storage summary mismatch between inspect and summarize") + } + + assertStorageTotalsMatchLevels(t, inspectOut) + assertStorageTotalsMatchLevels(t, reanalysisOut) + assertAccountTotalsMatchLevels(t, inspectOut.AccountTrie) + assertAccountTotalsMatchLevels(t, reanalysisOut.AccountTrie) + + var histogramTotal uint64 + for _, count := range inspectOut.StorageSummary.DepthHistogram { + histogramTotal += count + } + if histogramTotal != inspectOut.StorageSummary.TotalStorageTries { + t.Fatalf("depth histogram total %d does not match total storage tries %d", histogramTotal, inspectOut.StorageSummary.TotalStorageTries) + } +} + +type inspectJSONOutput struct { + // Reuse storageStats for AccountTrie JSON to avoid introducing a parallel + // account summary test type. AccountTrie JSON includes Levels+Summary, + // which map directly; other storageStats fields remain zero-values. + AccountTrie storageStats `json:"AccountTrie"` + + StorageSummary struct { + TotalStorageTries uint64 `json:"TotalStorageTries"` + Totals jsonLevel `json:"Totals"` + Levels []jsonLevel `json:"Levels"` + DepthHistogram [trieStatLevels]uint64 `json:"DepthHistogram"` + } `json:"StorageSummary"` +} + +func loadInspectJSON(t *testing.T, path string) inspectJSONOutput { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("failed to read %s: %v", path, err) + } + var out inspectJSONOutput + if err := json.Unmarshal(raw, &out); err != nil { + t.Fatalf("failed to decode %s: %v", path, err) + } + return out +} + +func assertStorageTotalsMatchLevels(t *testing.T, out inspectJSONOutput) { + t.Helper() + var fromLevels jsonLevel + for _, level := range out.StorageSummary.Levels { + fromLevels.Short += level.Short + fromLevels.Full += level.Full + fromLevels.Value += level.Value + fromLevels.Size += level.Size + } + if fromLevels.Short != out.StorageSummary.Totals.Short || fromLevels.Full != out.StorageSummary.Totals.Full || fromLevels.Value != out.StorageSummary.Totals.Value || fromLevels.Size != out.StorageSummary.Totals.Size { + t.Fatalf("storage totals mismatch: levels=%+v totals=%+v", fromLevels, out.StorageSummary.Totals) + } +} + +func assertAccountTotalsMatchLevels(t *testing.T, account storageStats) { + t.Helper() + var fromLevels jsonLevel + for _, level := range account.Levels { + fromLevels.Short += level.Short + fromLevels.Full += level.Full + fromLevels.Value += level.Value + fromLevels.Size += level.Size + } + if fromLevels.Short != account.Summary.Short || fromLevels.Full != account.Summary.Full || fromLevels.Value != account.Summary.Value || fromLevels.Size != account.Summary.Size { + t.Fatalf("account totals mismatch: levels=%+v totals=%+v", fromLevels, account.Summary) + } +} + +// TestInspectContract tests the InspectContract function on a single account +// with storage and snapshot data. +func TestInspectContract(t *testing.T) { + diskdb := rawdb.NewMemoryDatabase() + db := newTestDatabase(diskdb, rawdb.HashScheme) + + // Create a contract address and its storage trie. + address := common.HexToAddress("0x1234567890abcdef1234567890abcdef12345678") + accountHash := crypto.Keccak256Hash(address.Bytes()) + + // Build a storage trie with some entries. + storageTrie := NewEmpty(db) + storageSlots := make(map[common.Hash][]byte) + for i := 0; i < 10; i++ { + k := crypto.Keccak256Hash([]byte{byte(i)}) + v := []byte{byte(i + 1)} + storageTrie.MustUpdate(k.Bytes(), v) + storageSlots[k] = v + } + storageRoot, storageNodes := storageTrie.Commit(true) + db.Update(storageRoot, types.EmptyRootHash, trienode.NewWithNodeSet(storageNodes)) + db.Commit(storageRoot) + + // Build the account trie with the contract account. + account := types.StateAccount{ + Nonce: 1, + Balance: uint256.NewInt(1000), + Root: storageRoot, + CodeHash: crypto.Keccak256(nil), + } + accountRLP, err := rlp.EncodeToBytes(&account) + if err != nil { + t.Fatalf("failed to encode account: %v", err) + } + + accountTrie := NewEmpty(db) + accountTrie.MustUpdate(crypto.Keccak256(address.Bytes()), accountRLP) + stateRoot, accountNodes := accountTrie.Commit(true) + db.Update(stateRoot, types.EmptyRootHash, trienode.NewWithNodeSet(accountNodes)) + db.Commit(stateRoot) + + // Write snapshot data for the account and its storage slots. + rawdb.WriteAccountSnapshot(diskdb, accountHash, accountRLP) + for k, v := range storageSlots { + rawdb.WriteStorageSnapshot(diskdb, accountHash, k, v) + } + + // InspectContract should succeed without error. + if err := InspectContract(db, diskdb, stateRoot, address); err != nil { + t.Fatalf("InspectContract failed: %v", err) + } +} + +func makeAccountsWithStorage(db *testDb, size int, storage bool) (addresses [][20]byte, accounts [][]byte) { + // Make the random benchmark deterministic + random := rand.New(rand.NewSource(0)) + + addresses = make([][20]byte, size) + for i := 0; i < len(addresses); i++ { + data := make([]byte, 20) + random.Read(data) + copy(addresses[i][:], data) + } + accounts = make([][]byte, len(addresses)) + for i := 0; i < len(accounts); i++ { + var ( + nonce = uint64(random.Int63()) + root = types.EmptyRootHash + code = crypto.Keccak256(nil) + ) + if storage { + trie := NewEmpty(db) + for range random.Uint32()%256 + 1 { // non-zero + k, v := make([]byte, 32), make([]byte, 32) + random.Read(k) + random.Read(v) + trie.MustUpdate(k, v) + } + var nodes *trienode.NodeSet + root, nodes = trie.Commit(true) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Commit(root) + } + numBytes := random.Uint32() % 33 // [0, 32] bytes + balanceBytes := make([]byte, numBytes) + random.Read(balanceBytes) + balance := new(uint256.Int).SetBytes(balanceBytes) + data, _ := rlp.EncodeToBytes(&types.StateAccount{ + Nonce: nonce, + Balance: balance, + Root: root, + CodeHash: code, + }) + accounts[i] = data + } + return addresses, accounts +} diff --git a/trie/levelstats.go b/trie/levelstats.go new file mode 100644 index 0000000000..c73d652146 --- /dev/null +++ b/trie/levelstats.go @@ -0,0 +1,135 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "fmt" + "sync/atomic" +) + +const trieStatLevels = 16 + +// LevelStats tracks the type and count of trie nodes at each level in a trie. +// +// Note: theoretically it is possible to have up to 64 trie levels, but +// LevelStats supports exactly 16 levels and panics on deeper paths. +type LevelStats struct { + level [trieStatLevels]stat +} + +// NewLevelStats creates an empty trie statistics collector. +func NewLevelStats() *LevelStats { + return &LevelStats{} +} + +// Copy returns a deep copy of the statistics. +func (s *LevelStats) Copy() *LevelStats { + cpy := NewLevelStats() + for i := range s.level { + cpy.level[i].short.Store(s.level[i].short.Load()) + cpy.level[i].full.Store(s.level[i].full.Load()) + cpy.level[i].value.Store(s.level[i].value.Load()) + cpy.level[i].size.Store(s.level[i].size.Load()) + } + return cpy +} + +// MaxDepth iterates each level and finds the deepest level with at least one +// trie node. +func (s *LevelStats) MaxDepth() int { + depth := 0 + for i := range s.level { + if s.level[i].short.Load() != 0 || s.level[i].full.Load() != 0 || s.level[i].value.Load() != 0 { + depth = i + } + } + return depth +} + +// TotalNodes returns the total number of nodes across all levels and types. +func (s *LevelStats) TotalNodes() uint64 { + var total uint64 + for i := range s.level { + total += s.level[i].short.Load() + s.level[i].full.Load() + s.level[i].value.Load() + } + return total +} + +// add increases the node count by one for the specified node type and depth. +func (s *LevelStats) add(n node, depth uint32) { + d := int(depth) + switch (n).(type) { + case *shortNode: + s.level[d].short.Add(1) + case *fullNode: + s.level[d].full.Add(1) + case valueNode: + s.level[d].value.Add(1) + default: + panic(fmt.Sprintf("%T: invalid node: %v", n, n)) + } +} + +// addSize increases the raw byte-size tally at the specified depth. +func (s *LevelStats) addSize(depth uint32, size uint64) { + s.level[depth].size.Add(size) +} + +// AddLeaf records a leaf depth. Witness collection reuses the value-node bucket +// for leaf accounting. It panics if the depth is outside [0, 15]. +func (s *LevelStats) AddLeaf(depth int) { + s.level[depth].value.Add(1) +} + +// LeafDepths returns leaf counts grouped by depth. +func (s *LevelStats) LeafDepths() [trieStatLevels]int64 { + var leaves [trieStatLevels]int64 + for i := range s.level { + leaves[i] = int64(s.level[i].value.Load()) + } + return leaves +} + +// stat is a specific level's count of each node type. +type stat struct { + short atomic.Uint64 + full atomic.Uint64 + value atomic.Uint64 + size atomic.Uint64 +} + +// empty is a helper that returns whether there are any trie nodes at the level. +func (s *stat) empty() bool { + if s.full.Load() == 0 && s.short.Load() == 0 && s.value.Load() == 0 && s.size.Load() == 0 { + return true + } + return false +} + +// load is a helper that loads each node type's value. +func (s *stat) load() (uint64, uint64, uint64, uint64) { + return s.short.Load(), s.full.Load(), s.value.Load(), s.size.Load() +} + +// add is a helper that adds two level's stats together. +func (s *stat) add(other *stat) *stat { + s.short.Add(other.short.Load()) + s.full.Add(other.full.Load()) + s.value.Add(other.value.Load()) + s.size.Add(other.size.Load()) + return s +} diff --git a/trie/levelstats_test.go b/trie/levelstats_test.go new file mode 100644 index 0000000000..90581eb1ab --- /dev/null +++ b/trie/levelstats_test.go @@ -0,0 +1,37 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import "testing" + +func TestLevelStatsAddLeafDepthBounds(t *testing.T) { + stats := NewLevelStats() + stats.AddLeaf(15) + + if got := stats.LeafDepths()[15]; got != 1 { + t.Fatalf("leaf count at depth 15 = %d, want 1", got) + } +} + +func TestLevelStatsAddLeafPanicsOnDepth16(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Fatal("expected panic for depth >= 16") + } + }() + NewLevelStats().AddLeaf(16) +} diff --git a/trie/node.go b/trie/node.go index 74fac4fd4e..7022116048 100644 --- a/trie/node.go +++ b/trie/node.go @@ -17,6 +17,7 @@ package trie import ( + "bytes" "fmt" "io" "strings" @@ -160,11 +161,14 @@ func decodeNodeUnsafe(hash, buf []byte) (node, error) { if err != nil { return nil, fmt.Errorf("decode error: %v", err) } - switch c, _ := rlp.CountValues(elems); c { - case 2: + c, err := rlp.CountValues(elems) + switch { + case err != nil: + return nil, fmt.Errorf("invalid node list: %v", err) + case c == 2: n, err := decodeShort(hash, elems) return n, wrapError(err, "short") - case 17: + case c == 17: n, err := decodeFull(hash, elems) return n, wrapError(err, "full") default: @@ -224,7 +228,7 @@ func decodeRef(buf []byte) (node, []byte, error) { case kind == rlp.List: // 'embedded' node reference. The encoding must be smaller // than a hash in order to be valid. - if size := len(buf) - len(rest); size > hashLen { + if size := len(buf) - len(rest); size >= hashLen { err := fmt.Errorf("oversized embedded node (size is %d bytes, want size < %d)", size, hashLen) return nil, buf, err } @@ -242,6 +246,74 @@ func decodeRef(buf []byte) (node, []byte, error) { } } +// decodeNodeElements parses the RLP encoding of a trie node and returns all the +// elements in raw byte format. +// +// For full node, it returns a slice of 17 elements; +// For short node, it returns a slice of 2 elements; +func decodeNodeElements(buf []byte) ([][]byte, error) { + if len(buf) == 0 { + return nil, io.ErrUnexpectedEOF + } + return rlp.SplitListValues(buf) +} + +// encodeNodeElements encodes the provided node elements into a rlp list. +func encodeNodeElements(elements [][]byte) ([]byte, error) { + if len(elements) != 2 && len(elements) != 17 { + return nil, fmt.Errorf("invalid number of elements: %d", len(elements)) + } + return rlp.MergeListValues(elements) +} + +// NodeDifference accepts two RLP-encoding nodes and figures out the difference +// between them. +// +// An error is returned if any of the provided blob is nil, or the type of nodes +// are different. +func NodeDifference(oldvalue []byte, newvalue []byte) (int, []int, [][]byte, error) { + oldElems, err := decodeNodeElements(oldvalue) + if err != nil { + return 0, nil, nil, err + } + newElems, err := decodeNodeElements(newvalue) + if err != nil { + return 0, nil, nil, err + } + if len(oldElems) != len(newElems) { + return 0, nil, nil, fmt.Errorf("different node type, old elements: %d, new elements: %d", len(oldElems), len(newElems)) + } + var ( + indices = make([]int, 0, len(oldElems)) + diff = make([][]byte, 0, len(oldElems)) + ) + for i := 0; i < len(oldElems); i++ { + if !bytes.Equal(oldElems[i], newElems[i]) { + indices = append(indices, i) + diff = append(diff, oldElems[i]) + } + } + return len(oldElems), indices, diff, nil +} + +// ReassembleNode accepts a RLP-encoding node along with a set of mutations, +// applying the modification diffs according to the indices and re-assemble. +func ReassembleNode(blob []byte, mutations [][][]byte, indices [][]int) ([]byte, error) { + if len(mutations) == 0 && len(indices) == 0 { + return blob, nil + } + elements, err := decodeNodeElements(blob) + if err != nil { + return nil, err + } + for i := 0; i < len(mutations); i++ { + for j, pos := range indices[i] { + elements[pos] = mutations[i][j] + } + } + return encodeNodeElements(elements) +} + // wraps a decoding error with information about the path to the // invalid child node (for debugging encoding issues). type decodeError struct { diff --git a/trie/node_test.go b/trie/node_test.go index 9b8b33748f..b4f427ca77 100644 --- a/trie/node_test.go +++ b/trie/node_test.go @@ -18,9 +18,13 @@ package trie import ( "bytes" + "math/rand" + "reflect" + "slices" "testing" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/testrand" "github.com/ethereum/go-ethereum/rlp" ) @@ -94,6 +98,286 @@ func TestDecodeFullNode(t *testing.T) { } } +func makeTestLeafNode(small bool) []byte { + l := leafNodeEncoder{} + l.Key = hexToCompact(keybytesToHex(testrand.Bytes(10))) + if small { + l.Val = testrand.Bytes(10) + } else { + l.Val = testrand.Bytes(32) + } + buf := rlp.NewEncoderBuffer(nil) + l.encode(buf) + return buf.ToBytes() +} + +func makeTestFullNode(small bool) []byte { + n := fullnodeEncoder{} + for i := 0; i < 16; i++ { + switch rand.Intn(3) { + case 0: + // write nil + case 1: + // write hash + n.Children[i] = testrand.Bytes(32) + case 2: + // write embedded node + n.Children[i] = makeTestLeafNode(small) + } + } + n.Children[16] = testrand.Bytes(32) // value + buf := rlp.NewEncoderBuffer(nil) + n.encode(buf) + return buf.ToBytes() +} + +func TestEncodeDecodeNodeElements(t *testing.T) { + var nodes [][]byte + nodes = append(nodes, makeTestFullNode(true)) + nodes = append(nodes, makeTestFullNode(false)) + nodes = append(nodes, makeTestLeafNode(true)) + nodes = append(nodes, makeTestLeafNode(false)) + + for _, blob := range nodes { + elements, err := decodeNodeElements(blob) + if err != nil { + t.Fatalf("Failed to decode node elements: %v", err) + } + enc, err := encodeNodeElements(elements) + if err != nil { + t.Fatalf("Failed to encode node elements: %v", err) + } + if !bytes.Equal(enc, blob) { + t.Fatalf("Unexpected encoded node element, want: %v, got: %v", blob, enc) + } + } +} + +func makeTestLeafNodePair() ([]byte, []byte, [][]byte, []int) { + var ( + na = leafNodeEncoder{} + nb = leafNodeEncoder{} + ) + key := keybytesToHex(testrand.Bytes(10)) + na.Key = hexToCompact(key) + nb.Key = hexToCompact(key) + + valA := testrand.Bytes(32) + valB := testrand.Bytes(32) + na.Val = valA + nb.Val = valB + + bufa, bufb := rlp.NewEncoderBuffer(nil), rlp.NewEncoderBuffer(nil) + na.encode(bufa) + nb.encode(bufb) + diff, _ := rlp.EncodeToBytes(valA) + return bufa.ToBytes(), bufb.ToBytes(), [][]byte{diff}, []int{1} +} + +func makeTestFullNodePair() ([]byte, []byte, [][]byte, []int) { + var ( + na = fullnodeEncoder{} + nb = fullnodeEncoder{} + indices []int + values [][]byte + ) + for i := 0; i < 16; i++ { + switch rand.Intn(3) { + case 0: + // write nil + case 1: + // write same + var child []byte + if rand.Intn(2) == 0 { + child = testrand.Bytes(32) // hashnode + } else { + child = makeTestLeafNode(true) // embedded node + } + na.Children[i] = child + nb.Children[i] = child + case 2: + // write different + var ( + va []byte + diff []byte + ) + rnd := rand.Intn(3) + if rnd == 0 { + va = testrand.Bytes(32) // hashnode + diff, _ = rlp.EncodeToBytes(va) + } else if rnd == 1 { + va = makeTestLeafNode(true) // embedded node + diff = va + } else { + va = nil + diff = rlp.EmptyString + } + vb := testrand.Bytes(32) // hashnode + na.Children[i] = va + nb.Children[i] = vb + + indices = append(indices, i) + values = append(values, diff) + } + } + na.Children[16] = nil + nb.Children[16] = nil + + bufa, bufb := rlp.NewEncoderBuffer(nil), rlp.NewEncoderBuffer(nil) + na.encode(bufa) + nb.encode(bufb) + return bufa.ToBytes(), bufb.ToBytes(), values, indices +} + +func TestNodeDifference(t *testing.T) { + type testsuite struct { + old []byte + new []byte + expErr bool + expIndices []int + expValues [][]byte + } + var tests = []testsuite{ + // Invalid node data + { + old: nil, new: nil, expErr: true, + }, + { + old: testrand.Bytes(32), new: nil, expErr: true, + }, + { + old: nil, new: testrand.Bytes(32), expErr: true, + }, + { + old: bytes.Repeat([]byte{0x1}, 32), new: bytes.Repeat([]byte{0x2}, 32), expErr: true, + }, + // Different node type + { + old: makeTestLeafNode(true), new: makeTestFullNode(true), expErr: true, + }, + } + for range 10 { + va, vb, elements, indices := makeTestLeafNodePair() + tests = append(tests, testsuite{ + old: va, + new: vb, + expErr: false, + expIndices: indices, + expValues: elements, + }) + } + for range 10 { + va, vb, elements, indices := makeTestFullNodePair() + tests = append(tests, testsuite{ + old: va, + new: vb, + expErr: false, + expIndices: indices, + expValues: elements, + }) + } + + for i, test := range tests { + _, indices, values, err := NodeDifference(test.old, test.new) + if test.expErr && err == nil { + t.Fatalf("Expect error, got nil %d", i) + } + if !test.expErr && err != nil { + t.Fatalf("Unexpect error, %v", err) + } + if err == nil { + if !slices.Equal(indices, test.expIndices) { + t.Fatalf("Unexpected indices, want: %v, got: %v", test.expIndices, indices) + } + if !slices.EqualFunc(values, test.expValues, bytes.Equal) { + t.Fatalf("Unexpected values, want: %v, got: %v", test.expValues, values) + } + } + } +} + +func TestReassembleFullNode(t *testing.T) { + var fn fullnodeEncoder + for i := 0; i < 16; i++ { + if rand.Intn(2) == 0 { + fn.Children[i] = testrand.Bytes(32) + } + } + buf := rlp.NewEncoderBuffer(nil) + fn.encode(buf) + enc := buf.ToBytes() + + // Generate a list of diffs + var ( + values [][][]byte + indices [][]int + ) + for i := 0; i < 10; i++ { + var ( + pos = make(map[int]struct{}) + poslist []int + valuelist [][]byte + ) + for j := 0; j < 3; j++ { + p := rand.Intn(16) + if _, ok := pos[p]; ok { + continue + } + pos[p] = struct{}{} + + nh := testrand.Bytes(32) + diff, _ := rlp.EncodeToBytes(nh) + poslist = append(poslist, p) + valuelist = append(valuelist, diff) + fn.Children[p] = nh + } + values = append(values, valuelist) + indices = append(indices, poslist) + } + reassembled, err := ReassembleNode(enc, values, indices) + if err != nil { + t.Fatalf("Failed to re-assemble full node %v", err) + } + buf2 := rlp.NewEncoderBuffer(nil) + fn.encode(buf2) + enc2 := buf2.ToBytes() + if !reflect.DeepEqual(enc2, reassembled) { + t.Fatalf("Unexpeted reassembled node") + } +} + +func TestReassembleShortNode(t *testing.T) { + var ln leafNodeEncoder + ln.Key = hexToCompact(keybytesToHex(testrand.Bytes(10))) + ln.Val = testrand.Bytes(10) + buf := rlp.NewEncoderBuffer(nil) + ln.encode(buf) + enc := buf.ToBytes() + + // Generate a list of diffs + var ( + values [][][]byte + indices [][]int + ) + for i := 0; i < 10; i++ { + val := testrand.Bytes(10) + ln.Val = val + diff, _ := rlp.EncodeToBytes(val) + values = append(values, [][]byte{diff}) + indices = append(indices, []int{1}) + } + reassembled, err := ReassembleNode(enc, values, indices) + if err != nil { + t.Fatalf("Failed to re-assemble full node %v", err) + } + buf2 := rlp.NewEncoderBuffer(nil) + ln.encode(buf2) + enc2 := buf2.ToBytes() + if !reflect.DeepEqual(enc2, reassembled) { + t.Fatalf("Unexpeted reassembled node") + } +} + // goos: darwin // goarch: arm64 // pkg: github.com/ethereum/go-ethereum/trie diff --git a/trie/proof.go b/trie/proof.go index 1a06ed5d5e..58075daf9b 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -454,7 +454,7 @@ func hasRightElement(node node, key []byte) bool { // // The firstKey is paired with firstProof, not necessarily the same as keys[0] // (unless firstProof is an existent proof). Similarly, lastKey and lastProof -// are paired. +// are paired. The firstKey should be less than or equal to all keys in the list. // // Expect the normal case, this function can also be used to verify the following // range proofs: @@ -520,9 +520,14 @@ func VerifyRangeProof(rootHash common.Hash, firstKey []byte, keys [][]byte, valu } return false, nil } - var lastKey = keys[len(keys)-1] + // Short circuit if the key of first element is greater than firstKey. + // A nil firstKey slice is equivalent to an empty slice. + if bytes.Compare(firstKey, keys[0]) > 0 { + return false, errors.New("unexpected key-value pairs preceding the requested range") + } // Special case, there is only one element and two edge keys are same. // In this case, we can't construct two edge paths. So handle it here. + var lastKey = keys[len(keys)-1] if len(keys) == 1 && bytes.Equal(firstKey, lastKey) { root, val, err := proofToPath(rootHash, nil, firstKey, proof, false) if err != nil { @@ -577,7 +582,9 @@ func VerifyRangeProof(rootHash common.Hash, firstKey []byte, keys [][]byte, valu tr.root = nil } for index, key := range keys { - tr.Update(key, values[index]) + if err := tr.Update(key, values[index]); err != nil { + return false, err + } } if tr.Hash() != rootHash { return false, fmt.Errorf("invalid proof, want hash %x, got %x", rootHash, tr.Hash()) diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 7c7bd184bf..1f150ede8c 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -134,9 +134,9 @@ func (t *StateTrie) GetAccountByHash(addrHash common.Hash) (*types.StateAccount, // PrefetchAccount attempts to resolve specific accounts from the database // to accelerate subsequent trie operations. func (t *StateTrie) PrefetchAccount(addresses []common.Address) error { - var keys [][]byte - for _, addr := range addresses { - keys = append(keys, crypto.Keccak256(addr.Bytes())) + keys := make([][]byte, len(addresses)) + for i, addr := range addresses { + keys[i] = crypto.Keccak256(addr.Bytes()) } return t.trie.Prefetch(keys) } @@ -157,9 +157,9 @@ func (t *StateTrie) GetStorage(_ common.Address, key []byte) ([]byte, error) { // PrefetchStorage attempts to resolve specific storage slots from the database // to accelerate subsequent trie operations. func (t *StateTrie) PrefetchStorage(_ common.Address, keys [][]byte) error { - var keylist [][]byte - for _, key := range keys { - keylist = append(keylist, crypto.Keccak256(key)) + keylist := make([][]byte, len(keys)) + for i, key := range keys { + keylist[i] = crypto.Keccak256(key) } return t.trie.Prefetch(keylist) } diff --git a/trie/transition.go b/trie/transitiontrie/transition.go similarity index 87% rename from trie/transition.go rename to trie/transitiontrie/transition.go index c6eecd3937..4c73022082 100644 --- a/trie/transition.go +++ b/trie/transitiontrie/transition.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package trie +package transitiontrie import ( "fmt" @@ -22,8 +22,9 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/bintrie" "github.com/ethereum/go-ethereum/trie/trienode" - "github.com/ethereum/go-verkle" ) // TransitionTrie is a trie that implements a façade design pattern, presenting @@ -31,13 +32,16 @@ import ( // first from the overlay trie, and falls back to the base trie if the key isn't // found. All writes go to the overlay trie. type TransitionTrie struct { - overlay *VerkleTrie - base *SecureTrie + overlay *bintrie.BinaryTrie + base *trie.SecureTrie storage bool } // NewTransitionTrie creates a new TransitionTrie. -func NewTransitionTrie(base *SecureTrie, overlay *VerkleTrie, st bool) *TransitionTrie { +// Note: base can be nil when using TransitionTrie as a wrapper for BinaryTrie +// to work around import cycles. This is a temporary hack that should be +// refactored in future PRs (see core/state/reader.go for details). +func NewTransitionTrie(base *trie.SecureTrie, overlay *bintrie.BinaryTrie, st bool) *TransitionTrie { return &TransitionTrie{ overlay: overlay, base: base, @@ -46,12 +50,12 @@ func NewTransitionTrie(base *SecureTrie, overlay *VerkleTrie, st bool) *Transiti } // Base returns the base trie. -func (t *TransitionTrie) Base() *SecureTrie { +func (t *TransitionTrie) Base() *trie.SecureTrie { return t.base } // Overlay returns the overlay trie. -func (t *TransitionTrie) Overlay() *VerkleTrie { +func (t *TransitionTrie) Overlay() *bintrie.BinaryTrie { return t.overlay } @@ -61,7 +65,10 @@ func (t *TransitionTrie) GetKey(key []byte) []byte { if key := t.overlay.GetKey(key); key != nil { return key } - return t.base.GetKey(key) + if t.base != nil { + return t.base.GetKey(key) + } + return nil } // GetStorage returns the value for key stored in the trie. The value bytes must @@ -74,8 +81,11 @@ func (t *TransitionTrie) GetStorage(addr common.Address, key []byte) ([]byte, er if len(val) != 0 { return val, nil } - // TODO also insert value into overlay - return t.base.GetStorage(addr, key) + if t.base != nil { + // TODO also insert value into overlay + return t.base.GetStorage(addr, key) + } + return nil, nil } // PrefetchStorage attempts to resolve specific storage slots from the database @@ -102,7 +112,10 @@ func (t *TransitionTrie) GetAccount(address common.Address) (*types.StateAccount if data != nil { return data, nil } - return t.base.GetAccount(address) + if t.base != nil { + return t.base.GetAccount(address) + } + return nil, nil } // PrefetchAccount attempts to resolve specific accounts from the database @@ -174,7 +187,7 @@ func (t *TransitionTrie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSe // NodeIterator returns an iterator that returns nodes of the trie. Iteration // starts at the key after the given start key. -func (t *TransitionTrie) NodeIterator(startKey []byte) (NodeIterator, error) { +func (t *TransitionTrie) NodeIterator(startKey []byte) (trie.NodeIterator, error) { panic("not implemented") // TODO: Implement } @@ -197,14 +210,10 @@ func (t *TransitionTrie) IsVerkle() bool { // UpdateStem updates a group of values, given the stem they are using. If // a value already exists, it is overwritten. +// TODO: This is Verkle-specific and requires access to private fields. +// Not currently used in the codebase. func (t *TransitionTrie) UpdateStem(key []byte, values [][]byte) error { - trie := t.overlay - switch root := trie.root.(type) { - case *verkle.InternalNode: - return root.InsertValuesAtStem(key, values, t.overlay.nodeResolver) - default: - panic("invalid root type") - } + panic("UpdateStem is not implemented for TransitionTrie") } // Copy creates a deep copy of the transition trie. diff --git a/trie/trie_test.go b/trie/trie_test.go index f6e1a014fe..ff10ca862f 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -880,6 +880,7 @@ func (b *spongeBatch) ValueSize() int { return 100 } func (b *spongeBatch) Write() error { return nil } func (b *spongeBatch) Reset() {} func (b *spongeBatch) Replay(w ethdb.KeyValueWriter) error { return nil } +func (b *spongeBatch) Close() {} // TestCommitSequence tests that the trie.Commit operation writes the elements // of the trie in the expected order. diff --git a/trie/utils/verkle.go b/trie/utils/verkle.go deleted file mode 100644 index dea210c046..0000000000 --- a/trie/utils/verkle.go +++ /dev/null @@ -1,299 +0,0 @@ -// Copyright 2023 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package utils - -import ( - "encoding/binary" - "sync" - - "github.com/crate-crypto/go-ipa/bandersnatch/fr" - "github.com/ethereum/go-ethereum/common/lru" - "github.com/ethereum/go-ethereum/metrics" - "github.com/ethereum/go-verkle" - "github.com/holiman/uint256" -) - -const ( - BasicDataLeafKey = 0 - CodeHashLeafKey = 1 - - BasicDataVersionOffset = 0 - BasicDataCodeSizeOffset = 5 - BasicDataNonceOffset = 8 - BasicDataBalanceOffset = 16 -) - -var ( - zero = uint256.NewInt(0) - verkleNodeWidthLog2 = 8 - headerStorageOffset = uint256.NewInt(64) - codeOffset = uint256.NewInt(128) - verkleNodeWidth = uint256.NewInt(256) - codeStorageDelta = uint256.NewInt(0).Sub(codeOffset, headerStorageOffset) - mainStorageOffsetLshVerkleNodeWidth = new(uint256.Int).Lsh(uint256.NewInt(1), 248-uint(verkleNodeWidthLog2)) - - index0Point *verkle.Point // pre-computed commitment of polynomial [2+256*64] - - // cacheHitGauge is the metric to track how many cache hit occurred. - cacheHitGauge = metrics.NewRegisteredGauge("trie/verkle/cache/hit", nil) - - // cacheMissGauge is the metric to track how many cache miss occurred. - cacheMissGauge = metrics.NewRegisteredGauge("trie/verkle/cache/miss", nil) -) - -func init() { - // The byte array is the Marshalled output of the point computed as such: - // - // var ( - // config = verkle.GetConfig() - // fr verkle.Fr - // ) - // verkle.FromLEBytes(&fr, []byte{2, 64}) - // point := config.CommitToPoly([]verkle.Fr{fr}, 1) - index0Point = new(verkle.Point) - err := index0Point.SetBytes([]byte{34, 25, 109, 242, 193, 5, 144, 224, 76, 52, 189, 92, 197, 126, 9, 145, 27, 152, 199, 130, 165, 3, 210, 27, 193, 131, 142, 28, 110, 26, 16, 191}) - if err != nil { - panic(err) - } -} - -// PointCache is the LRU cache for storing evaluated address commitment. -type PointCache struct { - lru lru.BasicLRU[string, *verkle.Point] - lock sync.RWMutex -} - -// NewPointCache returns the cache with specified size. -func NewPointCache(maxItems int) *PointCache { - return &PointCache{ - lru: lru.NewBasicLRU[string, *verkle.Point](maxItems), - } -} - -// Get returns the cached commitment for the specified address, or computing -// it on the flight. -func (c *PointCache) Get(addr []byte) *verkle.Point { - c.lock.Lock() - defer c.lock.Unlock() - - p, ok := c.lru.Get(string(addr)) - if ok { - cacheHitGauge.Inc(1) - return p - } - cacheMissGauge.Inc(1) - p = evaluateAddressPoint(addr) - c.lru.Add(string(addr), p) - return p -} - -// GetStem returns the first 31 bytes of the tree key as the tree stem. It only -// works for the account metadata whose treeIndex is 0. -func (c *PointCache) GetStem(addr []byte) []byte { - p := c.Get(addr) - return pointToHash(p, 0)[:31] -} - -// GetTreeKey performs both the work of the spec's get_tree_key function, and that -// of pedersen_hash: it builds the polynomial in pedersen_hash without having to -// create a mostly zero-filled buffer and "type cast" it to a 128-long 16-byte -// array. Since at most the first 5 coefficients of the polynomial will be non-zero, -// these 5 coefficients are created directly. -func GetTreeKey(address []byte, treeIndex *uint256.Int, subIndex byte) []byte { - if len(address) < 32 { - var aligned [32]byte - address = append(aligned[:32-len(address)], address...) - } - // poly = [2+256*64, address_le_low, address_le_high, tree_index_le_low, tree_index_le_high] - var poly [5]fr.Element - - // 32-byte address, interpreted as two little endian - // 16-byte numbers. - verkle.FromLEBytes(&poly[1], address[:16]) - verkle.FromLEBytes(&poly[2], address[16:]) - - // treeIndex must be interpreted as a 32-byte aligned little-endian integer. - // e.g: if treeIndex is 0xAABBCC, we need the byte representation to be 0xCCBBAA00...00. - // poly[3] = LE({CC,BB,AA,00...0}) (16 bytes), poly[4]=LE({00,00,...}) (16 bytes). - // - // To avoid unnecessary endianness conversions for go-ipa, we do some trick: - // - poly[3]'s byte representation is the same as the *top* 16 bytes (trieIndexBytes[16:]) of - // 32-byte aligned big-endian representation (BE({00,...,AA,BB,CC})). - // - poly[4]'s byte representation is the same as the *low* 16 bytes (trieIndexBytes[:16]) of - // the 32-byte aligned big-endian representation (BE({00,00,...}). - trieIndexBytes := treeIndex.Bytes32() - verkle.FromBytes(&poly[3], trieIndexBytes[16:]) - verkle.FromBytes(&poly[4], trieIndexBytes[:16]) - - cfg := verkle.GetConfig() - ret := cfg.CommitToPoly(poly[:], 0) - - // add a constant point corresponding to poly[0]=[2+256*64]. - ret.Add(ret, index0Point) - - return pointToHash(ret, subIndex) -} - -// GetTreeKeyWithEvaluatedAddress is basically identical to GetTreeKey, the only -// difference is a part of polynomial is already evaluated. -// -// Specifically, poly = [2+256*64, address_le_low, address_le_high] is already -// evaluated. -func GetTreeKeyWithEvaluatedAddress(evaluated *verkle.Point, treeIndex *uint256.Int, subIndex byte) []byte { - var poly [5]fr.Element - - // little-endian, 32-byte aligned treeIndex - var index [32]byte - for i := 0; i < len(treeIndex); i++ { - binary.LittleEndian.PutUint64(index[i*8:(i+1)*8], treeIndex[i]) - } - verkle.FromLEBytes(&poly[3], index[:16]) - verkle.FromLEBytes(&poly[4], index[16:]) - - cfg := verkle.GetConfig() - ret := cfg.CommitToPoly(poly[:], 0) - - // add the pre-evaluated address - ret.Add(ret, evaluated) - - return pointToHash(ret, subIndex) -} - -// BasicDataKey returns the verkle tree key of the basic data field for -// the specified account. -func BasicDataKey(address []byte) []byte { - return GetTreeKey(address, zero, BasicDataLeafKey) -} - -// CodeHashKey returns the verkle tree key of the code hash field for -// the specified account. -func CodeHashKey(address []byte) []byte { - return GetTreeKey(address, zero, CodeHashLeafKey) -} - -func codeChunkIndex(chunk *uint256.Int) (*uint256.Int, byte) { - var ( - chunkOffset = new(uint256.Int).Add(codeOffset, chunk) - treeIndex, subIndexMod = new(uint256.Int).DivMod(chunkOffset, verkleNodeWidth, new(uint256.Int)) - ) - return treeIndex, byte(subIndexMod.Uint64()) -} - -// CodeChunkKey returns the verkle tree key of the code chunk for the -// specified account. -func CodeChunkKey(address []byte, chunk *uint256.Int) []byte { - treeIndex, subIndex := codeChunkIndex(chunk) - return GetTreeKey(address, treeIndex, subIndex) -} - -func StorageIndex(storageKey []byte) (*uint256.Int, byte) { - // If the storage slot is in the header, we need to add the header offset. - var key uint256.Int - key.SetBytes(storageKey) - if key.Cmp(codeStorageDelta) < 0 { - // This addition is always safe; it can't ever overflow since pos. - -package utils - -import ( - "bytes" - "testing" - - "github.com/ethereum/go-verkle" - "github.com/holiman/uint256" -) - -func TestTreeKey(t *testing.T) { - var ( - address = []byte{0x01} - addressEval = evaluateAddressPoint(address) - smallIndex = uint256.NewInt(1) - largeIndex = uint256.NewInt(10000) - smallStorage = []byte{0x1} - largeStorage = bytes.Repeat([]byte{0xff}, 16) - ) - if !bytes.Equal(BasicDataKey(address), BasicDataKeyWithEvaluatedAddress(addressEval)) { - t.Fatal("Unmatched basic data key") - } - if !bytes.Equal(CodeHashKey(address), CodeHashKeyWithEvaluatedAddress(addressEval)) { - t.Fatal("Unmatched code hash key") - } - if !bytes.Equal(CodeChunkKey(address, smallIndex), CodeChunkKeyWithEvaluatedAddress(addressEval, smallIndex)) { - t.Fatal("Unmatched code chunk key") - } - if !bytes.Equal(CodeChunkKey(address, largeIndex), CodeChunkKeyWithEvaluatedAddress(addressEval, largeIndex)) { - t.Fatal("Unmatched code chunk key") - } - if !bytes.Equal(StorageSlotKey(address, smallStorage), StorageSlotKeyWithEvaluatedAddress(addressEval, smallStorage)) { - t.Fatal("Unmatched storage slot key") - } - if !bytes.Equal(StorageSlotKey(address, largeStorage), StorageSlotKeyWithEvaluatedAddress(addressEval, largeStorage)) { - t.Fatal("Unmatched storage slot key") - } -} - -// goos: darwin -// goarch: amd64 -// pkg: github.com/ethereum/go-ethereum/trie/utils -// cpu: VirtualApple @ 2.50GHz -// BenchmarkTreeKey -// BenchmarkTreeKey-8 398731 2961 ns/op 32 B/op 1 allocs/op -func BenchmarkTreeKey(b *testing.B) { - // Initialize the IPA settings which can be pretty expensive. - verkle.GetConfig() - - b.ReportAllocs() - b.ResetTimer() - - for i := 0; i < b.N; i++ { - BasicDataKey([]byte{0x01}) - } -} - -// goos: darwin -// goarch: amd64 -// pkg: github.com/ethereum/go-ethereum/trie/utils -// cpu: VirtualApple @ 2.50GHz -// BenchmarkTreeKeyWithEvaluation -// BenchmarkTreeKeyWithEvaluation-8 513855 2324 ns/op 32 B/op 1 allocs/op -func BenchmarkTreeKeyWithEvaluation(b *testing.B) { - // Initialize the IPA settings which can be pretty expensive. - verkle.GetConfig() - - addr := []byte{0x01} - eval := evaluateAddressPoint(addr) - - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - BasicDataKeyWithEvaluatedAddress(eval) - } -} - -// goos: darwin -// goarch: amd64 -// pkg: github.com/ethereum/go-ethereum/trie/utils -// cpu: VirtualApple @ 2.50GHz -// BenchmarkStorageKey -// BenchmarkStorageKey-8 230516 4584 ns/op 96 B/op 3 allocs/op -func BenchmarkStorageKey(b *testing.B) { - // Initialize the IPA settings which can be pretty expensive. - verkle.GetConfig() - - b.ReportAllocs() - b.ResetTimer() - - for i := 0; i < b.N; i++ { - StorageSlotKey([]byte{0x01}, bytes.Repeat([]byte{0xff}, 32)) - } -} - -// goos: darwin -// goarch: amd64 -// pkg: github.com/ethereum/go-ethereum/trie/utils -// cpu: VirtualApple @ 2.50GHz -// BenchmarkStorageKeyWithEvaluation -// BenchmarkStorageKeyWithEvaluation-8 320125 3753 ns/op 96 B/op 3 allocs/op -func BenchmarkStorageKeyWithEvaluation(b *testing.B) { - // Initialize the IPA settings which can be pretty expensive. - verkle.GetConfig() - - addr := []byte{0x01} - eval := evaluateAddressPoint(addr) - - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - StorageSlotKeyWithEvaluatedAddress(eval, bytes.Repeat([]byte{0xff}, 32)) - } -} diff --git a/trie/verkle.go b/trie/verkle.go deleted file mode 100644 index 186ac1f642..0000000000 --- a/trie/verkle.go +++ /dev/null @@ -1,457 +0,0 @@ -// Copyright 2023 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package trie - -import ( - "bytes" - "encoding/binary" - "errors" - "fmt" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/trie/trienode" - "github.com/ethereum/go-ethereum/trie/utils" - "github.com/ethereum/go-ethereum/triedb/database" - "github.com/ethereum/go-verkle" - "github.com/holiman/uint256" -) - -var ( - errInvalidRootType = errors.New("invalid node type for root") -) - -// VerkleTrie is a wrapper around VerkleNode that implements the trie.Trie -// interface so that Verkle trees can be reused verbatim. -type VerkleTrie struct { - root verkle.VerkleNode - cache *utils.PointCache - reader *Reader - tracer *PrevalueTracer -} - -// NewVerkleTrie constructs a verkle tree based on the specified root hash. -func NewVerkleTrie(root common.Hash, db database.NodeDatabase, cache *utils.PointCache) (*VerkleTrie, error) { - reader, err := NewReader(root, common.Hash{}, db) - if err != nil { - return nil, err - } - t := &VerkleTrie{ - root: verkle.New(), - cache: cache, - reader: reader, - tracer: NewPrevalueTracer(), - } - // Parse the root verkle node if it's not empty. - if root != types.EmptyVerkleHash && root != types.EmptyRootHash { - blob, err := t.nodeResolver(nil) - if err != nil { - return nil, err - } - node, err := verkle.ParseNode(blob, 0) - if err != nil { - return nil, err - } - t.root = node - } - return t, nil -} - -// GetKey returns the sha3 preimage of a hashed key that was previously used -// to store a value. -func (t *VerkleTrie) GetKey(key []byte) []byte { - return key -} - -// GetAccount implements state.Trie, retrieving the account with the specified -// account address. If the specified account is not in the verkle tree, nil will -// be returned. If the tree is corrupted, an error will be returned. -func (t *VerkleTrie) GetAccount(addr common.Address) (*types.StateAccount, error) { - var ( - acc = &types.StateAccount{} - values [][]byte - err error - ) - switch n := t.root.(type) { - case *verkle.InternalNode: - values, err = n.GetValuesAtStem(t.cache.GetStem(addr[:]), t.nodeResolver) - if err != nil { - return nil, fmt.Errorf("GetAccount (%x) error: %v", addr, err) - } - default: - return nil, errInvalidRootType - } - if values == nil { - return nil, nil - } - basicData := values[utils.BasicDataLeafKey] - acc.Nonce = binary.BigEndian.Uint64(basicData[utils.BasicDataNonceOffset:]) - acc.Balance = new(uint256.Int).SetBytes(basicData[utils.BasicDataBalanceOffset : utils.BasicDataBalanceOffset+16]) - acc.CodeHash = values[utils.CodeHashLeafKey] - - // TODO account.Root is leave as empty. How should we handle the legacy account? - return acc, nil -} - -// PrefetchAccount attempts to resolve specific accounts from the database -// to accelerate subsequent trie operations. -func (t *VerkleTrie) PrefetchAccount(addresses []common.Address) error { - for _, addr := range addresses { - if _, err := t.GetAccount(addr); err != nil { - return err - } - } - return nil -} - -// GetStorage implements state.Trie, retrieving the storage slot with the specified -// account address and storage key. If the specified slot is not in the verkle tree, -// nil will be returned. If the tree is corrupted, an error will be returned. -func (t *VerkleTrie) GetStorage(addr common.Address, key []byte) ([]byte, error) { - k := utils.StorageSlotKeyWithEvaluatedAddress(t.cache.Get(addr.Bytes()), key) - val, err := t.root.Get(k, t.nodeResolver) - if err != nil { - return nil, err - } - return common.TrimLeftZeroes(val), nil -} - -// PrefetchStorage attempts to resolve specific storage slots from the database -// to accelerate subsequent trie operations. -func (t *VerkleTrie) PrefetchStorage(addr common.Address, keys [][]byte) error { - for _, key := range keys { - if _, err := t.GetStorage(addr, key); err != nil { - return err - } - } - return nil -} - -// UpdateAccount implements state.Trie, writing the provided account into the tree. -// If the tree is corrupted, an error will be returned. -func (t *VerkleTrie) UpdateAccount(addr common.Address, acc *types.StateAccount, codeLen int) error { - var ( - err error - basicData [32]byte - values = make([][]byte, verkle.NodeWidth) - stem = t.cache.GetStem(addr[:]) - ) - - // Code size is encoded in BasicData as a 3-byte big-endian integer. Spare bytes are present - // before the code size to support bigger integers in the future. PutUint32(...) requires - // 4 bytes, so we need to shift the offset 1 byte to the left. - binary.BigEndian.PutUint32(basicData[utils.BasicDataCodeSizeOffset-1:], uint32(codeLen)) - binary.BigEndian.PutUint64(basicData[utils.BasicDataNonceOffset:], acc.Nonce) - if acc.Balance.ByteLen() > 16 { - panic("balance too large") - } - acc.Balance.WriteToSlice(basicData[utils.BasicDataBalanceOffset : utils.BasicDataBalanceOffset+16]) - values[utils.BasicDataLeafKey] = basicData[:] - values[utils.CodeHashLeafKey] = acc.CodeHash[:] - - switch root := t.root.(type) { - case *verkle.InternalNode: - err = root.InsertValuesAtStem(stem, values, t.nodeResolver) - default: - return errInvalidRootType - } - if err != nil { - return fmt.Errorf("UpdateAccount (%x) error: %v", addr, err) - } - - return nil -} - -// UpdateStorage implements state.Trie, writing the provided storage slot into -// the tree. If the tree is corrupted, an error will be returned. -func (t *VerkleTrie) UpdateStorage(address common.Address, key, value []byte) error { - // Left padding the slot value to 32 bytes. - var v [32]byte - if len(value) >= 32 { - copy(v[:], value[:32]) - } else { - copy(v[32-len(value):], value[:]) - } - k := utils.StorageSlotKeyWithEvaluatedAddress(t.cache.Get(address.Bytes()), key) - return t.root.Insert(k, v[:], t.nodeResolver) -} - -// DeleteAccount leaves the account untouched, as no account deletion can happen -// in verkle. -// There is a special corner case, in which an account that is prefunded, CREATE2-d -// and then SELFDESTRUCT-d should see its funds drained. EIP161 says that account -// should be removed, but this is verboten by the verkle spec. This contains a -// workaround in which the method checks for this corner case, and if so, overwrites -// the balance with 0. This will be removed once the spec has been clarified. -func (t *VerkleTrie) DeleteAccount(addr common.Address) error { - k := utils.BasicDataKeyWithEvaluatedAddress(t.cache.Get(addr.Bytes())) - values, err := t.root.(*verkle.InternalNode).GetValuesAtStem(k, t.nodeResolver) - if err != nil { - return fmt.Errorf("Error getting data at %x in delete: %w", k, err) - } - var prefunded bool - for i, v := range values { - switch i { - case 0: - prefunded = len(v) == 32 - case 1: - prefunded = len(v) == 32 && bytes.Equal(v, types.EmptyCodeHash[:]) - default: - prefunded = v == nil - } - if !prefunded { - break - } - } - if prefunded { - t.root.Insert(k, common.Hash{}.Bytes(), t.nodeResolver) - } - return nil -} - -// RollBackAccount removes the account info + code from the tree, unlike DeleteAccount -// that will overwrite it with 0s. The first 64 storage slots are also removed. -func (t *VerkleTrie) RollBackAccount(addr common.Address) error { - var ( - evaluatedAddr = t.cache.Get(addr.Bytes()) - basicDataKey = utils.BasicDataKeyWithEvaluatedAddress(evaluatedAddr) - ) - basicDataBytes, err := t.root.Get(basicDataKey, t.nodeResolver) - if err != nil { - return fmt.Errorf("rollback: error finding code size: %w", err) - } - if len(basicDataBytes) == 0 { - return errors.New("rollback: basic data is not existent") - } - // The code size is encoded in BasicData as a 3-byte big-endian integer. Spare bytes are present - // before the code size to support bigger integers in the future. - // LittleEndian.Uint32(...) expects 4-bytes, so we need to shift the offset 1-byte to the left. - codeSize := binary.BigEndian.Uint32(basicDataBytes[utils.BasicDataCodeSizeOffset-1:]) - - // Delete the account header + first 64 slots + first 128 code chunks - _, err = t.root.(*verkle.InternalNode).DeleteAtStem(basicDataKey[:31], t.nodeResolver) - if err != nil { - return fmt.Errorf("error rolling back account header: %w", err) - } - - // Delete all further code - for i, chunknr := uint64(31*128), uint64(128); i < uint64(codeSize); i, chunknr = i+31*256, chunknr+256 { - // evaluate group key at the start of a new group - offset := uint256.NewInt(chunknr) - key := utils.CodeChunkKeyWithEvaluatedAddress(evaluatedAddr, offset) - - if _, err = t.root.(*verkle.InternalNode).DeleteAtStem(key[:], t.nodeResolver); err != nil { - return fmt.Errorf("error deleting code chunk stem (addr=%x, offset=%d) error: %w", addr[:], offset, err) - } - } - return nil -} - -// DeleteStorage implements state.Trie, deleting the specified storage slot from -// the trie. If the storage slot was not existent in the trie, no error will be -// returned. If the trie is corrupted, an error will be returned. -func (t *VerkleTrie) DeleteStorage(addr common.Address, key []byte) error { - var zero [32]byte - k := utils.StorageSlotKeyWithEvaluatedAddress(t.cache.Get(addr.Bytes()), key) - return t.root.Insert(k, zero[:], t.nodeResolver) -} - -// Hash returns the root hash of the tree. It does not write to the database and -// can be used even if the tree doesn't have one. -func (t *VerkleTrie) Hash() common.Hash { - return t.root.Commit().Bytes() -} - -// Commit writes all nodes to the tree's memory database. -func (t *VerkleTrie) Commit(_ bool) (common.Hash, *trienode.NodeSet) { - root := t.root.(*verkle.InternalNode) - nodes, err := root.BatchSerialize() - if err != nil { - // Error return from this function indicates error in the code logic - // of BatchSerialize, and we fail catastrophically if this is the case. - panic(fmt.Errorf("BatchSerialize failed: %v", err)) - } - nodeset := trienode.NewNodeSet(common.Hash{}) - for _, node := range nodes { - // Hash parameter is not used in pathdb - nodeset.AddNode(node.Path, trienode.NewNodeWithPrev(common.Hash{}, node.SerializedBytes, t.tracer.Get(node.Path))) - } - // Serialize root commitment form - return t.Hash(), nodeset -} - -// NodeIterator implements state.Trie, returning an iterator that returns -// nodes of the trie. Iteration starts at the key after the given start key. -// -// TODO(gballet, rjl493456442) implement it. -func (t *VerkleTrie) NodeIterator(startKey []byte) (NodeIterator, error) { - panic("not implemented") -} - -// Prove implements state.Trie, constructing a Merkle proof for key. The result -// contains all encoded nodes on the path to the value at key. The value itself -// is also included in the last node and can be retrieved by verifying the proof. -// -// If the trie does not contain a value for key, the returned proof contains all -// nodes of the longest existing prefix of the key (at least the root), ending -// with the node that proves the absence of the key. -// -// TODO(gballet, rjl493456442) implement it. -func (t *VerkleTrie) Prove(key []byte, proofDb ethdb.KeyValueWriter) error { - panic("not implemented") -} - -// Copy returns a deep-copied verkle tree. -func (t *VerkleTrie) Copy() *VerkleTrie { - return &VerkleTrie{ - root: t.root.Copy(), - cache: t.cache, - reader: t.reader, - tracer: t.tracer.Copy(), - } -} - -// IsVerkle indicates if the trie is a Verkle trie. -func (t *VerkleTrie) IsVerkle() bool { - return true -} - -// Proof builds and returns the verkle multiproof for keys, built against -// the pre tree. The post tree is passed in order to add the post values -// to that proof. -func (t *VerkleTrie) Proof(posttrie *VerkleTrie, keys [][]byte) (*verkle.VerkleProof, verkle.StateDiff, error) { - var postroot verkle.VerkleNode - if posttrie != nil { - postroot = posttrie.root - } - proof, _, _, _, err := verkle.MakeVerkleMultiProof(t.root, postroot, keys, t.nodeResolver) - if err != nil { - return nil, nil, err - } - p, kvps, err := verkle.SerializeProof(proof) - if err != nil { - return nil, nil, err - } - return p, kvps, nil -} - -// ChunkedCode represents a sequence of 32-bytes chunks of code (31 bytes of which -// are actual code, and 1 byte is the pushdata offset). -type ChunkedCode []byte - -// Copy the values here so as to avoid an import cycle -const ( - PUSH1 = byte(0x60) - PUSH32 = byte(0x7f) -) - -// ChunkifyCode generates the chunked version of an array representing EVM bytecode -func ChunkifyCode(code []byte) ChunkedCode { - var ( - chunkOffset = 0 // offset in the chunk - chunkCount = len(code) / 31 - codeOffset = 0 // offset in the code - ) - if len(code)%31 != 0 { - chunkCount++ - } - chunks := make([]byte, chunkCount*32) - for i := 0; i < chunkCount; i++ { - // number of bytes to copy, 31 unless the end of the code has been reached. - end := 31 * (i + 1) - if len(code) < end { - end = len(code) - } - copy(chunks[i*32+1:], code[31*i:end]) // copy the code itself - - // chunk offset = taken from the last chunk. - if chunkOffset > 31 { - // skip offset calculation if push data covers the whole chunk - chunks[i*32] = 31 - chunkOffset = 1 - continue - } - chunks[32*i] = byte(chunkOffset) - chunkOffset = 0 - - // Check each instruction and update the offset it should be 0 unless - // a PUSH-N overflows. - for ; codeOffset < end; codeOffset++ { - if code[codeOffset] >= PUSH1 && code[codeOffset] <= PUSH32 { - codeOffset += int(code[codeOffset] - PUSH1 + 1) - if codeOffset+1 >= 31*(i+1) { - codeOffset++ - chunkOffset = codeOffset - 31*(i+1) - break - } - } - } - } - return chunks -} - -// UpdateContractCode implements state.Trie, writing the provided contract code -// into the trie. -// Note that the code-size *must* be already saved by a previous UpdateAccount call. -func (t *VerkleTrie) UpdateContractCode(addr common.Address, codeHash common.Hash, code []byte) error { - var ( - chunks = ChunkifyCode(code) - values [][]byte - key []byte - err error - ) - for i, chunknr := 0, uint64(0); i < len(chunks); i, chunknr = i+32, chunknr+1 { - groupOffset := (chunknr + 128) % 256 - if groupOffset == 0 /* start of new group */ || chunknr == 0 /* first chunk in header group */ { - values = make([][]byte, verkle.NodeWidth) - key = utils.CodeChunkKeyWithEvaluatedAddress(t.cache.Get(addr.Bytes()), uint256.NewInt(chunknr)) - } - values[groupOffset] = chunks[i : i+32] - - if groupOffset == 255 || len(chunks)-i <= 32 { - switch root := t.root.(type) { - case *verkle.InternalNode: - err = root.InsertValuesAtStem(key[:31], values, t.nodeResolver) - if err != nil { - return fmt.Errorf("UpdateContractCode (addr=%x) error: %w", addr[:], err) - } - default: - return errInvalidRootType - } - } - } - return nil -} - -func (t *VerkleTrie) ToDot() string { - return verkle.ToDot(t.root) -} - -func (t *VerkleTrie) nodeResolver(path []byte) ([]byte, error) { - blob, err := t.reader.Node(path, common.Hash{}) - if err != nil { - return nil, err - } - t.tracer.Put(path, blob) - return blob, nil -} - -// Witness returns a set containing all trie nodes that have been accessed. -func (t *VerkleTrie) Witness() map[string][]byte { - panic("not implemented") -} diff --git a/trie/verkle_test.go b/trie/verkle_test.go deleted file mode 100644 index 1832e3db13..0000000000 --- a/trie/verkle_test.go +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright 2023 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package trie - -import ( - "bytes" - "reflect" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/trie/utils" - "github.com/holiman/uint256" -) - -var ( - accounts = map[common.Address]*types.StateAccount{ - {1}: { - Nonce: 100, - Balance: uint256.NewInt(100), - CodeHash: common.Hash{0x1}.Bytes(), - }, - {2}: { - Nonce: 200, - Balance: uint256.NewInt(200), - CodeHash: common.Hash{0x2}.Bytes(), - }, - } - storages = map[common.Address]map[common.Hash][]byte{ - {1}: { - common.Hash{10}: []byte{10}, - common.Hash{11}: []byte{11}, - common.MaxHash: []byte{0xff}, - }, - {2}: { - common.Hash{20}: []byte{20}, - common.Hash{21}: []byte{21}, - common.MaxHash: []byte{0xff}, - }, - } -) - -func TestVerkleTreeReadWrite(t *testing.T) { - db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme) - tr, _ := NewVerkleTrie(types.EmptyVerkleHash, db, utils.NewPointCache(100)) - - for addr, acct := range accounts { - if err := tr.UpdateAccount(addr, acct, 0); err != nil { - t.Fatalf("Failed to update account, %v", err) - } - for key, val := range storages[addr] { - if err := tr.UpdateStorage(addr, key.Bytes(), val); err != nil { - t.Fatalf("Failed to update storage, %v", err) - } - } - } - - for addr, acct := range accounts { - stored, err := tr.GetAccount(addr) - if err != nil { - t.Fatalf("Failed to get account, %v", err) - } - if !reflect.DeepEqual(stored, acct) { - t.Fatal("account is not matched") - } - for key, val := range storages[addr] { - stored, err := tr.GetStorage(addr, key.Bytes()) - if err != nil { - t.Fatalf("Failed to get storage, %v", err) - } - if !bytes.Equal(stored, val) { - t.Fatal("storage is not matched") - } - } - } -} - -func TestVerkleRollBack(t *testing.T) { - db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme) - tr, _ := NewVerkleTrie(types.EmptyVerkleHash, db, utils.NewPointCache(100)) - - for addr, acct := range accounts { - // create more than 128 chunks of code - code := make([]byte, 129*32) - for i := 0; i < len(code); i += 2 { - code[i] = 0x60 - code[i+1] = byte(i % 256) - } - if err := tr.UpdateAccount(addr, acct, len(code)); err != nil { - t.Fatalf("Failed to update account, %v", err) - } - for key, val := range storages[addr] { - if err := tr.UpdateStorage(addr, key.Bytes(), val); err != nil { - t.Fatalf("Failed to update storage, %v", err) - } - } - hash := crypto.Keccak256Hash(code) - if err := tr.UpdateContractCode(addr, hash, code); err != nil { - t.Fatalf("Failed to update contract, %v", err) - } - } - - // Check that things were created - for addr, acct := range accounts { - stored, err := tr.GetAccount(addr) - if err != nil { - t.Fatalf("Failed to get account, %v", err) - } - if !reflect.DeepEqual(stored, acct) { - t.Fatal("account is not matched") - } - for key, val := range storages[addr] { - stored, err := tr.GetStorage(addr, key.Bytes()) - if err != nil { - t.Fatalf("Failed to get storage, %v", err) - } - if !bytes.Equal(stored, val) { - t.Fatal("storage is not matched") - } - } - } - - // ensure there is some code in the 2nd group of the 1st account - keyOf2ndGroup := utils.CodeChunkKeyWithEvaluatedAddress(tr.cache.Get(common.Address{1}.Bytes()), uint256.NewInt(128)) - chunk, err := tr.root.Get(keyOf2ndGroup, nil) - if err != nil { - t.Fatalf("Failed to get account, %v", err) - } - if len(chunk) == 0 { - t.Fatal("account was not created ") - } - - // Rollback first account and check that it is gone - addr1 := common.Address{1} - err = tr.RollBackAccount(addr1) - if err != nil { - t.Fatalf("error rolling back address 1: %v", err) - } - - // ensure the account is gone - stored, err := tr.GetAccount(addr1) - if err != nil { - t.Fatalf("Failed to get account, %v", err) - } - if stored != nil { - t.Fatal("account was not deleted") - } - - // ensure that the last code chunk is also gone from the tree - chunk, err = tr.root.Get(keyOf2ndGroup, nil) - if err != nil { - t.Fatalf("Failed to get account, %v", err) - } - if len(chunk) != 0 { - t.Fatal("account was not deleted") - } -} diff --git a/triedb/database.go b/triedb/database.go index 331cea81f1..996acf667c 100644 --- a/triedb/database.go +++ b/triedb/database.go @@ -129,8 +129,8 @@ func (db *Database) StateReader(blockRoot common.Hash) (database.StateReader, er return db.backend.StateReader(blockRoot) } -// HistoricReader constructs a reader for accessing the requested historic state. -func (db *Database) HistoricReader(root common.Hash) (*pathdb.HistoricalStateReader, error) { +// HistoricStateReader constructs a reader for accessing the requested historic state. +func (db *Database) HistoricStateReader(root common.Hash) (*pathdb.HistoricalStateReader, error) { pdb, ok := db.backend.(*pathdb.Database) if !ok { return nil, errors.New("not supported") @@ -138,6 +138,15 @@ func (db *Database) HistoricReader(root common.Hash) (*pathdb.HistoricalStateRea return pdb.HistoricReader(root) } +// HistoricNodeReader constructs a reader for accessing the historical trie node. +func (db *Database) HistoricNodeReader(root common.Hash) (*pathdb.HistoricalNodeReader, error) { + pdb, ok := db.backend.(*pathdb.Database) + if !ok { + return nil, errors.New("not supported") + } + return pdb.HistoricNodeReader(root) +} + // Update performs a state transition by committing dirty nodes contained in the // given set in order to update state from the specified parent to the specified // root. The held pre-images accumulated up to this point will be flushed in case diff --git a/triedb/hashdb/database.go b/triedb/hashdb/database.go index 7f7a045f66..d280d25dc9 100644 --- a/triedb/hashdb/database.go +++ b/triedb/hashdb/database.go @@ -670,6 +670,9 @@ func (db *Database) Close() error { // NodeReader returns a reader for accessing trie nodes within the specified state. // An error will be returned if the specified state is not available. func (db *Database) NodeReader(root common.Hash) (database.NodeReader, error) { + if root == types.EmptyRootHash { + return &reader{db: db}, nil + } if _, err := db.node(root); err != nil { return nil, fmt.Errorf("state %#x is not available, %v", root, err) } diff --git a/triedb/pathdb/buffer.go b/triedb/pathdb/buffer.go index 138962110f..5d3099285f 100644 --- a/triedb/pathdb/buffer.go +++ b/triedb/pathdb/buffer.go @@ -132,7 +132,7 @@ func (b *buffer) size() uint64 { // flush persists the in-memory dirty trie node into the disk if the configured // memory threshold is reached. Note, all data must be written atomically. -func (b *buffer) flush(root common.Hash, db ethdb.KeyValueStore, freezer ethdb.AncientWriter, progress []byte, nodesCache, statesCache *fastcache.Cache, id uint64, postFlush func()) { +func (b *buffer) flush(root common.Hash, db ethdb.KeyValueStore, freezers []ethdb.AncientWriter, progress []byte, nodesCache, statesCache *fastcache.Cache, id uint64, postFlush func()) { if b.done != nil { panic("duplicated flush operation") } @@ -165,11 +165,9 @@ func (b *buffer) flush(root common.Hash, db ethdb.KeyValueStore, freezer ethdb.A // // This step is crucial to guarantee that the corresponding state history remains // available for state rollback. - if freezer != nil { - if err := freezer.SyncAncient(); err != nil { - b.flushErr = err - return - } + if err := syncHistory(freezers...); err != nil { + b.flushErr = err + return } nodes := b.nodes.write(batch, nodesCache) accounts, slots := b.states.write(batch, progress, statesCache) @@ -182,6 +180,8 @@ func (b *buffer) flush(root common.Hash, db ethdb.KeyValueStore, freezer ethdb.A b.flushErr = err return } + batch.Close() + commitBytesMeter.Mark(int64(size)) commitNodesMeter.Mark(int64(nodes)) commitAccountsMeter.Mark(int64(accounts)) diff --git a/triedb/pathdb/config.go b/triedb/pathdb/config.go index 2675b701e3..b89132fc47 100644 --- a/triedb/pathdb/config.go +++ b/triedb/pathdb/config.go @@ -44,6 +44,26 @@ const ( // pause time will increase when the database writes happen. defaultBufferSize = 64 * 1024 * 1024 + // maxFullValueCheckpoint defines the maximum allowed encoding frequency (1/16) + // for storing nodes in full format. With this setting, a node may be written + // to the trienode history as a full value at the specified frequency. + // + // Note that the frequency is not strict: the actual decision is probabilistic. + // Only the overall long-term full-value encoding rate is enforced. + // + // Values beyond this limit are considered ineffective, as the trienode history + // is already well compressed. Increasing it further will only degrade read + // performance linearly. + maxFullValueCheckpoint = 16 + + // defaultFullValueCheckpoint defines the default full-value encoding frequency + // (1/8) for storing nodes in full format. With this setting, nodes may be + // written to the trienode history as full values at the specified rate. + // + // This strikes a balance between effective compression of the trienode history + // and acceptable read performance. + defaultFullValueCheckpoint = 8 + // Maximum diff layers allowed in the layer tree. DefaultMaxDiffLayers = 128 ) @@ -51,6 +71,8 @@ const ( // Defaults contains default settings for Ethereum mainnet. var Defaults = &Config{ StateHistory: params.FullImmutabilityThreshold, + TrienodeHistory: -1, + FullValueCheckpoint: defaultFullValueCheckpoint, EnableStateIndexing: false, TrieCleanSize: defaultTrieCleanSize, StateCleanSize: defaultStateCleanSize, @@ -60,27 +82,34 @@ var Defaults = &Config{ // ReadOnly is the config in order to open database in read only mode. var ReadOnly = &Config{ - ReadOnly: true, - TrieCleanSize: defaultTrieCleanSize, - StateCleanSize: defaultStateCleanSize, - MaxDiffLayers: DefaultMaxDiffLayers, + ReadOnly: true, + TrienodeHistory: -1, + TrieCleanSize: defaultTrieCleanSize, + StateCleanSize: defaultStateCleanSize, + MaxDiffLayers: DefaultMaxDiffLayers, + FullValueCheckpoint: defaultFullValueCheckpoint, } // Config contains the settings for database. type Config struct { + TrieCleanSize int // Maximum memory allowance (in bytes) for caching clean trie data + StateCleanSize int // Maximum memory allowance (in bytes) for caching clean state data + WriteBufferSize int // Maximum memory allowance (in bytes) for write buffer + ReadOnly bool // Flag whether the database is opened in read only mode + JournalDirectory string // Absolute path of journal directory (null means the journal data is persisted in key-value store) + + // Historical state configurations StateHistory uint64 // Number of recent blocks to maintain state history for, 0: full chain MaxDiffLayers int // Maximum diff layers allowed in the layer tree. + TrienodeHistory int64 // Number of recent blocks to maintain trienode history for, 0: full chain, negative: disable EnableStateIndexing bool // Whether to enable state history indexing for external state access - TrieCleanSize int // Maximum memory allowance (in bytes) for caching clean trie data - StateCleanSize int // Maximum memory allowance (in bytes) for caching clean state data - WriteBufferSize int // Maximum memory allowance (in bytes) for write buffer - ReadOnly bool // Flag whether the database is opened in read only mode - JournalDirectory string // Absolute path of journal directory (null means the journal data is persisted in key-value store) + FullValueCheckpoint uint32 // The rate at which trie nodes are encoded in full-value format // Testing configurations - SnapshotNoBuild bool // Flag Whether the state generation is disabled - NoAsyncFlush bool // Flag whether the background buffer flushing is disabled - NoAsyncGeneration bool // Flag whether the background generation is disabled + SnapshotNoBuild bool // Flag Whether the state generation is disabled + NoAsyncFlush bool // Flag whether the background buffer flushing is disabled + NoAsyncGeneration bool // Flag whether the background generation is disabled + NoHistoryIndexDelay bool // Flag whether the history index delay is disabled } // sanitize checks the provided user configurations and changes anything that's @@ -91,11 +120,18 @@ func (c *Config) sanitize() *Config { log.Warn("Sanitizing invalid node buffer size", "provided", common.StorageSize(conf.WriteBufferSize), "updated", common.StorageSize(maxBufferSize)) conf.WriteBufferSize = maxBufferSize } - if conf.MaxDiffLayers <= 0 { log.Warn("Sanitizing invalid max diff layers", "provided", conf.MaxDiffLayers, "updated", DefaultMaxDiffLayers) conf.MaxDiffLayers = DefaultMaxDiffLayers } + if conf.FullValueCheckpoint > maxFullValueCheckpoint { + log.Warn("Sanitizing trienode history full value checkpoint", "provided", conf.FullValueCheckpoint, "updated", maxFullValueCheckpoint) + conf.FullValueCheckpoint = maxFullValueCheckpoint + } + if conf.FullValueCheckpoint == 0 { + conf.FullValueCheckpoint = 1 + log.Info("Disabling diff mode trie node history encoding") + } return &conf } @@ -114,6 +150,13 @@ func (c *Config) fields() []interface{} { } else { list = append(list, "state-history", fmt.Sprintf("last %d blocks", c.StateHistory)) } + if c.TrienodeHistory >= 0 { + if c.TrienodeHistory == 0 { + list = append(list, "trie-history", "entire chain") + } else { + list = append(list, "trie-history", fmt.Sprintf("last %d blocks", c.TrienodeHistory)) + } + } if c.EnableStateIndexing { list = append(list, "index-history", true) } diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index 0e00318c5a..4c0846da87 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -31,8 +31,8 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/trie/bintrie" "github.com/ethereum/go-ethereum/trie/trienode" - "github.com/ethereum/go-verkle" ) // layer is the interface implemented by all state layers which includes some @@ -45,7 +45,7 @@ type layer interface { // Note: // - the returned node is not a copy, please don't modify it. // - no error will be returned if the requested node is not found in database. - node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, *nodeLoc, error) + node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, nodeLoc, error) // account directly retrieves the account RLP associated with a particular // hash in the slim data format. An error will be returned if the read @@ -97,16 +97,16 @@ func merkleNodeHasher(blob []byte) (common.Hash, error) { return crypto.Keccak256Hash(blob), nil } -// verkleNodeHasher computes the hash of the given verkle node. -func verkleNodeHasher(blob []byte) (common.Hash, error) { +// binaryNodeHasher computes the hash of the given verkle node. +func binaryNodeHasher(blob []byte) (common.Hash, error) { if len(blob) == 0 { return types.EmptyVerkleHash, nil } - n, err := verkle.ParseNode(blob, 0) + n, err := bintrie.DeserializeNode(blob, 0) if err != nil { return common.Hash{}, err } - return n.Commit().Bytes(), nil + return n.Hash(), nil } // Database is a multiple-layered structure for maintaining in-memory states @@ -137,6 +137,9 @@ type Database struct { stateFreezer ethdb.ResettableAncientStore // Freezer for storing state histories, nil possible in tests stateIndexer *historyIndexer // History indexer historical state data, nil possible + trienodeFreezer ethdb.ResettableAncientStore // Freezer for storing trienode histories, nil possible in tests + trienodeIndexer *historyIndexer // History indexer for historical trienode data + lock sync.RWMutex // Lock to prevent mutations from happening at the same time } @@ -163,17 +166,20 @@ func New(diskdb ethdb.Database, config *Config, isVerkle bool) *Database { // compress the shared key prefix. if isVerkle { db.diskdb = rawdb.NewTable(diskdb, string(rawdb.VerklePrefix)) - db.hasher = verkleNodeHasher + db.hasher = binaryNodeHasher } // Construct the layer tree by resolving the in-disk singleton state // and in-memory layer journal. db.tree = newLayerTree(db.loadLayers()) - // Repair the state history, which might not be aligned with the state - // in the key-value store due to an unclean shutdown. - if err := db.repairHistory(); err != nil { - log.Crit("Failed to repair state history", "err", err) + // Repair the history, which might not be aligned with the persistent + // state in the key-value store due to an unclean shutdown. + states, trienodes, err := repairHistory(db.diskdb, isVerkle, db.config.ReadOnly, db.tree.bottom().stateID(), db.config.TrienodeHistory >= 0) + if err != nil { + log.Crit("Failed to repair history", "err", err) } + db.stateFreezer, db.trienodeFreezer = states, trienodes + // Disable database in case node is still in the initial state sync stage. if rawdb.ReadSnapSyncStatusFlag(diskdb) == rawdb.StateSyncRunning && !db.readOnly { if err := db.Disable(); err != nil { @@ -187,11 +193,8 @@ func New(diskdb ethdb.Database, config *Config, isVerkle bool) *Database { if err := db.setStateGenerator(); err != nil { log.Crit("Failed to setup the generator", "err", err) } - // TODO (rjl493456442) disable the background indexing in read-only mode - if db.stateFreezer != nil && db.config.EnableStateIndexing { - db.stateIndexer = newHistoryIndexer(db.diskdb, db.stateFreezer, db.tree.bottom().stateID(), typeStateHistory) - log.Info("Enabled state history indexing") - } + db.setHistoryIndexer() + fields := config.fields() if db.isVerkle { fields = append(fields, "verkle", true) @@ -200,59 +203,28 @@ func New(diskdb ethdb.Database, config *Config, isVerkle bool) *Database { return db } -// repairHistory truncates leftover state history objects, which may occur due -// to an unclean shutdown or other unexpected reasons. -func (db *Database) repairHistory() error { - // Open the freezer for state history. This mechanism ensures that - // only one database instance can be opened at a time to prevent - // accidental mutation. - ancient, err := db.diskdb.AncientDatadir() - if err != nil { - // TODO error out if ancient store is disabled. A tons of unit tests - // disable the ancient store thus the error here will immediately fail - // all of them. Fix the tests first. - return nil - } - freezer, err := rawdb.NewStateFreezer(ancient, db.isVerkle, db.readOnly) - if err != nil { - log.Crit("Failed to open state history freezer", "err", err) +// setHistoryIndexer initializes the indexers for both state history and +// trienode history if available. Note that this function may be called while +// existing indexers are still running, so they must be closed beforehand. +func (db *Database) setHistoryIndexer() { + // TODO (rjl493456442) disable the background indexing in read-only mode + if !db.config.EnableStateIndexing { + return } - db.stateFreezer = freezer - - // Reset the entire state histories if the trie database is not initialized - // yet. This action is necessary because these state histories are not - // expected to exist without an initialized trie database. - id := db.tree.bottom().stateID() - if id == 0 { - frozen, err := db.stateFreezer.Ancients() - if err != nil { - log.Crit("Failed to retrieve head of state history", "err", err) - } - if frozen != 0 { - // Purge all state history indexing data first - batch := db.diskdb.NewBatch() - rawdb.DeleteStateHistoryIndexMetadata(batch) - rawdb.DeleteStateHistoryIndexes(batch) - if err := batch.Write(); err != nil { - log.Crit("Failed to purge state history index", "err", err) - } - if err := db.stateFreezer.Reset(); err != nil { - log.Crit("Failed to reset state histories", "err", err) - } - log.Info("Truncated extraneous state history") + if db.stateFreezer != nil { + if db.stateIndexer != nil { + db.stateIndexer.close() } - return nil - } - // Truncate the extra state histories above in freezer in case it's not - // aligned with the disk layer. It might happen after a unclean shutdown. - pruned, err := truncateFromHead(db.stateFreezer, typeStateHistory, id) - if err != nil { - log.Crit("Failed to truncate extra state histories", "err", err) + db.stateIndexer = newHistoryIndexer(db.diskdb, db.stateFreezer, db.tree.bottom().stateID(), typeStateHistory, db.config.NoHistoryIndexDelay) + log.Info("Enabled state history indexing") } - if pruned != 0 { - log.Warn("Truncated extra state histories", "number", pruned) + if db.trienodeFreezer != nil { + if db.trienodeIndexer != nil { + db.trienodeIndexer.close() + } + db.trienodeIndexer = newHistoryIndexer(db.diskdb, db.trienodeFreezer, db.tree.bottom().stateID(), typeTrienodeHistory, db.config.NoHistoryIndexDelay) + log.Info("Enabled trienode history indexing") } - return nil } // setStateGenerator loads the state generation progress marker and potentially @@ -333,8 +305,13 @@ func (db *Database) Update(root common.Hash, parentRoot common.Hash, block uint6 if err := db.modifyAllowed(); err != nil { return err } - // TODO(rjl493456442) tracking the origins in the following PRs. - if err := db.tree.add(root, parentRoot, block, NewNodeSetWithOrigin(nodes.Nodes(), nil), states); err != nil { + var nodesWithOrigins *nodeSetWithOrigin + if db.config.TrienodeHistory >= 0 { + nodesWithOrigins = NewNodeSetWithOrigin(nodes.NodeAndOrigins()) + } else { + nodesWithOrigins = NewNodeSetWithOrigin(nodes.Nodes(), nil) + } + if err := db.tree.add(root, parentRoot, block, nodesWithOrigins, states); err != nil { return err } // Keep 128 diff layers in the memory, persistent layer is 129th. @@ -422,18 +399,9 @@ func (db *Database) Enable(root common.Hash) error { // all root->id mappings should be removed as well. Since // mappings can be huge and might take a while to clear // them, just leave them in disk and wait for overwriting. - if db.stateFreezer != nil { - // Purge all state history indexing data first - batch.Reset() - rawdb.DeleteStateHistoryIndexMetadata(batch) - rawdb.DeleteStateHistoryIndexes(batch) - if err := batch.Write(); err != nil { - return err - } - if err := db.stateFreezer.Reset(); err != nil { - return err - } - } + purgeHistory(db.stateFreezer, db.diskdb, typeStateHistory) + purgeHistory(db.trienodeFreezer, db.diskdb, typeTrienodeHistory) + // Re-enable the database as the final step. db.waitSync = false rawdb.WriteSnapSyncStatusFlag(db.diskdb, rawdb.StateSyncFinished) @@ -446,11 +414,8 @@ func (db *Database) Enable(root common.Hash) error { // To ensure the history indexer always matches the current state, we must: // 1. Close any existing indexer // 2. Re-initialize the indexer so it starts indexing from the new state root. - if db.stateIndexer != nil && db.stateFreezer != nil && db.config.EnableStateIndexing { - db.stateIndexer.close() - db.stateIndexer = newHistoryIndexer(db.diskdb, db.stateFreezer, db.tree.bottom().stateID(), typeStateHistory) - log.Info("Re-enabled state history indexing") - } + db.setHistoryIndexer() + log.Info("Rebuilt trie database", "root", root) return nil } @@ -506,6 +471,12 @@ func (db *Database) Recover(root common.Hash) error { if err != nil { return err } + if db.trienodeFreezer != nil { + _, err = truncateFromHead(db.trienodeFreezer, typeTrienodeHistory, dl.stateID()) + if err != nil { + return err + } + } log.Debug("Recovered state", "root", root, "elapsed", common.PrettyDuration(time.Since(start))) return nil } @@ -566,11 +537,21 @@ func (db *Database) Close() error { if db.stateIndexer != nil { db.stateIndexer.close() } + if db.trienodeIndexer != nil { + db.trienodeIndexer.close() + } // Close the attached state history freezer. - if db.stateFreezer == nil { - return nil + if db.stateFreezer != nil { + if err := db.stateFreezer.Close(); err != nil { + return err + } + } + if db.trienodeFreezer != nil { + if err := db.trienodeFreezer.Close(); err != nil { + return err + } } - return db.stateFreezer.Close() + return nil } // Size returns the current storage size of the memory cache in front of the diff --git a/triedb/pathdb/database_test.go b/triedb/pathdb/database_test.go index 31fcac3c84..37a1bfaf1e 100644 --- a/triedb/pathdb/database_test.go +++ b/triedb/pathdb/database_test.go @@ -184,6 +184,7 @@ func newTester(t *testing.T, config *testerConfig) *tester { WriteBufferSize: config.writeBufferSize(), NoAsyncFlush: true, JournalDirectory: config.journalDir, + NoHistoryIndexDelay: true, }, config.isVerkle) obj = &tester{ @@ -900,7 +901,7 @@ func TestDatabaseIndexRecovery(t *testing.T) { var ( dIndex int roots = env.roots - hr = newHistoryReader(env.db.diskdb, env.db.stateFreezer) + hr = newStateHistoryReader(env.db.diskdb, env.db.stateFreezer) ) for i, root := range roots { if root == dRoot { @@ -961,7 +962,7 @@ func TestDatabaseIndexRecovery(t *testing.T) { // Ensure the truncated state histories become accessible bRoot = env.db.tree.bottom().rootHash() - hr = newHistoryReader(env.db.diskdb, env.db.stateFreezer) + hr = newStateHistoryReader(env.db.diskdb, env.db.stateFreezer) for i, root := range roots { if root == bRoot { break diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go index ae523c979c..8ca3a39cf3 100644 --- a/triedb/pathdb/difflayer.go +++ b/triedb/pathdb/difflayer.go @@ -79,7 +79,7 @@ func (dl *diffLayer) parentLayer() layer { // node implements the layer interface, retrieving the trie node blob with the // provided node information. No error will be returned if the node is not found. -func (dl *diffLayer) node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, *nodeLoc, error) { +func (dl *diffLayer) node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, nodeLoc, error) { // Hold the lock, ensure the parent won't be changed during the // state accessing. dl.lock.RLock() @@ -91,7 +91,7 @@ func (dl *diffLayer) node(owner common.Hash, path []byte, depth int) ([]byte, co dirtyNodeHitMeter.Mark(1) dirtyNodeHitDepthHist.Update(int64(depth)) dirtyNodeReadMeter.Mark(int64(len(n.Blob))) - return n.Blob, n.Hash, &nodeLoc{loc: locDiffLayer, depth: depth}, nil + return n.Blob, n.Hash, nodeLoc{loc: locDiffLayer, depth: depth}, nil } // Trie node unknown to this layer, resolve from parent return dl.parent.node(owner, path, depth+1) diff --git a/triedb/pathdb/disklayer.go b/triedb/pathdb/disklayer.go index 42cb703d1b..ed8027ae5f 100644 --- a/triedb/pathdb/disklayer.go +++ b/triedb/pathdb/disklayer.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) @@ -111,12 +112,12 @@ func (dl *diskLayer) markStale() { // node implements the layer interface, retrieving the trie node with the // provided node info. No error will be returned if the node is not found. -func (dl *diskLayer) node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, *nodeLoc, error) { +func (dl *diskLayer) node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, nodeLoc, error) { dl.lock.RLock() defer dl.lock.RUnlock() if dl.stale { - return nil, common.Hash{}, nil, errSnapshotStale + return nil, common.Hash{}, nodeLoc{}, errSnapshotStale } // Try to retrieve the trie node from the not-yet-written node buffer first // (both the live one and the frozen one). Note the buffer is lock free since @@ -128,7 +129,7 @@ func (dl *diskLayer) node(owner common.Hash, path []byte, depth int) ([]byte, co dirtyNodeHitMeter.Mark(1) dirtyNodeReadMeter.Mark(int64(len(n.Blob))) dirtyNodeHitDepthHist.Update(int64(depth)) - return n.Blob, n.Hash, &nodeLoc{loc: locDirtyCache, depth: depth}, nil + return n.Blob, n.Hash, nodeLoc{loc: locDirtyCache, depth: depth}, nil } } } @@ -140,7 +141,7 @@ func (dl *diskLayer) node(owner common.Hash, path []byte, depth int) ([]byte, co if blob := dl.nodes.Get(nil, key); len(blob) > 0 { cleanNodeHitMeter.Mark(1) cleanNodeReadMeter.Mark(int64(len(blob))) - return blob, crypto.Keccak256Hash(blob), &nodeLoc{loc: locCleanCache, depth: depth}, nil + return blob, crypto.Keccak256Hash(blob), nodeLoc{loc: locCleanCache, depth: depth}, nil } cleanNodeMissMeter.Mark(1) } @@ -160,7 +161,7 @@ func (dl *diskLayer) node(owner common.Hash, path []byte, depth int) ([]byte, co dl.nodes.Set(key, blob) cleanNodeWriteMeter.Mark(int64(len(blob))) } - return blob, crypto.Keccak256Hash(blob), &nodeLoc{loc: locDiskLayer, depth: depth}, nil + return blob, crypto.Keccak256Hash(blob), nodeLoc{loc: locDiskLayer, depth: depth}, nil } // account directly retrieves the account RLP associated with a particular @@ -278,7 +279,7 @@ func (dl *diskLayer) storage(accountHash, storageHash common.Hash, depth int) ([ // If the layer is being generated, ensure the requested storage slot // has already been covered by the generator. - key := append(accountHash[:], storageHash[:]...) + key := storageKeySlice(accountHash, storageHash) marker := dl.genMarker() if marker != nil && bytes.Compare(key, marker) > 0 { return nil, errNotCoveredYet @@ -329,36 +330,61 @@ func (dl *diskLayer) update(root common.Hash, id uint64, block uint64, nodes *no return newDiffLayer(dl, root, id, block, nodes, states) } -// writeStateHistory stores the state history and indexes if indexing is +// writeHistory stores the specified history and indexes if indexing is // permitted. // // What's more, this function also returns a flag indicating whether the // buffer flushing is required, ensuring the persistent state ID is always // greater than or equal to the first history ID. -func (dl *diskLayer) writeStateHistory(diff *diffLayer) (bool, error) { - // Short circuit if state history is not permitted - if dl.db.stateFreezer == nil { +func (dl *diskLayer) writeHistory(typ historyType, diff *diffLayer) (bool, error) { + var ( + limit uint64 + freezer ethdb.AncientStore + indexer *historyIndexer + writeFunc func(writer ethdb.AncientWriter, dl *diffLayer) error + ) + switch typ { + case typeStateHistory: + freezer = dl.db.stateFreezer + indexer = dl.db.stateIndexer + writeFunc = writeStateHistory + limit = dl.db.config.StateHistory + case typeTrienodeHistory: + freezer = dl.db.trienodeFreezer + indexer = dl.db.trienodeIndexer + writeFunc = func(writer ethdb.AncientWriter, diff *diffLayer) error { + return writeTrienodeHistory(writer, diff, dl.db.config.FullValueCheckpoint) + } + // Skip the history commit if the trienode history is not permitted + if dl.db.config.TrienodeHistory < 0 { + return false, nil + } + limit = uint64(dl.db.config.TrienodeHistory) + default: + panic(fmt.Sprintf("unknown history type: %v", typ)) + } + // Short circuit if the history freezer is nil + if freezer == nil { return false, nil } // Bail out with an error if writing the state history fails. // This can happen, for example, if the device is full. - err := writeStateHistory(dl.db.stateFreezer, diff) + err := writeFunc(freezer, diff) if err != nil { return false, err } - // Notify the state history indexer for newly created history - if dl.db.stateIndexer != nil { - if err := dl.db.stateIndexer.extend(diff.stateID()); err != nil { + // Notify the history indexer for newly created history + if indexer != nil { + if err := indexer.extend(diff.stateID()); err != nil { return false, err } } // Determine if the persisted history object has exceeded the // configured limitation. - limit := dl.db.config.StateHistory if limit == 0 { return false, nil } - tail, err := dl.db.stateFreezer.Tail() + tail, err := freezer.Tail() if err != nil { return false, err } // firstID = tail+1 @@ -381,14 +407,14 @@ func (dl *diskLayer) writeStateHistory(diff *diffLayer) (bool, error) { // These measures ensure the persisted state ID always remains greater // than or equal to the first history ID. if persistentID := rawdb.ReadPersistentStateID(dl.db.diskdb); persistentID < newFirst { - log.Debug("Skip tail truncation", "persistentID", persistentID, "tailID", tail+1, "headID", diff.stateID(), "limit", limit) + log.Debug("Skip tail truncation", "type", typ, "persistentID", persistentID, "tailID", tail+1, "headID", diff.stateID(), "limit", limit) return true, nil } - pruned, err := truncateFromTail(dl.db.stateFreezer, typeStateHistory, newFirst-1) + pruned, err := truncateFromTail(freezer, typ, newFirst-1) if err != nil { return false, err } - log.Debug("Pruned state history", "items", pruned, "tailid", newFirst) + log.Debug("Pruned history", "type", typ, "items", pruned, "tailid", newFirst) return false, nil } @@ -402,10 +428,22 @@ func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) { // Construct and store the state history first. If crash happens after storing // the state history but without flushing the corresponding states(journal), // the stored state history will be truncated from head in the next restart. - flush, err := dl.writeStateHistory(bottom) + flushA, err := dl.writeHistory(typeStateHistory, bottom) if err != nil { return nil, err } + // Construct and store the trienode history first. If crash happens after + // storing the trienode history but without flushing the corresponding + // states(journal), the stored trienode history will be truncated from head + // in the next restart. + flushB, err := dl.writeHistory(typeTrienodeHistory, bottom) + if err != nil { + return nil, err + } + // Since the state history and trienode history may be configured with different + // lengths, the buffer will be flushed once either of them meets its threshold. + flush := flushA || flushB + // Mark the diskLayer as stale before applying any mutations on top. dl.stale = true @@ -454,7 +492,7 @@ func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) { // Freeze the live buffer and schedule background flushing dl.frozen = combined - dl.frozen.flush(bottom.root, dl.db.diskdb, dl.db.stateFreezer, progress, dl.nodes, dl.states, bottom.stateID(), func() { + dl.frozen.flush(bottom.root, dl.db.diskdb, []ethdb.AncientWriter{dl.db.stateFreezer, dl.db.trienodeFreezer}, progress, dl.nodes, dl.states, bottom.stateID(), func() { // Resume the background generation if it's not completed yet. // The generator is assumed to be available if the progress is // not nil. @@ -510,12 +548,17 @@ func (dl *diskLayer) revert(h *stateHistory) (*diskLayer, error) { dl.stale = true - // Unindex the corresponding state history + // Unindex the corresponding history if dl.db.stateIndexer != nil { if err := dl.db.stateIndexer.shorten(dl.id); err != nil { return nil, err } } + if dl.db.trienodeIndexer != nil { + if err := dl.db.trienodeIndexer.shorten(dl.id); err != nil { + return nil, err + } + } // State change may be applied to node buffer, or the persistent // state, depends on if node buffer is empty or not. If the node // buffer is not empty, it means that the state transition that diff --git a/triedb/pathdb/flush.go b/triedb/pathdb/flush.go index 6563dbccff..4f816cf6a6 100644 --- a/triedb/pathdb/flush.go +++ b/triedb/pathdb/flush.go @@ -116,15 +116,16 @@ func writeStates(batch ethdb.Batch, genMarker []byte, accountData map[common.Has continue } slots += 1 + key := storageKeySlice(addrHash, storageHash) if len(blob) == 0 { rawdb.DeleteStorageSnapshot(batch, addrHash, storageHash) if clean != nil { - clean.Set(append(addrHash[:], storageHash[:]...), nil) + clean.Set(key, nil) } } else { rawdb.WriteStorageSnapshot(batch, addrHash, storageHash, blob) if clean != nil { - clean.Set(append(addrHash[:], storageHash[:]...), blob) + clean.Set(key, blob) } } } diff --git a/triedb/pathdb/generate.go b/triedb/pathdb/generate.go index 2efbbbb4e1..d3d26fff26 100644 --- a/triedb/pathdb/generate.go +++ b/triedb/pathdb/generate.go @@ -148,6 +148,7 @@ func (g *generator) stop() { g.abort <- ch <-ch g.running = false + log.Debug("Snapshot generation has been terminated") } // completed returns the flag indicating if the whole generation is done. diff --git a/triedb/pathdb/history.go b/triedb/pathdb/history.go index d78999f218..0a9f7091fa 100644 --- a/triedb/pathdb/history.go +++ b/triedb/pathdb/history.go @@ -22,6 +22,7 @@ import ( "iter" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) @@ -121,6 +122,20 @@ func (ident stateIdent) String() string { return ident.addressHash.Hex() + ident.path } +func (ident stateIdent) bloomSize() int { + if ident.typ == typeAccount { + return 0 + } + if ident.typ == typeStorage { + return 0 + } + scheme := accountIndexScheme + if ident.addressHash != (common.Hash{}) { + scheme = storageIndexScheme + } + return scheme.getBitmapSize(len(ident.path)) +} + // newAccountIdent constructs a state identifier for an account. func newAccountIdent(addressHash common.Hash) stateIdent { return stateIdent{ @@ -180,17 +195,62 @@ func newStorageIdentQuery(address common.Address, addressHash common.Hash, stora } } -// newTrienodeIdentQuery constructs a state identifier for a trie node. -// the addressHash denotes the address hash of the associated account; -// the path denotes the path of the node within the trie; -// -// nolint:unused -func newTrienodeIdentQuery(addrHash common.Hash, path []byte) stateIdentQuery { - return stateIdentQuery{ - stateIdent: newTrienodeIdent(addrHash, string(path)), +// indexElem defines the element for indexing. +type indexElem interface { + key() stateIdent + ext() []uint16 +} + +type accountIndexElem struct { + addressHash common.Hash +} + +func (a accountIndexElem) key() stateIdent { + return stateIdent{ + typ: typeAccount, + addressHash: a.addressHash, } } +func (a accountIndexElem) ext() []uint16 { + return nil +} + +type storageIndexElem struct { + addressHash common.Hash + storageHash common.Hash +} + +func (a storageIndexElem) key() stateIdent { + return stateIdent{ + typ: typeStorage, + addressHash: a.addressHash, + storageHash: a.storageHash, + } +} + +func (a storageIndexElem) ext() []uint16 { + return nil +} + +type trienodeIndexElem struct { + owner common.Hash + path string + data []uint16 +} + +func (a trienodeIndexElem) key() stateIdent { + return stateIdent{ + typ: typeTrienode, + addressHash: a.owner, + path: a.path, + } +} + +func (a trienodeIndexElem) ext() []uint16 { + return a.data +} + // history defines the interface of historical data, shared by stateHistory // and trienodeHistory. type history interface { @@ -198,7 +258,7 @@ type history interface { typ() historyType // forEach returns an iterator to traverse the state entries in the history. - forEach() iter.Seq[stateIdent] + forEach() iter.Seq[indexElem] } var ( @@ -262,3 +322,139 @@ func truncateFromTail(store ethdb.AncientStore, typ historyType, ntail uint64) ( // Associated root->id mappings are left in the database. return int(ntail - otail), nil } + +// purgeHistory resets the history and also purges the associated index data. +func purgeHistory(store ethdb.ResettableAncientStore, disk ethdb.KeyValueStore, typ historyType) { + if store == nil { + return + } + frozen, err := store.Ancients() + if err != nil { + log.Crit("Failed to retrieve head of history", "type", typ, "err", err) + } + if frozen == 0 { + return + } + // Purge all state history indexing data first + batch := disk.NewBatch() + if typ == typeStateHistory { + rawdb.DeleteStateHistoryIndexMetadata(batch) + rawdb.DeleteStateHistoryIndexes(batch) + } else { + rawdb.DeleteTrienodeHistoryIndexMetadata(batch) + rawdb.DeleteTrienodeHistoryIndexes(batch) + } + if err := batch.Write(); err != nil { + log.Crit("Failed to purge history index", "type", typ, "err", err) + } + if err := store.Reset(); err != nil { + log.Crit("Failed to reset history", "type", typ, "err", err) + } + log.Info("Truncated extraneous history", "type", typ) +} + +// syncHistory explicitly sync the provided history stores. +func syncHistory(stores ...ethdb.AncientWriter) error { + for _, store := range stores { + if store == nil { + continue + } + if err := store.SyncAncient(); err != nil { + return err + } + } + return nil +} + +// repairHistory truncates any leftover history objects in either the state +// history or the trienode history, which may occur due to an unclean shutdown +// or other unexpected events. +// +// Additionally, this mechanism ensures that the state history and trienode +// history remain aligned. Since the trienode history is optional and not +// required by regular users, a gap between the trienode history and the +// persistent state may appear if the trienode history was disabled during the +// previous run. This process detects and resolves such gaps, preventing +// unexpected panics. +func repairHistory(db ethdb.Database, isVerkle bool, readOnly bool, stateID uint64, enableTrienode bool) (ethdb.ResettableAncientStore, ethdb.ResettableAncientStore, error) { + ancient, err := db.AncientDatadir() + if err != nil { + // TODO error out if ancient store is disabled. A tons of unit tests + // disable the ancient store thus the error here will immediately fail + // all of them. Fix the tests first. + return nil, nil, nil + } + // State history is mandatory as it is the key component that ensures + // resilience to deep reorgs. + states, err := rawdb.NewStateFreezer(ancient, isVerkle, readOnly) + if err != nil { + log.Crit("Failed to open state history freezer", "err", err) + } + + // Trienode history is optional and only required for building archive + // node with state proofs. + var trienodes ethdb.ResettableAncientStore + if enableTrienode { + trienodes, err = rawdb.NewTrienodeFreezer(ancient, isVerkle, readOnly) + if err != nil { + log.Crit("Failed to open trienode history freezer", "err", err) + } + } + + // Reset the both histories if the trie database is not initialized yet. + // This action is necessary because these histories are not expected + // to exist without an initialized trie database. + if stateID == 0 { + purgeHistory(states, db, typeStateHistory) + purgeHistory(trienodes, db, typeTrienodeHistory) + return states, trienodes, nil + } + // Truncate excessive history entries in either the state history or + // the trienode history, ensuring both histories remain aligned with + // the state. + shead, err := states.Ancients() + if err != nil { + return nil, nil, err + } + if stateID > shead { // Gap is not permitted in the state history + return nil, nil, fmt.Errorf("gap between state [#%d] and state history [#%d]", stateID, shead) + } + truncTo := min(shead, stateID) + + if trienodes != nil { + thead, err := trienodes.Ancients() + if err != nil { + return nil, nil, err + } + if stateID <= thead { + truncTo = min(truncTo, thead) + } else { + if thead == 0 { + _, err = trienodes.TruncateTail(stateID) + if err != nil { + return nil, nil, err + } + log.Warn("Initialized trienode history") + } else { + return nil, nil, fmt.Errorf("gap between state [#%d] and trienode history [#%d]", stateID, thead) + } + } + } + // Truncate the extra history elements above in freezer in case it's not + // aligned with the state. It might happen after an unclean shutdown. + truncate := func(store ethdb.AncientStore, typ historyType, nhead uint64) { + if store == nil { + return + } + pruned, err := truncateFromHead(store, typ, nhead) + if err != nil { + log.Crit("Failed to truncate extra histories", "typ", typ, "err", err) + } + if pruned != 0 { + log.Warn("Truncated extra histories", "typ", typ, "number", pruned) + } + } + truncate(states, typeStateHistory, truncTo) + truncate(trienodes, typeTrienodeHistory, truncTo) + return states, trienodes, nil +} diff --git a/triedb/pathdb/history_index.go b/triedb/pathdb/history_index.go index 5b4c91d7e6..eee4c10273 100644 --- a/triedb/pathdb/history_index.go +++ b/triedb/pathdb/history_index.go @@ -20,28 +20,33 @@ import ( "errors" "fmt" "math" - "sort" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" ) -// parseIndex parses the index data with the supplied byte stream. The index data -// is a list of fixed-sized metadata. Empty metadata is regarded as invalid. -func parseIndex(blob []byte) ([]*indexBlockDesc, error) { +// parseIndex parses the index data from the provided byte stream. The index data +// is a sequence of fixed-size metadata entries, and any empty metadata entry is +// considered invalid. +// +// Each metadata entry consists of two components: the indexBlockDesc and an +// optional extension bitmap. The bitmap length may vary across different categories, +// but must remain consistent within the same category. +func parseIndex(blob []byte, bitmapSize int) ([]*indexBlockDesc, error) { if len(blob) == 0 { return nil, errors.New("empty state history index") } - if len(blob)%indexBlockDescSize != 0 { - return nil, fmt.Errorf("corrupted state index, len: %d", len(blob)) + size := indexBlockDescSize + bitmapSize + if len(blob)%size != 0 { + return nil, fmt.Errorf("corrupted state index, len: %d, bitmap size: %d", len(blob), bitmapSize) } var ( lastID uint32 - descList []*indexBlockDesc + descList = make([]*indexBlockDesc, 0, len(blob)/size) ) - for i := 0; i < len(blob)/indexBlockDescSize; i++ { + for i := 0; i < len(blob)/size; i++ { var desc indexBlockDesc - desc.decode(blob[i*indexBlockDescSize : (i+1)*indexBlockDescSize]) + desc.decode(blob[i*size : (i+1)*size]) if desc.empty() { return nil, errors.New("empty state history index block") } @@ -70,33 +75,35 @@ func parseIndex(blob []byte) ([]*indexBlockDesc, error) { // indexReader is the structure to look up the state history index records // associated with the specific state element. type indexReader struct { - db ethdb.KeyValueReader - descList []*indexBlockDesc - readers map[uint32]*blockReader - state stateIdent + db ethdb.KeyValueReader + descList []*indexBlockDesc + readers map[uint32]*blockReader + state stateIdent + bitmapSize int } // loadIndexData loads the index data associated with the specified state. -func loadIndexData(db ethdb.KeyValueReader, state stateIdent) ([]*indexBlockDesc, error) { +func loadIndexData(db ethdb.KeyValueReader, state stateIdent, bitmapSize int) ([]*indexBlockDesc, error) { blob := readStateIndex(state, db) if len(blob) == 0 { return nil, nil } - return parseIndex(blob) + return parseIndex(blob, bitmapSize) } // newIndexReader constructs a index reader for the specified state. Reader with // empty data is allowed. -func newIndexReader(db ethdb.KeyValueReader, state stateIdent) (*indexReader, error) { - descList, err := loadIndexData(db, state) +func newIndexReader(db ethdb.KeyValueReader, state stateIdent, bitmapSize int) (*indexReader, error) { + descList, err := loadIndexData(db, state, bitmapSize) if err != nil { return nil, err } return &indexReader{ - descList: descList, - readers: make(map[uint32]*blockReader), - db: db, - state: state, + descList: descList, + readers: make(map[uint32]*blockReader), + db: db, + state: state, + bitmapSize: bitmapSize, }, nil } @@ -107,11 +114,9 @@ func (r *indexReader) refresh() error { // may have been modified by additional elements written to the disk. if len(r.descList) != 0 { last := r.descList[len(r.descList)-1] - if !last.full() { - delete(r.readers, last.id) - } + delete(r.readers, last.id) } - descList, err := loadIndexData(r.db, r.state) + descList, err := loadIndexData(r.db, r.state, r.bitmapSize) if err != nil { return err } @@ -122,27 +127,15 @@ func (r *indexReader) refresh() error { // readGreaterThan locates the first element that is greater than the specified // id. If no such element is found, MaxUint64 is returned. func (r *indexReader) readGreaterThan(id uint64) (uint64, error) { - index := sort.Search(len(r.descList), func(i int) bool { - return id < r.descList[i].max - }) - if index == len(r.descList) { - return math.MaxUint64, nil + it := r.newIterator(nil) + found := it.SeekGT(id) + if err := it.Error(); err != nil { + return 0, err } - desc := r.descList[index] - - br, ok := r.readers[desc.id] - if !ok { - var err error - blob := readStateIndexBlock(r.state, r.db, desc.id) - br, err = newBlockReader(blob) - if err != nil { - return 0, err - } - r.readers[desc.id] = br + if !found { + return math.MaxUint64, nil } - // The supplied ID is not greater than block.max, ensuring that an element - // satisfying the condition can be found. - return br.readGreaterThan(id) + return it.ID(), nil } // indexWriter is responsible for writing index data for a specific state (either @@ -152,57 +145,75 @@ func (r *indexReader) readGreaterThan(id uint64) (uint64, error) { // history ids) is stored in these second-layer index blocks, which are size // limited. type indexWriter struct { - descList []*indexBlockDesc // The list of index block descriptions - bw *blockWriter // The live index block writer - frozen []*blockWriter // The finalized index block writers, waiting for flush - lastID uint64 // The ID of the latest tracked history - state stateIdent - db ethdb.KeyValueReader + descList []*indexBlockDesc // The list of index block descriptions + bw *blockWriter // The live index block writer + frozen []*blockWriter // The finalized index block writers, waiting for flush + lastID uint64 // The ID of the latest tracked history + state stateIdent // The identifier of the state being indexed + bitmapSize int // The size of optional extension bitmap + db ethdb.KeyValueReader } -// newIndexWriter constructs the index writer for the specified state. -func newIndexWriter(db ethdb.KeyValueReader, state stateIdent) (*indexWriter, error) { +// newIndexWriter constructs the index writer for the specified state. Additionally, +// it takes an integer as the limit and prunes all existing elements above that ID. +// It's essential as the recovery mechanism after unclean shutdown during the history +// indexing. +func newIndexWriter(db ethdb.KeyValueReader, state stateIdent, limit uint64, bitmapSize int) (*indexWriter, error) { blob := readStateIndex(state, db) if len(blob) == 0 { - desc := newIndexBlockDesc(0) - bw, _ := newBlockWriter(nil, desc) + desc := newIndexBlockDesc(0, bitmapSize) + bw, _ := newBlockWriter(nil, desc, 0 /* useless if the block is empty */, bitmapSize != 0) return &indexWriter{ - descList: []*indexBlockDesc{desc}, - bw: bw, - state: state, - db: db, + descList: []*indexBlockDesc{desc}, + bw: bw, + state: state, + db: db, + bitmapSize: bitmapSize, }, nil } - descList, err := parseIndex(blob) + descList, err := parseIndex(blob, bitmapSize) if err != nil { return nil, err } + // Trim trailing blocks whose elements all exceed the limit. + for i := len(descList) - 1; i > 0 && descList[i].max > limit; i-- { + // The previous block has the elements that exceed the limit, + // therefore the current block can be entirely dropped. + if descList[i-1].max >= limit { + descList = descList[:i] + } + } + // Take the last block for appending new elements lastDesc := descList[len(descList)-1] indexBlock := readStateIndexBlock(state, db, lastDesc.id) - bw, err := newBlockWriter(indexBlock, lastDesc) + + // Construct the writer for the last block. All elements in this block + // that exceed the limit will be truncated. + bw, err := newBlockWriter(indexBlock, lastDesc, limit, bitmapSize != 0) if err != nil { return nil, err } return &indexWriter{ - descList: descList, - lastID: lastDesc.max, - bw: bw, - state: state, - db: db, + descList: descList, + lastID: bw.last(), + bw: bw, + state: state, + db: db, + bitmapSize: bitmapSize, }, nil } // append adds the new element into the index writer. -func (w *indexWriter) append(id uint64) error { +func (w *indexWriter) append(id uint64, ext []uint16) error { if id <= w.lastID { return fmt.Errorf("append element out of order, last: %d, this: %d", w.lastID, id) } - if w.bw.full() { + if w.bw.estimateFull(ext) { if err := w.rotate(); err != nil { return err } } - if err := w.bw.append(id); err != nil { + if err := w.bw.append(id, ext); err != nil { return err } w.lastID = id @@ -215,10 +226,10 @@ func (w *indexWriter) append(id uint64) error { func (w *indexWriter) rotate() error { var ( err error - desc = newIndexBlockDesc(w.bw.desc.id + 1) + desc = newIndexBlockDesc(w.bw.desc.id+1, w.bitmapSize) ) w.frozen = append(w.frozen, w.bw) - w.bw, err = newBlockWriter(nil, desc) + w.bw, err = newBlockWriter(nil, desc, 0 /* useless if the block is empty */, w.bitmapSize != 0) if err != nil { return err } @@ -250,7 +261,8 @@ func (w *indexWriter) finish(batch ethdb.Batch) { } w.frozen = nil // release all the frozen writers - buf := make([]byte, 0, indexBlockDescSize*len(descList)) + size := indexBlockDescSize + w.bitmapSize + buf := make([]byte, 0, size*len(descList)) for _, desc := range descList { buf = append(buf, desc.encode()...) } @@ -259,49 +271,64 @@ func (w *indexWriter) finish(batch ethdb.Batch) { // indexDeleter is responsible for deleting index data for a specific state. type indexDeleter struct { - descList []*indexBlockDesc // The list of index block descriptions - bw *blockWriter // The live index block writer - dropped []uint32 // The list of index block id waiting for deleting - lastID uint64 // The ID of the latest tracked history - state stateIdent - db ethdb.KeyValueReader + descList []*indexBlockDesc // The list of index block descriptions + bw *blockWriter // The live index block writer + dropped []uint32 // The list of index block id waiting for deleting + lastID uint64 // The ID of the latest tracked history + state stateIdent // The identifier of the state being indexed + bitmapSize int // The size of optional extension bitmap + db ethdb.KeyValueReader } // newIndexDeleter constructs the index deleter for the specified state. -func newIndexDeleter(db ethdb.KeyValueReader, state stateIdent) (*indexDeleter, error) { +func newIndexDeleter(db ethdb.KeyValueReader, state stateIdent, limit uint64, bitmapSize int) (*indexDeleter, error) { blob := readStateIndex(state, db) if len(blob) == 0 { // TODO(rjl493456442) we can probably return an error here, // deleter with no data is meaningless. - desc := newIndexBlockDesc(0) - bw, _ := newBlockWriter(nil, desc) + desc := newIndexBlockDesc(0, bitmapSize) + bw, _ := newBlockWriter(nil, desc, 0 /* useless if the block is empty */, bitmapSize != 0) return &indexDeleter{ - descList: []*indexBlockDesc{desc}, - bw: bw, - state: state, - db: db, + descList: []*indexBlockDesc{desc}, + bw: bw, + state: state, + bitmapSize: bitmapSize, + db: db, }, nil } - descList, err := parseIndex(blob) + descList, err := parseIndex(blob, bitmapSize) if err != nil { return nil, err } + // Trim trailing blocks whose elements all exceed the limit. + for i := len(descList) - 1; i > 0 && descList[i].max > limit; i-- { + // The previous block has the elements that exceed the limit, + // therefore the current block can be entirely dropped. + if descList[i-1].max >= limit { + descList = descList[:i] + } + } + // Take the block for deleting element from lastDesc := descList[len(descList)-1] indexBlock := readStateIndexBlock(state, db, lastDesc.id) - bw, err := newBlockWriter(indexBlock, lastDesc) + + // Construct the writer for the last block. All elements in this block + // that exceed the limit will be truncated. + bw, err := newBlockWriter(indexBlock, lastDesc, limit, bitmapSize != 0) if err != nil { return nil, err } return &indexDeleter{ - descList: descList, - lastID: lastDesc.max, - bw: bw, - state: state, - db: db, + descList: descList, + lastID: bw.last(), + bw: bw, + state: state, + bitmapSize: bitmapSize, + db: db, }, nil } -// empty returns an flag indicating whether the state index is empty. +// empty returns whether the state index is empty. func (d *indexDeleter) empty() bool { return d.bw.empty() && len(d.descList) == 1 } @@ -334,7 +361,7 @@ func (d *indexDeleter) pop(id uint64) error { // Open the previous block writer for deleting lastDesc := d.descList[len(d.descList)-1] indexBlock := readStateIndexBlock(d.state, d.db, lastDesc.id) - bw, err := newBlockWriter(indexBlock, lastDesc) + bw, err := newBlockWriter(indexBlock, lastDesc, lastDesc.max, d.bitmapSize != 0) if err != nil { return err } @@ -360,7 +387,8 @@ func (d *indexDeleter) finish(batch ethdb.Batch) { if d.empty() { deleteStateIndex(d.state, batch) } else { - buf := make([]byte, 0, indexBlockDescSize*len(d.descList)) + size := indexBlockDescSize + d.bitmapSize + buf := make([]byte, 0, size*len(d.descList)) for _, desc := range d.descList { buf = append(buf, desc.encode()...) } diff --git a/triedb/pathdb/history_index_block.go b/triedb/pathdb/history_index_block.go index 5abdee682a..bb823bb13f 100644 --- a/triedb/pathdb/history_index_block.go +++ b/triedb/pathdb/history_index_block.go @@ -17,31 +17,37 @@ package pathdb import ( + "bytes" "encoding/binary" "errors" "fmt" "math" - "sort" + + "github.com/ethereum/go-ethereum/log" ) const ( - indexBlockDescSize = 14 // The size of index block descriptor - indexBlockEntriesCap = 4096 // The maximum number of entries can be grouped in a block - indexBlockRestartLen = 256 // The restart interval length of index block - historyIndexBatch = 512 * 1024 // The number of state history indexes for constructing or deleting as batch + indexBlockDescSize = 14 // The size of index block descriptor + indexBlockMaxSize = 4096 // The maximum size of a single index block + indexBlockRestartLen = 256 // The restart interval length of index block ) // indexBlockDesc represents a descriptor for an index block, which contains a // list of state mutation records associated with a specific state (either an // account or a storage slot). type indexBlockDesc struct { - max uint64 // The maximum state ID retained within the block - entries uint16 // The number of state mutation records retained within the block - id uint32 // The id of the index block + max uint64 // The maximum state ID retained within the block + entries uint16 // The number of state mutation records retained within the block + id uint32 // The id of the index block + extBitmap []byte // Optional fixed-size bitmap for the included extension elements } -func newIndexBlockDesc(id uint32) *indexBlockDesc { - return &indexBlockDesc{id: id} +func newIndexBlockDesc(id uint32, bitmapSize int) *indexBlockDesc { + var bitmap []byte + if bitmapSize > 0 { + bitmap = make([]byte, bitmapSize) + } + return &indexBlockDesc{id: id, extBitmap: bitmap} } // empty indicates whether the block is empty with no element retained. @@ -49,26 +55,33 @@ func (d *indexBlockDesc) empty() bool { return d.entries == 0 } -// full indicates whether the number of elements in the block exceeds the -// preconfigured limit. -func (d *indexBlockDesc) full() bool { - return d.entries >= indexBlockEntriesCap -} - // encode packs index block descriptor into byte stream. func (d *indexBlockDesc) encode() []byte { - var buf [indexBlockDescSize]byte + buf := make([]byte, indexBlockDescSize+len(d.extBitmap)) binary.BigEndian.PutUint64(buf[0:8], d.max) binary.BigEndian.PutUint16(buf[8:10], d.entries) binary.BigEndian.PutUint32(buf[10:14], d.id) + copy(buf[indexBlockDescSize:], d.extBitmap) return buf[:] } -// decode unpacks index block descriptor from byte stream. +// decode unpacks index block descriptor from byte stream. It's unsafe to mutate +// the provided byte stream after the function call. func (d *indexBlockDesc) decode(blob []byte) { d.max = binary.BigEndian.Uint64(blob[:8]) d.entries = binary.BigEndian.Uint16(blob[8:10]) d.id = binary.BigEndian.Uint32(blob[10:14]) + d.extBitmap = blob[indexBlockDescSize:] // no-deep copy! +} + +// copy returns a deep-copied object. +func (d *indexBlockDesc) copy() *indexBlockDesc { + return &indexBlockDesc{ + max: d.max, + entries: d.entries, + id: d.id, + extBitmap: bytes.Clone(d.extBitmap), + } } // parseIndexBlock parses the index block with the supplied byte stream. @@ -96,20 +109,38 @@ func (d *indexBlockDesc) decode(blob []byte) { // A uint16 can cover offsets in the range [0, 65536), which is more than enough // to store 4096 integers. // -// Each chunk begins with the full value of the first integer, followed by -// subsequent integers representing the differences between the current value -// and the preceding one. Integers are encoded with variable-size for best -// storage efficiency. Each chunk can be illustrated as below. +// Each chunk begins with a full integer value for the first element, followed +// by subsequent integers encoded as differences (deltas) from their preceding +// values. All integers use variable-length encoding for optimal space efficiency. +// +// In the updated format, each element in the chunk may optionally include an +// "extension" section. If an extension is present, it starts with a var-size +// integer indicating the length of the remaining extension payload, followed by +// that many bytes. If no extension is present, the element format is identical +// to the original version (i.e., only the integer or delta value is encoded). // -// Restart ---> +----------------+ -// | Full integer | -// +----------------+ -// | Diff with prev | -// +----------------+ -// | ... | -// +----------------+ -// | Diff with prev | -// +----------------+ +// In the trienode history index, the extension field contains the list of +// trie node IDs that fall within this range. For the given state transition, +// these IDs represent the specific nodes in this range that were mutated. +// +// Whether an element includes an extension is determined by the block reader +// based on the specification. Conceptually, a chunk is structured as: +// +// Restart ---> +----------------+ +// | Full integer | +// +----------------+ +// | (Extension?) | +// +----------------+ +// | Diff with prev | +// +----------------+ +// | (Extension?) | +// +----------------+ +// | ... | +// +----------------+ +// | Diff with prev | +// +----------------+ +// | (Extension?) | +// +----------------+ // // Empty index block is regarded as invalid. func parseIndexBlock(blob []byte) ([]uint16, []byte, error) { @@ -147,104 +178,98 @@ func parseIndexBlock(blob []byte) ([]uint16, []byte, error) { type blockReader struct { restarts []uint16 data []byte + hasExt bool } // newBlockReader constructs the block reader with the supplied block data. -func newBlockReader(blob []byte) (*blockReader, error) { +func newBlockReader(blob []byte, hasExt bool) (*blockReader, error) { restarts, data, err := parseIndexBlock(blob) if err != nil { return nil, err } return &blockReader{ restarts: restarts, - data: data, // safe to own the slice + data: data, // safe to own the slice + hasExt: hasExt, // flag whether extension should be resolved }, nil } // readGreaterThan locates the first element in the block that is greater than // the specified value. If no such element is found, MaxUint64 is returned. func (br *blockReader) readGreaterThan(id uint64) (uint64, error) { - var err error - index := sort.Search(len(br.restarts), func(i int) bool { - item, n := binary.Uvarint(br.data[br.restarts[i]:]) - if n <= 0 { - err = fmt.Errorf("failed to decode item at restart %d", br.restarts[i]) - } - return item > id - }) - if err != nil { + it := br.newIterator(nil) + found := it.SeekGT(id) + if err := it.Error(); err != nil { return 0, err } - if index == 0 { - item, _ := binary.Uvarint(br.data[br.restarts[0]:]) - return item, nil - } - var ( - start int - limit int - result uint64 - ) - if index == len(br.restarts) { - // The element being searched falls within the last restart section, - // there is no guarantee such element can be found. - start = int(br.restarts[len(br.restarts)-1]) - limit = len(br.data) - } else { - // The element being searched falls within the non-last restart section, - // such element can be found for sure. - start = int(br.restarts[index-1]) - limit = int(br.restarts[index]) - } - pos := start - for pos < limit { - x, n := binary.Uvarint(br.data[pos:]) - if pos == start { - result = x - } else { - result += x - } - if result > id { - return result, nil - } - pos += n - } - // The element which is greater than specified id is not found. - if index == len(br.restarts) { + if !found { return math.MaxUint64, nil } - // The element which is the first one greater than the specified id - // is exactly the one located at the restart point. - item, _ := binary.Uvarint(br.data[br.restarts[index]:]) - return item, nil + return it.ID(), nil } type blockWriter struct { desc *indexBlockDesc // Descriptor of the block restarts []uint16 // Offsets into the data slice, marking the start of each section data []byte // Aggregated encoded data slice + hasExt bool // Flag whether the extension field for each element exists } -func newBlockWriter(blob []byte, desc *indexBlockDesc) (*blockWriter, error) { +// newBlockWriter constructs a block writer. In addition to the existing data +// and block description, it takes an element ID and prunes all existing elements +// above that ID. It's essential as the recovery mechanism after unclean shutdown +// during the history indexing. +func newBlockWriter(blob []byte, desc *indexBlockDesc, limit uint64, hasExt bool) (*blockWriter, error) { if len(blob) == 0 { return &blockWriter{ - desc: desc, - data: make([]byte, 0, 1024), + desc: desc, + data: make([]byte, 0, 1024), + hasExt: hasExt, }, nil } restarts, data, err := parseIndexBlock(blob) if err != nil { return nil, err } - return &blockWriter{ + writer := &blockWriter{ desc: desc, restarts: restarts, data: data, // safe to own the slice - }, nil + hasExt: hasExt, + } + var trimmed int + for !writer.empty() && writer.last() > limit { + if err := writer.pop(writer.last()); err != nil { + return nil, err + } + trimmed += 1 + } + if trimmed > 0 { + log.Debug("Truncated extraneous elements", "count", trimmed, "limit", limit) + } + return writer, nil +} + +// setBitmap applies the given extension elements into the bitmap. +func (b *blockWriter) setBitmap(ext []uint16) { + for _, n := range ext { + // Node ID zero is intentionally filtered out. Any element in this range + // can indicate that the sub-tree's root node was mutated, so storing zero + // is redundant and saves one byte for bitmap. + if n != 0 { + setBit(b.desc.extBitmap, int(n-1)) + } + } } // append adds a new element to the block. The new element must be greater than // the previous one. The provided ID is assumed to always be greater than 0. -func (b *blockWriter) append(id uint64) error { +// +// ext refers to the optional extension field attached to the appended element. +// This extension mechanism is used by trie-node history and represents a list of +// trie node IDs that fall within the range covered by the index element +// (typically corresponding to a sub-trie in trie-node history). +func (b *blockWriter) append(id uint64, ext []uint16) error { if id == 0 { return errors.New("invalid zero id") } @@ -271,13 +296,29 @@ func (b *blockWriter) append(id uint64) error { // element. b.data = binary.AppendUvarint(b.data, id-b.desc.max) } + // Extension validation + if (len(ext) == 0) != !b.hasExt { + if len(ext) == 0 { + return errors.New("missing extension") + } + return errors.New("unexpected extension") + } + // Append the extension if it is not nil. The extension is prefixed with a + // length indicator, and the block reader MUST understand this scheme and + // decode the extension accordingly. + if len(ext) > 0 { + b.setBitmap(ext) + enc := encodeIDs(ext) + b.data = binary.AppendUvarint(b.data, uint64(len(enc))) + b.data = append(b.data, enc...) + } b.desc.entries++ b.desc.max = id return nil } // scanSection traverses the specified section and terminates if fn returns true. -func (b *blockWriter) scanSection(section int, fn func(uint64, int) bool) { +func (b *blockWriter) scanSection(section int, fn func(uint64, int, []uint16) bool) error { var ( value uint64 start = int(b.restarts[section]) @@ -296,27 +337,47 @@ func (b *blockWriter) scanSection(section int, fn func(uint64, int) bool) { } else { value += x } - if fn(value, pos) { - return + // Resolve the extension if exists + var ( + err error + ext []uint16 + extLen int + ) + if b.hasExt { + l, ln := binary.Uvarint(b.data[pos+n:]) + extLen = ln + int(l) + ext, err = decodeIDs(b.data[pos+n+ln : pos+n+extLen]) + } + if err != nil { + return err + } + if fn(value, pos, ext) { + return nil } + // Shift to next position pos += n + pos += extLen } + return nil } // sectionLast returns the last element in the specified section. -func (b *blockWriter) sectionLast(section int) uint64 { +func (b *blockWriter) sectionLast(section int) (uint64, error) { var n uint64 - b.scanSection(section, func(v uint64, _ int) bool { + if err := b.scanSection(section, func(v uint64, _ int, _ []uint16) bool { n = v return false - }) - return n + }); err != nil { + return 0, err + } + return n, nil } // sectionSearch looks up the specified value in the given section, // the position and the preceding value will be returned if found. -func (b *blockWriter) sectionSearch(section int, n uint64) (found bool, prev uint64, pos int) { - b.scanSection(section, func(v uint64, p int) bool { +// It assumes that the preceding element exists in the section. +func (b *blockWriter) sectionSearch(section int, n uint64) (found bool, prev uint64, pos int, err error) { + if err := b.scanSection(section, func(v uint64, p int, _ []uint16) bool { if n == v { pos = p found = true @@ -324,8 +385,24 @@ func (b *blockWriter) sectionSearch(section int, n uint64) (found bool, prev uin } prev = v return false // continue iteration - }) - return found, prev, pos + }); err != nil { + return false, 0, 0, err + } + return found, prev, pos, nil +} + +// rebuildBitmap scans the entire block and rebuilds the bitmap. +func (b *blockWriter) rebuildBitmap() error { + clear(b.desc.extBitmap) + for i := 0; i < len(b.restarts); i++ { + if err := b.scanSection(i, func(v uint64, p int, ext []uint16) bool { + b.setBitmap(ext) + return false // continue iteration + }); err != nil { + return err + } + } + return nil } // pop removes the last element from the block. The assumption is held that block @@ -339,9 +416,9 @@ func (b *blockWriter) pop(id uint64) error { } // If there is only one entry left, the entire block should be reset if b.desc.entries == 1 { - //b.desc.min = 0 b.desc.max = 0 b.desc.entries = 0 + clear(b.desc.extBitmap) b.restarts = nil b.data = b.data[:0] return nil @@ -351,28 +428,45 @@ func (b *blockWriter) pop(id uint64) error { if b.desc.entries%indexBlockRestartLen == 1 { b.data = b.data[:b.restarts[len(b.restarts)-1]] b.restarts = b.restarts[:len(b.restarts)-1] - b.desc.max = b.sectionLast(len(b.restarts) - 1) + last, err := b.sectionLast(len(b.restarts) - 1) + if err != nil { + return err + } + b.desc.max = last b.desc.entries -= 1 - return nil + return b.rebuildBitmap() } // Look up the element preceding the one to be popped, in order to update // the maximum element in the block. - found, prev, pos := b.sectionSearch(len(b.restarts)-1, id) + found, prev, pos, err := b.sectionSearch(len(b.restarts)-1, id) + if err != nil { + return err + } if !found { return fmt.Errorf("pop element is not found, last: %d, this: %d", b.desc.max, id) } b.desc.max = prev b.data = b.data[:pos] b.desc.entries -= 1 - return nil + return b.rebuildBitmap() } func (b *blockWriter) empty() bool { return b.desc.empty() } -func (b *blockWriter) full() bool { - return b.desc.full() +func (b *blockWriter) estimateFull(ext []uint16) bool { + size := 8 + 2*len(ext) + return len(b.data)+size > indexBlockMaxSize +} + +// last returns the last element in the block. It should only be called when +// writer is not empty, otherwise the returned data is meaningless. +func (b *blockWriter) last() uint64 { + if b.empty() { + return 0 + } + return b.desc.max } // finish finalizes the index block encoding by appending the encoded restart points diff --git a/triedb/pathdb/history_index_block_test.go b/triedb/pathdb/history_index_block_test.go index c251cea2ec..923ae29348 100644 --- a/triedb/pathdb/history_index_block_test.go +++ b/triedb/pathdb/history_index_block_test.go @@ -17,6 +17,7 @@ package pathdb import ( + "bytes" "math" "math/rand" "slices" @@ -24,16 +25,36 @@ import ( "testing" ) +func randomExt(bitmapSize int, n int) []uint16 { + if bitmapSize == 0 { + return nil + } + var ( + limit = bitmapSize * 8 + extList []uint16 + ) + for i := 0; i < n; i++ { + extList = append(extList, uint16(rand.Intn(limit+1))) + } + return extList +} + func TestBlockReaderBasic(t *testing.T) { + testBlockReaderBasic(t, 0) + testBlockReaderBasic(t, 2) + testBlockReaderBasic(t, 34) +} + +func testBlockReaderBasic(t *testing.T, bitmapSize int) { elements := []uint64{ 1, 5, 10, 11, 20, } - bw, _ := newBlockWriter(nil, newIndexBlockDesc(0)) + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, bitmapSize), 0, bitmapSize != 0) for i := 0; i < len(elements); i++ { - bw.append(elements[i]) + bw.append(elements[i], randomExt(bitmapSize, 5)) } - br, err := newBlockReader(bw.finish()) + br, err := newBlockReader(bw.finish(), bitmapSize != 0) if err != nil { t.Fatalf("Failed to construct the block reader, %v", err) } @@ -60,18 +81,24 @@ func TestBlockReaderBasic(t *testing.T) { } func TestBlockReaderLarge(t *testing.T) { + testBlockReaderLarge(t, 0) + testBlockReaderLarge(t, 2) + testBlockReaderLarge(t, 34) +} + +func testBlockReaderLarge(t *testing.T, bitmapSize int) { var elements []uint64 for i := 0; i < 1000; i++ { elements = append(elements, rand.Uint64()) } slices.Sort(elements) - bw, _ := newBlockWriter(nil, newIndexBlockDesc(0)) + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, bitmapSize), 0, bitmapSize != 0) for i := 0; i < len(elements); i++ { - bw.append(elements[i]) + bw.append(elements[i], randomExt(bitmapSize, 5)) } - br, err := newBlockReader(bw.finish()) + br, err := newBlockReader(bw.finish(), bitmapSize != 0) if err != nil { t.Fatalf("Failed to construct the block reader, %v", err) } @@ -95,34 +122,91 @@ func TestBlockReaderLarge(t *testing.T) { } func TestBlockWriterBasic(t *testing.T) { - bw, _ := newBlockWriter(nil, newIndexBlockDesc(0)) + testBlockWriteBasic(t, 0) + testBlockWriteBasic(t, 2) + testBlockWriteBasic(t, 34) +} + +func testBlockWriteBasic(t *testing.T, bitmapSize int) { + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, bitmapSize), 0, bitmapSize != 0) if !bw.empty() { t.Fatal("expected empty block") } - bw.append(2) - if err := bw.append(1); err == nil { + bw.append(2, randomExt(bitmapSize, 5)) + if err := bw.append(1, randomExt(bitmapSize, 5)); err == nil { t.Fatal("out-of-order insertion is not expected") } + var maxElem uint64 for i := 0; i < 10; i++ { - bw.append(uint64(i + 3)) + bw.append(uint64(i+3), randomExt(bitmapSize, 5)) + maxElem = uint64(i + 3) } - bw, err := newBlockWriter(bw.finish(), newIndexBlockDesc(0)) + bw, err := newBlockWriter(bw.finish(), newIndexBlockDesc(0, bitmapSize), maxElem, bitmapSize != 0) if err != nil { t.Fatalf("Failed to construct the block writer, %v", err) } for i := 0; i < 10; i++ { - if err := bw.append(uint64(i + 100)); err != nil { + if err := bw.append(uint64(i+100), randomExt(bitmapSize, 5)); err != nil { t.Fatalf("Failed to append value %d: %v", i, err) } } bw.finish() } +func TestBlockWriterWithLimit(t *testing.T) { + testBlockWriterWithLimit(t, 0) + testBlockWriterWithLimit(t, 2) + testBlockWriterWithLimit(t, 34) +} + +func testBlockWriterWithLimit(t *testing.T, bitmapSize int) { + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, bitmapSize), 0, bitmapSize != 0) + + var bitmaps [][]byte + for i := 0; i < indexBlockRestartLen+2; i++ { + bw.append(uint64(i+1), randomExt(bitmapSize, 5)) + bitmaps = append(bitmaps, bytes.Clone(bw.desc.extBitmap)) + } + for i := 0; i < indexBlockRestartLen+2; i++ { + limit := uint64(i + 1) + + desc := bw.desc.copy() + block, err := newBlockWriter(bytes.Clone(bw.finish()), desc, limit, bitmapSize != 0) + if err != nil { + t.Fatalf("Failed to construct the block writer, %v", err) + } + if block.desc.max != limit { + t.Fatalf("Test %d, unexpected max value, got %d, want %d", i, block.desc.max, limit) + } + if !bytes.Equal(desc.extBitmap, bitmaps[i]) { + t.Fatalf("Test %d, unexpected bitmap, got: %v, want: %v", i, block.desc.extBitmap, bitmaps[i]) + } + + // Re-fill the elements + var maxElem uint64 + for elem := limit + 1; elem < indexBlockRestartLen+4; elem++ { + if err := block.append(elem, randomExt(bitmapSize, 5)); err != nil { + t.Fatalf("Failed to append value %d: %v", elem, err) + } + maxElem = elem + } + if block.desc.max != maxElem { + t.Fatalf("Test %d, unexpected max value, got %d, want %d", i, block.desc.max, maxElem) + } + } +} + func TestBlockWriterDelete(t *testing.T) { - bw, _ := newBlockWriter(nil, newIndexBlockDesc(0)) + testBlockWriterDelete(t, 0) + testBlockWriterDelete(t, 2) + testBlockWriterDelete(t, 34) +} + +func testBlockWriterDelete(t *testing.T, bitmapSize int) { + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, bitmapSize), 0, bitmapSize != 0) for i := 0; i < 10; i++ { - bw.append(uint64(i + 1)) + bw.append(uint64(i+1), randomExt(bitmapSize, 5)) } // Pop unknown id, the request should be rejected if err := bw.pop(100); err == nil { @@ -144,12 +228,18 @@ func TestBlockWriterDelete(t *testing.T) { } func TestBlcokWriterDeleteWithData(t *testing.T) { + testBlcokWriterDeleteWithData(t, 0) + testBlcokWriterDeleteWithData(t, 2) + testBlcokWriterDeleteWithData(t, 34) +} + +func testBlcokWriterDeleteWithData(t *testing.T, bitmapSize int) { elements := []uint64{ 1, 5, 10, 11, 20, } - bw, _ := newBlockWriter(nil, newIndexBlockDesc(0)) + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, bitmapSize), 0, bitmapSize != 0) for i := 0; i < len(elements); i++ { - bw.append(elements[i]) + bw.append(elements[i], randomExt(bitmapSize, 5)) } // Re-construct the block writer with data @@ -158,7 +248,10 @@ func TestBlcokWriterDeleteWithData(t *testing.T) { max: 20, entries: 5, } - bw, err := newBlockWriter(bw.finish(), desc) + if bitmapSize > 0 { + desc.extBitmap = make([]byte, bitmapSize) + } + bw, err := newBlockWriter(bw.finish(), desc, elements[len(elements)-1], bitmapSize != 0) if err != nil { t.Fatalf("Failed to construct block writer %v", err) } @@ -169,7 +262,7 @@ func TestBlcokWriterDeleteWithData(t *testing.T) { newTail := elements[i-1] // Ensure the element can still be queried with no issue - br, err := newBlockReader(bw.finish()) + br, err := newBlockReader(bw.finish(), bitmapSize != 0) if err != nil { t.Fatalf("Failed to construct the block reader, %v", err) } @@ -201,26 +294,60 @@ func TestBlcokWriterDeleteWithData(t *testing.T) { } func TestCorruptedIndexBlock(t *testing.T) { - bw, _ := newBlockWriter(nil, newIndexBlockDesc(0)) + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, 0), 0, false) + + var maxElem uint64 for i := 0; i < 10; i++ { - bw.append(uint64(i + 1)) + bw.append(uint64(i+1), nil) + maxElem = uint64(i + 1) } buf := bw.finish() // Mutate the buffer manually buf[len(buf)-1]++ - _, err := newBlockWriter(buf, newIndexBlockDesc(0)) + _, err := newBlockWriter(buf, newIndexBlockDesc(0, 0), maxElem, false) if err == nil { t.Fatal("Corrupted index block data is not detected") } } // BenchmarkParseIndexBlock benchmarks the performance of parseIndexBlock. +// +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/triedb/pathdb +// cpu: Apple M1 Pro +// BenchmarkParseIndexBlock +// BenchmarkParseIndexBlock-8 35829495 34.16 ns/op func BenchmarkParseIndexBlock(b *testing.B) { // Generate a realistic index block blob - bw, _ := newBlockWriter(nil, newIndexBlockDesc(0)) + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, 0), 0, false) for i := 0; i < 4096; i++ { - bw.append(uint64(i * 2)) + bw.append(uint64(i*2), nil) + } + blob := bw.finish() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _, err := parseIndexBlock(blob) + if err != nil { + b.Fatalf("parseIndexBlock failed: %v", err) + } + } +} + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/triedb/pathdb +// cpu: Apple M1 Pro +// BenchmarkParseIndexBlockWithExt +// BenchmarkParseIndexBlockWithExt-8 35773242 33.72 ns/op +func BenchmarkParseIndexBlockWithExt(b *testing.B) { + // Generate a realistic index block blob + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, 34), 0, true) + for i := 0; i < 4096; i++ { + id, ext := uint64(i*2), randomExt(34, 3) + bw.append(id, ext) } blob := bw.finish() @@ -234,19 +361,58 @@ func BenchmarkParseIndexBlock(b *testing.B) { } // BenchmarkBlockWriterAppend benchmarks the performance of indexblock.writer +// +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/triedb/pathdb +// cpu: Apple M1 Pro +// BenchmarkBlockWriterAppend +// BenchmarkBlockWriterAppend-8 293611083 4.113 ns/op 3 B/op 0 allocs/op func BenchmarkBlockWriterAppend(b *testing.B) { b.ReportAllocs() b.ResetTimer() - desc := newIndexBlockDesc(0) - writer, _ := newBlockWriter(nil, desc) + var blockID uint32 + desc := newIndexBlockDesc(blockID, 0) + writer, _ := newBlockWriter(nil, desc, 0, false) + + for i := 0; i < b.N; i++ { + if writer.estimateFull(nil) { + blockID += 1 + desc = newIndexBlockDesc(blockID, 0) + writer, _ = newBlockWriter(nil, desc, 0, false) + } + if err := writer.append(writer.desc.max+1, nil); err != nil { + b.Error(err) + } + } +} + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/triedb/pathdb +// cpu: Apple M1 Pro +// BenchmarkBlockWriterAppendWithExt +// BenchmarkBlockWriterAppendWithExt-8 11123844 103.6 ns/op 42 B/op 2 allocs/op +func BenchmarkBlockWriterAppendWithExt(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + + var ( + bitmapSize = 34 + blockID uint32 + ) + desc := newIndexBlockDesc(blockID, bitmapSize) + writer, _ := newBlockWriter(nil, desc, 0, true) for i := 0; i < b.N; i++ { - if writer.full() { - desc = newIndexBlockDesc(0) - writer, _ = newBlockWriter(nil, desc) + ext := randomExt(bitmapSize, 3) + if writer.estimateFull(ext) { + blockID += 1 + desc = newIndexBlockDesc(blockID, bitmapSize) + writer, _ = newBlockWriter(nil, desc, 0, true) } - if err := writer.append(writer.desc.max + 1); err != nil { + if err := writer.append(writer.desc.max+1, ext); err != nil { b.Error(err) } } diff --git a/triedb/pathdb/history_index_iterator.go b/triedb/pathdb/history_index_iterator.go new file mode 100644 index 0000000000..e4aca24f5d --- /dev/null +++ b/triedb/pathdb/history_index_iterator.go @@ -0,0 +1,656 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see = bitmapElementThresholdTwoLevels { + return false, fmt.Errorf("invalid extension filter %d for 2 bytes bitmap", id) + } + return isBitSet(bitmap, n), nil + case bitmapBytesThreeLevels: + // Bitmap for 3-level trie with at most 16+16*16 elements inside + if n >= bitmapElementThresholdThreeLevels { + return false, fmt.Errorf("invalid extension filter %d for 34 bytes bitmap", id) + } else if n >= bitmapElementThresholdTwoLevels { + return isBitSet(bitmap, n), nil + } else { + // Check the element itself first + if isBitSet(bitmap, n) { + return true, nil + } + // Check descendants: the presence of any descendant implicitly + // represents a mutation of its ancestor. + return bitmap[2+2*n] != 0 || bitmap[3+2*n] != 0, nil + } + default: + return false, fmt.Errorf("unsupported bitmap size %d", len(bitmap)) + } +} + +// blockIterator is the iterator to traverse the indices within a single block. +type blockIterator struct { + // immutable fields + data []byte // Reference to the data segment within the block reader + restarts []uint16 // Offsets pointing to the restart sections within the data + hasExt bool // Flag whether the extension is included in the data + + // Optional extension filter + filter *extFilter // Filters index entries based on the extension field. + + // mutable fields + id uint64 // ID of the element at the iterators current position + ext []byte // Extension field of the element at the iterators current position + dataPtr int // Current read position within the data slice + restartPtr int // Index of the restart section where the iterator is currently positioned + exhausted bool // Flag whether the iterator has been exhausted + err error // Accumulated error during the traversal +} + +func (br *blockReader) newIterator(filter *extFilter) *blockIterator { + it := &blockIterator{ + data: br.data, // hold the slice directly with no deep copy + restarts: br.restarts, // hold the slice directly with no deep copy + hasExt: br.hasExt, // flag whether the extension should be resolved + filter: filter, // optional extension filter + } + it.reset() + return it +} + +func (it *blockIterator) set(dataPtr int, restartPtr int, id uint64, ext []byte) { + it.id = id + it.ext = ext + + it.dataPtr = dataPtr + it.restartPtr = restartPtr + it.exhausted = dataPtr == len(it.data) +} + +func (it *blockIterator) setErr(err error) { + if it.err != nil { + return + } + it.err = err +} + +func (it *blockIterator) reset() { + it.id = 0 + it.ext = nil + + it.dataPtr = -1 + it.restartPtr = -1 + it.exhausted = false + it.err = nil + + // Mark the iterator as exhausted if the associated index block is empty + if len(it.data) == 0 || len(it.restarts) == 0 { + it.exhausted = true + } +} + +func (it *blockIterator) resolveExt(pos int) ([]byte, int, error) { + if !it.hasExt { + return nil, 0, nil + } + length, n := binary.Uvarint(it.data[pos:]) + if n <= 0 { + return nil, 0, fmt.Errorf("too short for extension, pos: %d, datalen: %d", pos, len(it.data)) + } + if len(it.data[pos+n:]) < int(length) { + return nil, 0, fmt.Errorf("too short for extension, pos: %d, length: %d, datalen: %d", pos, length, len(it.data)) + } + return it.data[pos+n : pos+n+int(length)], n + int(length), nil +} + +// seekGT moves the iterator to the first element whose id is greater than the +// given number. It returns whether such element exists. +// +// Note, this operation will unset the exhausted status and subsequent traversal +// is allowed. +func (it *blockIterator) seekGT(id uint64) bool { + if it.err != nil { + return false + } + var err error + index := sort.Search(len(it.restarts), func(i int) bool { + item, n := binary.Uvarint(it.data[it.restarts[i]:]) + if n <= 0 { + err = fmt.Errorf("failed to decode item at restart %d", it.restarts[i]) + } + return item > id + }) + if err != nil { + it.setErr(err) + return false + } + if index == 0 { + pos := int(it.restarts[0]) + item, n := binary.Uvarint(it.data[pos:]) + if n <= 0 { + it.setErr(fmt.Errorf("failed to decode item at pos %d", it.restarts[0])) + return false + } + pos = pos + n + + ext, shift, err := it.resolveExt(pos) + if err != nil { + it.setErr(err) + return false + } + it.set(pos+shift, 0, item, ext) + return true + } + var ( + start int + limit int + restartIndex int // The restart section being searched below + ) + if index == len(it.restarts) { + // The element being searched falls within the last restart section, + // there is no guarantee such element can be found. + start = int(it.restarts[len(it.restarts)-1]) + limit = len(it.data) + restartIndex = len(it.restarts) - 1 + } else { + // The element being searched falls within the non-last restart section, + // such element can be found for sure. + start = int(it.restarts[index-1]) + limit = int(it.restarts[index]) + restartIndex = index - 1 + } + var ( + result uint64 + pos = start + ) + for pos < limit { + x, n := binary.Uvarint(it.data[pos:]) + if n <= 0 { + it.setErr(fmt.Errorf("failed to decode item at pos %d", pos)) + return false + } + if pos == start { + result = x + } else { + result += x + } + pos += n + + ext, shift, err := it.resolveExt(pos) + if err != nil { + it.setErr(err) + return false + } + pos += shift + + if result > id { + if pos == limit { + it.set(pos, restartIndex+1, result, ext) + } else { + it.set(pos, restartIndex, result, ext) + } + return true + } + } + // The element which is greater than specified id is not found. + if index == len(it.restarts) { + it.reset() + return false + } + // The element which is the first one greater than the specified id + // is exactly the one located at the restart point. + pos = int(it.restarts[index]) + item, n := binary.Uvarint(it.data[pos:]) + if n <= 0 { + it.setErr(fmt.Errorf("failed to decode item at pos %d", it.restarts[index])) + return false + } + pos = pos + n + + ext, shift, err := it.resolveExt(pos) + if err != nil { + it.setErr(err) + return false + } + it.set(pos+shift, index, item, ext) + return true +} + +// SeekGT implements HistoryIndexIterator, is the wrapper of the seekGT with +// optional extension filter logic applied. +func (it *blockIterator) SeekGT(id uint64) bool { + if !it.seekGT(id) { + return false + } + if it.filter == nil { + return true + } + for { + found, err := it.filter.exists(it.ext) + if err != nil { + it.setErr(err) + return false + } + if found { + break + } + if !it.next() { + return false + } + } + return true +} + +func (it *blockIterator) init() { + if it.dataPtr != -1 { + return + } + it.dataPtr = 0 + it.restartPtr = 0 +} + +// next moves the iterator to the next element. If the iterator has been exhausted, +// and boolean with false should be returned. +func (it *blockIterator) next() bool { + if it.exhausted || it.err != nil { + return false + } + it.init() + + // Decode the next element pointed by the iterator + v, n := binary.Uvarint(it.data[it.dataPtr:]) + if n <= 0 { + it.setErr(fmt.Errorf("failed to decode item at pos %d", it.dataPtr)) + return false + } + var val uint64 + if it.dataPtr == int(it.restarts[it.restartPtr]) { + val = v + } else { + val = it.id + v + } + + // Decode the extension field + ext, shift, err := it.resolveExt(it.dataPtr + n) + if err != nil { + it.setErr(err) + return false + } + + // Move to the next restart section if the data pointer crosses the boundary + nextRestartPtr := it.restartPtr + if it.restartPtr < len(it.restarts)-1 && it.dataPtr+n+shift == int(it.restarts[it.restartPtr+1]) { + nextRestartPtr = it.restartPtr + 1 + } + it.set(it.dataPtr+n+shift, nextRestartPtr, val, ext) + + return true +} + +// Next implements the HistoryIndexIterator, moving the iterator to the next +// element. It's a wrapper of next with optional extension filter logic applied. +func (it *blockIterator) Next() bool { + if !it.next() { + return false + } + if it.filter == nil { + return true + } + for { + found, err := it.filter.exists(it.ext) + if err != nil { + it.setErr(err) + return false + } + if found { + break + } + if !it.next() { + return false + } + } + return true +} + +// ID implements HistoryIndexIterator, returning the id of the element where the +// iterator is positioned at. +func (it *blockIterator) ID() uint64 { + return it.id +} + +// Error implements HistoryIndexIterator, returning any accumulated error. +// Exhausting all the elements is not considered to be an error. +func (it *blockIterator) Error() error { return it.err } + +// indexIterator is an iterator to traverse the history indices belonging to the +// specific state entry. +type indexIterator struct { + // immutable fields + descList []*indexBlockDesc + reader *indexReader + + // Optional extension filter + filter *extFilter + + // mutable fields + blockIt *blockIterator + blockPtr int + exhausted bool + err error +} + +// newBlockIter initializes the block iterator with the specified block ID. +func (r *indexReader) newBlockIter(id uint32, filter *extFilter) (*blockIterator, error) { + br, ok := r.readers[id] + if !ok { + var err error + br, err = newBlockReader(readStateIndexBlock(r.state, r.db, id), r.bitmapSize != 0) + if err != nil { + return nil, err + } + r.readers[id] = br + } + return br.newIterator(filter), nil +} + +// newIterator initializes the index iterator with the specified extension filter. +func (r *indexReader) newIterator(filter *extFilter) *indexIterator { + it := &indexIterator{ + descList: r.descList, + reader: r, + filter: filter, + } + it.reset() + return it +} + +func (it *indexIterator) setErr(err error) { + if it.err != nil { + return + } + it.err = err +} + +func (it *indexIterator) reset() { + it.blockIt = nil + it.blockPtr = -1 + it.exhausted = false + it.err = nil + + if len(it.descList) == 0 { + it.exhausted = true + } +} + +func (it *indexIterator) open(blockPtr int) error { + blockIt, err := it.reader.newBlockIter(it.descList[blockPtr].id, it.filter) + if err != nil { + return err + } + it.blockIt = blockIt + it.blockPtr = blockPtr + return nil +} + +func (it *indexIterator) applyFilter(index int) (int, error) { + if it.filter == nil { + return index, nil + } + for index < len(it.descList) { + found, err := it.filter.contains(it.descList[index].extBitmap) + if err != nil { + return 0, err + } + if found { + break + } + index++ + } + return index, nil +} + +// SeekGT moves the iterator to the first element whose id is greater than the +// given number. It returns whether such element exists. +// +// Note, this operation will unset the exhausted status and subsequent traversal +// is allowed. +func (it *indexIterator) SeekGT(id uint64) bool { + if it.err != nil { + return false + } + index := sort.Search(len(it.descList), func(i int) bool { + return id < it.descList[i].max + }) + index, err := it.applyFilter(index) + if err != nil { + it.setErr(err) + return false + } + if index == len(it.descList) { + return false + } + it.exhausted = false + + if it.blockIt == nil || it.blockPtr != index { + if err := it.open(index); err != nil { + it.setErr(err) + return false + } + } + // Terminate if the element which is greater than the id can be found in the + // last block; otherwise move to the next block. It may happen that all the + // target elements in this block are all less than id. + if it.blockIt.SeekGT(id) { + return true + } + return it.Next() +} + +func (it *indexIterator) init() error { + if it.blockIt != nil { + return nil + } + return it.open(0) +} + +// Next implements the HistoryIndexIterator, moving the iterator to the next +// element. If the iterator has been exhausted, and boolean with false should +// be returned. +func (it *indexIterator) Next() bool { + if it.exhausted || it.err != nil { + return false + } + if err := it.init(); err != nil { + it.setErr(err) + return false + } + if it.blockIt.Next() { + return true + } + it.blockPtr++ + + index, err := it.applyFilter(it.blockPtr) + if err != nil { + it.setErr(err) + return false + } + it.blockPtr = index + + if it.blockPtr == len(it.descList) { + it.exhausted = true + return false + } + if err := it.open(it.blockPtr); err != nil { + it.setErr(err) + return false + } + return it.blockIt.Next() +} + +// Error implements HistoryIndexIterator, returning any accumulated error. +// Exhausting all the elements is not considered to be an error. +func (it *indexIterator) Error() error { + if it.err != nil { + return it.err + } + if it.blockIt != nil { + return it.blockIt.Error() + } + return nil +} + +// ID implements HistoryIndexIterator, returning the id of the element where the +// iterator is positioned at. +func (it *indexIterator) ID() uint64 { + return it.blockIt.ID() +} + +// seqIter provides a simple iterator over a contiguous sequence of +// unsigned integers, ending at end (end is included). +type seqIter struct { + cur uint64 // current position + end uint64 // iteration stops at end-1 + done bool // true when iteration is exhausted +} + +func newSeqIter(last uint64) *seqIter { + return &seqIter{end: last + 1} +} + +// SeekGT positions the iterator at the smallest element > id. +// Returns false if no such element exists. +func (it *seqIter) SeekGT(id uint64) bool { + if id+1 >= it.end { + it.done = true + return false + } + it.cur = id + 1 + it.done = false + return true +} + +// Next advances the iterator. Returns false if exhausted. +func (it *seqIter) Next() bool { + if it.done { + return false + } + if it.cur+1 < it.end { + it.cur++ + return true + } + // this was the last element + it.done = true + return false +} + +// ID returns the id of the element where the iterator is positioned at. +func (it *seqIter) ID() uint64 { return it.cur } + +// Error returns any accumulated error. Exhausting all the elements is not +// considered to be an error. +func (it *seqIter) Error() error { return nil } diff --git a/triedb/pathdb/history_index_iterator_test.go b/triedb/pathdb/history_index_iterator_test.go new file mode 100644 index 0000000000..a11fd17666 --- /dev/null +++ b/triedb/pathdb/history_index_iterator_test.go @@ -0,0 +1,361 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see 0 { + filter = &filters[rand.Intn(len(filters))] + } else { + filter = nil + } + + var input uint64 + if rand.Intn(2) == 0 { + input = elements[rand.Intn(len(elements))] + } else { + input = uint64(rand.Uint32()) + } + + index := sort.Search(len(elements), func(i int) bool { + return elements[i] > input + }) + for index < len(elements) { + if checkExt(filter, ext[index]) { + break + } + index++ + } + + var ( + exp bool + expVal uint64 + remains []uint64 + ) + if index == len(elements) { + exp = false + } else { + exp = true + expVal = elements[index] + + index++ + for index < len(elements) { + if checkExt(filter, ext[index]) { + remains = append(remains, elements[index]) + } + index++ + } + } + + it := newIter(filter) + if err := checkSeekGT(it, input, exp, expVal); err != nil { + t.Fatal(err) + } + if exp { + if err := checkNext(it, remains); err != nil { + t.Fatal(err) + } + } + } +} + +func verifyTraversal(t *testing.T, elements []uint64, ext [][]uint16, newIter func(filter *extFilter) HistoryIndexIterator) { + set := make(map[extFilter]bool) + for _, extList := range ext { + for _, f := range extList { + set[extFilter(f)] = true + } + } + filters := slices.Collect(maps.Keys(set)) + + for i := 0; i < 16; i++ { + var filter *extFilter + if len(filters) > 0 { + filter = &filters[rand.Intn(len(filters))] + } else { + filter = nil + } + it := newIter(filter) + + var ( + pos int + exp []uint64 + ) + for pos < len(elements) { + if checkExt(filter, ext[pos]) { + exp = append(exp, elements[pos]) + } + pos++ + } + if err := checkNext(it, exp); err != nil { + t.Fatal(err) + } + } +} + +func TestBlockIteratorSeekGT(t *testing.T) { + for _, size := range []int{0, 2, 34} { + for _, n := range []int{1, indexBlockRestartLen, 3 * indexBlockRestartLen} { + data, elements, ext := makeTestIndexBlock(n, size) + + verifySeekGT(t, elements, ext, func(filter *extFilter) HistoryIndexIterator { + br, err := newBlockReader(data, size != 0) + if err != nil { + t.Fatalf("Failed to open the block for reading, %v", err) + } + return br.newIterator(filter) + }) + } + } +} + +func TestIndexIteratorSeekGT(t *testing.T) { + ident := newAccountIdent(common.Hash{0x1}) + + for _, size := range []int{0, 2, 34} { + for _, n := range []int{1, 4096, 3 * 4096} { + db := rawdb.NewMemoryDatabase() + elements, ext := makeTestIndexBlocks(db, ident, n, size) + + verifySeekGT(t, elements, ext, func(filter *extFilter) HistoryIndexIterator { + ir, err := newIndexReader(db, ident, size) + if err != nil { + t.Fatalf("Failed to open the index reader, %v", err) + } + return ir.newIterator(filter) + }) + } + } +} + +func TestBlockIteratorTraversal(t *testing.T) { + /* 0-size index block is not allowed + + data, elements := makeTestIndexBlock(0) + testBlockIterator(t, data, elements) + */ + + for _, size := range []int{0, 2, 34} { + for _, n := range []int{1, indexBlockRestartLen, 3 * indexBlockRestartLen} { + data, elements, ext := makeTestIndexBlock(n, size) + + verifyTraversal(t, elements, ext, func(filter *extFilter) HistoryIndexIterator { + br, err := newBlockReader(data, size != 0) + if err != nil { + t.Fatalf("Failed to open the block for reading, %v", err) + } + return br.newIterator(filter) + }) + } + } +} + +func TestIndexIteratorTraversal(t *testing.T) { + ident := newAccountIdent(common.Hash{0x1}) + + for _, size := range []int{0, 2, 34} { + for _, n := range []int{1, 4096, 3 * 4096} { + db := rawdb.NewMemoryDatabase() + elements, ext := makeTestIndexBlocks(db, ident, n, size) + + verifyTraversal(t, elements, ext, func(filter *extFilter) HistoryIndexIterator { + ir, err := newIndexReader(db, ident, size) + if err != nil { + t.Fatalf("Failed to open the index reader, %v", err) + } + return ir.newIterator(filter) + }) + } + } +} + +func TestSeqIterBasicIteration(t *testing.T) { + it := newSeqIter(5) // iterates over [1..5] + it.SeekGT(0) + + var ( + got []uint64 + expected = []uint64{1, 2, 3, 4, 5} + ) + got = append(got, it.ID()) + for it.Next() { + got = append(got, it.ID()) + } + if len(got) != len(expected) { + t.Fatalf("iteration length mismatch: got %v, expected %v", got, expected) + } + for i := range expected { + if got[i] != expected[i] { + t.Fatalf("element mismatch at %d: got %d, expected %d", i, got[i], expected[i]) + } + } +} + +func TestSeqIterSeekGT(t *testing.T) { + it := newSeqIter(5) + + tests := []struct { + input uint64 + ok bool + expected uint64 + }{ + {0, true, 1}, + {1, true, 2}, + {4, true, 5}, + {5, false, 0}, // 6 is out of range + } + for _, tt := range tests { + ok := it.SeekGT(tt.input) + if ok != tt.ok { + t.Fatalf("SeekGT(%d) ok mismatch: got %v, expected %v", tt.input, ok, tt.ok) + } + if ok && it.ID() != tt.expected { + t.Fatalf("SeekGT(%d) positioned at %d, expected %d", tt.input, it.ID(), tt.expected) + } + } +} diff --git a/triedb/pathdb/history_index_test.go b/triedb/pathdb/history_index_test.go index be9b7c4049..2644db46b5 100644 --- a/triedb/pathdb/history_index_test.go +++ b/triedb/pathdb/history_index_test.go @@ -29,19 +29,25 @@ import ( ) func TestIndexReaderBasic(t *testing.T) { + testIndexReaderBasic(t, 0) + testIndexReaderBasic(t, 2) + testIndexReaderBasic(t, 34) +} + +func testIndexReaderBasic(t *testing.T, bitmapSize int) { elements := []uint64{ 1, 5, 10, 11, 20, } db := rawdb.NewMemoryDatabase() - bw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa})) + bw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0, bitmapSize) for i := 0; i < len(elements); i++ { - bw.append(elements[i]) + bw.append(elements[i], randomExt(bitmapSize, 5)) } batch := db.NewBatch() bw.finish(batch) batch.Write() - br, err := newIndexReader(db, newAccountIdent(common.Hash{0xa})) + br, err := newIndexReader(db, newAccountIdent(common.Hash{0xa}), bitmapSize) if err != nil { t.Fatalf("Failed to construct the index reader, %v", err) } @@ -68,22 +74,28 @@ func TestIndexReaderBasic(t *testing.T) { } func TestIndexReaderLarge(t *testing.T) { + testIndexReaderLarge(t, 0) + testIndexReaderLarge(t, 2) + testIndexReaderLarge(t, 34) +} + +func testIndexReaderLarge(t *testing.T, bitmapSize int) { var elements []uint64 - for i := 0; i < 10*indexBlockEntriesCap; i++ { + for i := 0; i < 10*4096; i++ { elements = append(elements, rand.Uint64()) } slices.Sort(elements) db := rawdb.NewMemoryDatabase() - bw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa})) + bw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0, bitmapSize) for i := 0; i < len(elements); i++ { - bw.append(elements[i]) + bw.append(elements[i], randomExt(bitmapSize, 5)) } batch := db.NewBatch() bw.finish(batch) batch.Write() - br, err := newIndexReader(db, newAccountIdent(common.Hash{0xa})) + br, err := newIndexReader(db, newAccountIdent(common.Hash{0xa}), bitmapSize) if err != nil { t.Fatalf("Failed to construct the index reader, %v", err) } @@ -107,7 +119,7 @@ func TestIndexReaderLarge(t *testing.T) { } func TestEmptyIndexReader(t *testing.T) { - br, err := newIndexReader(rawdb.NewMemoryDatabase(), newAccountIdent(common.Hash{0xa})) + br, err := newIndexReader(rawdb.NewMemoryDatabase(), newAccountIdent(common.Hash{0xa}), 0) if err != nil { t.Fatalf("Failed to construct the index reader, %v", err) } @@ -121,57 +133,150 @@ func TestEmptyIndexReader(t *testing.T) { } func TestIndexWriterBasic(t *testing.T) { + testIndexWriterBasic(t, 0) + testIndexWriterBasic(t, 2) + testIndexWriterBasic(t, 34) +} + +func testIndexWriterBasic(t *testing.T, bitmapSize int) { db := rawdb.NewMemoryDatabase() - iw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa})) - iw.append(2) - if err := iw.append(1); err == nil { + iw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0, bitmapSize) + iw.append(2, randomExt(bitmapSize, 5)) + if err := iw.append(1, randomExt(bitmapSize, 5)); err == nil { t.Fatal("out-of-order insertion is not expected") } + var maxElem uint64 for i := 0; i < 10; i++ { - iw.append(uint64(i + 3)) + iw.append(uint64(i+3), randomExt(bitmapSize, 5)) + maxElem = uint64(i + 3) } batch := db.NewBatch() iw.finish(batch) batch.Write() - iw, err := newIndexWriter(db, newAccountIdent(common.Hash{0xa})) + iw, err := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), maxElem, bitmapSize) if err != nil { t.Fatalf("Failed to construct the block writer, %v", err) } for i := 0; i < 10; i++ { - if err := iw.append(uint64(i + 100)); err != nil { + if err := iw.append(uint64(i+100), randomExt(bitmapSize, 5)); err != nil { t.Fatalf("Failed to append item, %v", err) } } iw.finish(db.NewBatch()) } -func TestIndexWriterDelete(t *testing.T) { +func TestIndexWriterWithLimit(t *testing.T) { + testIndexWriterWithLimit(t, 0) + testIndexWriterWithLimit(t, 2) + testIndexWriterWithLimit(t, 34) +} + +func testIndexWriterWithLimit(t *testing.T, bitmapSize int) { db := rawdb.NewMemoryDatabase() - iw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa})) - for i := 0; i < indexBlockEntriesCap*4; i++ { - iw.append(uint64(i + 1)) + iw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0, bitmapSize) + + // 200 iterations (with around 50 bytes extension) is enough to cross + // the block boundary (4096 bytes) + for i := 0; i < 200; i++ { + iw.append(uint64(i+1), randomExt(bitmapSize, 50)) + } + batch := db.NewBatch() + iw.finish(batch) + batch.Write() + + for i := 0; i < 200; i++ { + limit := uint64(i + 1) + iw, err := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), limit, bitmapSize) + if err != nil { + t.Fatalf("Failed to construct the index writer, %v", err) + } + if iw.lastID != limit { + t.Fatalf("Test %d, unexpected max value, got %d, want %d", i, iw.lastID, limit) + } + // Re-fill the elements + var maxElem uint64 + for elem := limit + 1; elem < 500; elem++ { + if err := iw.append(elem, randomExt(bitmapSize, 5)); err != nil { + t.Fatalf("Failed to append value %d: %v", elem, err) + } + maxElem = elem + } + if iw.lastID != maxElem { + t.Fatalf("Test %d, unexpected max value, got %d, want %d", i, iw.lastID, maxElem) + } + } +} + +func TestIndexDeleterBasic(t *testing.T) { + testIndexDeleterBasic(t, 0) + testIndexDeleterBasic(t, 2) + testIndexDeleterBasic(t, 34) +} + +func testIndexDeleterBasic(t *testing.T, bitmapSize int) { + db := rawdb.NewMemoryDatabase() + iw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0, bitmapSize) + + // 200 iterations (with around 50 bytes extension) is enough to cross + // the block boundary (4096 bytes) + var maxElem uint64 + for i := 0; i < 200; i++ { + iw.append(uint64(i+1), randomExt(bitmapSize, 50)) + maxElem = uint64(i + 1) } batch := db.NewBatch() iw.finish(batch) batch.Write() // Delete unknown id, the request should be rejected - id, _ := newIndexDeleter(db, newAccountIdent(common.Hash{0xa})) - if err := id.pop(indexBlockEntriesCap * 5); err == nil { + id, _ := newIndexDeleter(db, newAccountIdent(common.Hash{0xa}), maxElem, bitmapSize) + if err := id.pop(500); err == nil { t.Fatal("Expect error to occur for unknown id") } - for i := indexBlockEntriesCap * 4; i >= 1; i-- { + for i := 200; i >= 1; i-- { if err := id.pop(uint64(i)); err != nil { t.Fatalf("Unexpected error for element popping, %v", err) } if id.lastID != uint64(i-1) { t.Fatalf("Unexpected lastID, want: %d, got: %d", uint64(i-1), iw.lastID) } - if rand.Intn(10) == 0 { - batch := db.NewBatch() - id.finish(batch) - batch.Write() + } +} + +func TestIndexDeleterWithLimit(t *testing.T) { + testIndexDeleterWithLimit(t, 0) + testIndexDeleterWithLimit(t, 2) + testIndexDeleterWithLimit(t, 34) +} + +func testIndexDeleterWithLimit(t *testing.T, bitmapSize int) { + db := rawdb.NewMemoryDatabase() + iw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0, bitmapSize) + + // 200 iterations (with around 50 bytes extension) is enough to cross + // the block boundary (4096 bytes) + for i := 0; i < 200; i++ { + iw.append(uint64(i+1), randomExt(bitmapSize, 50)) + } + batch := db.NewBatch() + iw.finish(batch) + batch.Write() + + for i := 0; i < 200; i++ { + limit := uint64(i + 1) + id, err := newIndexDeleter(db, newAccountIdent(common.Hash{0xa}), limit, bitmapSize) + if err != nil { + t.Fatalf("Failed to construct the index writer, %v", err) + } + if id.lastID != limit { + t.Fatalf("Test %d, unexpected max value, got %d, want %d", i, iw.lastID, limit) + } + // Keep removing elements + for elem := id.lastID; elem > 0; elem-- { + if err := id.pop(elem); err != nil { + t.Fatalf("Failed to pop value %d: %v", elem, err) + } } } } @@ -212,7 +317,7 @@ func TestBatchIndexerWrite(t *testing.T) { } } for addrHash, indexes := range accounts { - ir, _ := newIndexReader(db, newAccountIdent(addrHash)) + ir, _ := newIndexReader(db, newAccountIdent(addrHash), 0) for i := 0; i < len(indexes)-1; i++ { n, err := ir.readGreaterThan(indexes[i]) if err != nil { @@ -232,7 +337,7 @@ func TestBatchIndexerWrite(t *testing.T) { } for addrHash, slots := range storages { for slotHash, indexes := range slots { - ir, _ := newIndexReader(db, newStorageIdent(addrHash, slotHash)) + ir, _ := newIndexReader(db, newStorageIdent(addrHash, slotHash), 0) for i := 0; i < len(indexes)-1; i++ { n, err := ir.readGreaterThan(indexes[i]) if err != nil { diff --git a/triedb/pathdb/history_indexer.go b/triedb/pathdb/history_indexer.go index 893ccd6523..c9bf3e87f1 100644 --- a/triedb/pathdb/history_indexer.go +++ b/triedb/pathdb/history_indexer.go @@ -34,17 +34,15 @@ import ( const ( // The batch size for reading state histories - historyReadBatch = 1000 + historyReadBatch = 1000 + historyIndexBatch = 8 * 1024 * 1024 // The number of state history indexes for constructing or deleting as batch stateHistoryIndexV0 = uint8(0) // initial version of state index structure stateHistoryIndexVersion = stateHistoryIndexV0 // the current state index version trienodeHistoryIndexV0 = uint8(0) // initial version of trienode index structure trienodeHistoryIndexVersion = trienodeHistoryIndexV0 // the current trienode index version - // estimations for calculating the batch size for atomic database commit - estimatedStateHistoryIndexSize = 3 // The average size of each state history index entry is approximately 2–3 bytes - estimatedTrienodeHistoryIndexSize = 3 // The average size of each trienode history index entry is approximately 2-3 bytes - estimatedIndexBatchSizeFactor = 32 // The factor counts for the write amplification for each entry + indexerProcessBatchInSync = 100000 // threshold for history batch indexing when node is in sync stage. ) // indexVersion returns the latest index version for the given history type. @@ -125,18 +123,20 @@ func deleteIndexMetadata(db ethdb.KeyValueWriter, typ historyType) { // batchIndexer is responsible for performing batch indexing or unindexing // of historical data (e.g., state or trie node changes) atomically. type batchIndexer struct { - index map[stateIdent][]uint64 // List of history IDs for tracked state entry - pending int // Number of entries processed in the current batch. - delete bool // Operation mode: true for unindex, false for index. - lastID uint64 // ID of the most recently processed history. - typ historyType // Type of history being processed (e.g., state or trienode). - db ethdb.KeyValueStore // Key-value database used to store or delete index data. + index map[stateIdent][]uint64 // List of history IDs for tracked state entry + ext map[stateIdent][][]uint16 // List of extension for each state element + pending int // Number of entries processed in the current batch. + delete bool // Operation mode: true for unindex, false for index. + lastID uint64 // ID of the most recently processed history. + typ historyType // Type of history being processed (e.g., state or trienode). + db ethdb.KeyValueStore // Key-value database used to store or delete index data. } // newBatchIndexer constructs the batch indexer with the supplied mode. func newBatchIndexer(db ethdb.KeyValueStore, delete bool, typ historyType) *batchIndexer { return &batchIndexer{ index: make(map[stateIdent][]uint64), + ext: make(map[stateIdent][][]uint16), delete: delete, typ: typ, db: db, @@ -146,8 +146,10 @@ func newBatchIndexer(db ethdb.KeyValueStore, delete bool, typ historyType) *batc // process traverses the state entries within the provided history and tracks the mutation // records for them. func (b *batchIndexer) process(h history, id uint64) error { - for ident := range h.forEach() { - b.index[ident] = append(b.index[ident], id) + for elem := range h.forEach() { + key := elem.key() + b.index[key] = append(b.index[key], id) + b.ext[key] = append(b.ext[key], elem.ext()) b.pending++ } b.lastID = id @@ -155,22 +157,6 @@ func (b *batchIndexer) process(h history, id uint64) error { return b.finish(false) } -// makeBatch constructs a database batch based on the number of pending entries. -// The batch size is roughly estimated to minimize repeated resizing rounds, -// as accurately predicting the exact size is technically challenging. -func (b *batchIndexer) makeBatch() ethdb.Batch { - var size int - switch b.typ { - case typeStateHistory: - size = estimatedStateHistoryIndexSize - case typeTrienodeHistory: - size = estimatedTrienodeHistoryIndexSize - default: - panic(fmt.Sprintf("unknown history type %d", b.typ)) - } - return b.db.NewBatchWithSize(size * estimatedIndexBatchSizeFactor * b.pending) -} - // finish writes the accumulated state indexes into the disk if either the // memory limitation is reached or it's requested forcibly. func (b *batchIndexer) finish(force bool) error { @@ -181,30 +167,52 @@ func (b *batchIndexer) finish(force bool) error { return nil } var ( - batch = b.makeBatch() - batchMu sync.RWMutex - start = time.Now() - eg errgroup.Group + start = time.Now() + eg errgroup.Group + + batch = b.db.NewBatchWithSize(ethdb.IdealBatchSize) + batchSize int + batchMu sync.RWMutex + + writeBatch = func(fn func(batch ethdb.Batch)) error { + batchMu.Lock() + defer batchMu.Unlock() + + fn(batch) + if batch.ValueSize() >= ethdb.IdealBatchSize { + batchSize += batch.ValueSize() + if err := batch.Write(); err != nil { + return err + } + batch.Reset() + } + return nil + } ) eg.SetLimit(runtime.NumCPU()) + var indexed uint64 + if metadata := loadIndexMetadata(b.db, b.typ); metadata != nil { + indexed = metadata.Last + } for ident, list := range b.index { + ext := b.ext[ident] eg.Go(func() error { if !b.delete { - iw, err := newIndexWriter(b.db, ident) + iw, err := newIndexWriter(b.db, ident, indexed, ident.bloomSize()) if err != nil { return err } - for _, n := range list { - if err := iw.append(n); err != nil { + for i, n := range list { + if err := iw.append(n, ext[i]); err != nil { return err } } - batchMu.Lock() - iw.finish(batch) - batchMu.Unlock() + return writeBatch(func(batch ethdb.Batch) { + iw.finish(batch) + }) } else { - id, err := newIndexDeleter(b.db, ident) + id, err := newIndexDeleter(b.db, ident, indexed, ident.bloomSize()) if err != nil { return err } @@ -213,11 +221,10 @@ func (b *batchIndexer) finish(force bool) error { return err } } - batchMu.Lock() - id.finish(batch) - batchMu.Unlock() + return writeBatch(func(batch ethdb.Batch) { + id.finish(batch) + }) } - return nil }) } if err := eg.Wait(); err != nil { @@ -233,12 +240,16 @@ func (b *batchIndexer) finish(force bool) error { storeIndexMetadata(batch, b.typ, b.lastID-1) } } + batchSize += batch.ValueSize() + if err := batch.Write(); err != nil { return err } - log.Debug("Committed batch indexer", "type", b.typ, "entries", len(b.index), "records", b.pending, "elapsed", common.PrettyDuration(time.Since(start))) + log.Debug("Committed batch indexer", "type", b.typ, "entries", len(b.index), "records", b.pending, "size", common.StorageSize(batchSize), "elapsed", common.PrettyDuration(time.Since(start))) + b.pending = 0 - b.index = make(map[stateIdent][]uint64) + clear(b.index) + clear(b.ext) return nil } @@ -340,7 +351,8 @@ type interruptSignal struct { // If a state history is removed due to a rollback, the associated indexes should // be unmarked accordingly. type indexIniter struct { - disk ethdb.KeyValueStore + state *initerState + disk ethdb.Database freezer ethdb.AncientStore interrupt chan *interruptSignal done chan struct{} @@ -355,8 +367,9 @@ type indexIniter struct { wg sync.WaitGroup } -func newIndexIniter(disk ethdb.KeyValueStore, freezer ethdb.AncientStore, typ historyType, lastID uint64) *indexIniter { +func newIndexIniter(disk ethdb.Database, freezer ethdb.AncientStore, typ historyType, lastID uint64, noWait bool) *indexIniter { initer := &indexIniter{ + state: newIniterState(disk, noWait), disk: disk, freezer: freezer, interrupt: make(chan *interruptSignal), @@ -376,12 +389,7 @@ func newIndexIniter(disk ethdb.KeyValueStore, freezer ethdb.AncientStore, typ hi // Launch background indexer initer.wg.Add(1) - if recover { - log.Info("History indexer is recovering", "history", lastID, "indexed", metadata.Last) - go initer.recover(lastID) - } else { - go initer.run(lastID) - } + go initer.run(recover) return initer } @@ -391,6 +399,7 @@ func (i *indexIniter) close() { return default: close(i.closed) + i.state.close() i.wg.Wait() } } @@ -422,85 +431,109 @@ func (i *indexIniter) remain() uint64 { } } -func (i *indexIniter) run(lastID uint64) { +func (i *indexIniter) run(recover bool) { defer i.wg.Done() // Launch background indexing thread var ( - done = make(chan struct{}) - interrupt = new(atomic.Int32) + done chan struct{} + interrupt *atomic.Int32 - // checkDone indicates whether all requested state histories - // have been fully indexed. + // checkDone reports whether indexing has completed for all histories. checkDone = func() bool { metadata := loadIndexMetadata(i.disk, i.typ) - return metadata != nil && metadata.Last == lastID + return metadata != nil && metadata.Last == i.last.Load() } + // canExit reports whether the initial indexing phase has completed. + canExit = func() bool { + return !i.state.is(stateSyncing) && checkDone() + } + heartBeat = time.NewTimer(0) ) - go i.index(done, interrupt, lastID) + defer heartBeat.Stop() + if recover { + if aborted := i.recover(); aborted { + return + } + } for { select { case signal := <-i.interrupt: - // The indexing limit can only be extended or shortened continuously. newLastID := signal.newLastID - if newLastID != lastID+1 && newLastID != lastID-1 { - signal.result <- fmt.Errorf("invalid history id, last: %d, got: %d", lastID, newLastID) + oldLastID := i.last.Load() + + // The indexing limit can only be extended or shortened continuously. + if newLastID != oldLastID+1 && newLastID != oldLastID-1 { + signal.result <- fmt.Errorf("invalid history id, last: %d, got: %d", oldLastID, newLastID) continue } i.last.Store(newLastID) // update indexing range // The index limit is extended by one, update the limit without // interrupting the current background process. - if newLastID == lastID+1 { - lastID = newLastID + if newLastID == oldLastID+1 { signal.result <- nil - i.log.Debug("Extended history range", "last", lastID) + i.log.Debug("Extended history range", "last", newLastID) continue } - // The index limit is shortened by one, interrupt the current background - // process and relaunch with new target. - interrupt.Store(1) - <-done - + // The index limit is shortened, interrupt the current background + // process if it's active and update the target. + if done != nil { + interrupt.Store(1) + <-done + done, interrupt = nil, nil + } // If all state histories, including the one to be reverted, have // been fully indexed, unindex it here and shut down the initializer. if checkDone() { - i.log.Info("Truncate the extra history", "id", lastID) - if err := unindexSingle(lastID, i.disk, i.freezer, i.typ); err != nil { + i.log.Info("Truncate the extra history", "id", oldLastID) + if err := unindexSingle(oldLastID, i.disk, i.freezer, i.typ); err != nil { signal.result <- err return } close(i.done) signal.result <- nil - i.log.Info("Histories have been fully indexed", "last", lastID-1) + i.log.Info("Histories have been fully indexed", "last", i.last.Load()) return } - // Adjust the indexing target and relaunch the process - lastID = newLastID + // Adjust the indexing target signal.result <- nil - - done, interrupt = make(chan struct{}), new(atomic.Int32) - go i.index(done, interrupt, lastID) - i.log.Debug("Shortened history range", "last", lastID) + i.log.Debug("Shortened history range", "last", newLastID) case <-done: - if checkDone() { + done, interrupt = nil, nil + + if canExit() { close(i.done) - i.log.Info("Histories have been fully indexed", "last", lastID) return } - // Relaunch the background runner if some tasks are left - done, interrupt = make(chan struct{}), new(atomic.Int32) - go i.index(done, interrupt, lastID) - case <-i.closed: - interrupt.Store(1) - i.log.Info("Waiting background history index initer to exit") - <-done + case <-heartBeat.C: + heartBeat.Reset(time.Second * 15) - if checkDone() { + // Short circuit if the indexer is still busy + if done != nil { + continue + } + if canExit() { close(i.done) + return + } + // The local chain is still in the syncing phase. Only start the indexing + // when a sufficient amount of histories has accumulated. Batch indexing + // is more efficient than processing items individually. + if i.state.is(stateSyncing) && i.last.Load()-i.indexed.Load() < indexerProcessBatchInSync { + continue + } + done, interrupt = make(chan struct{}), new(atomic.Int32) + go i.index(done, interrupt, i.last.Load()) + + case <-i.closed: + if done != nil { + interrupt.Store(1) + i.log.Info("Waiting background history index initer to exit") + <-done } return } @@ -562,7 +595,7 @@ func (i *indexIniter) index(done chan struct{}, interrupt *atomic.Int32, lastID } return } - i.log.Info("Start history indexing", "beginID", beginID, "lastID", lastID) + i.log.Debug("Start history indexing", "beginID", beginID, "lastID", lastID) var ( current = beginID @@ -609,7 +642,7 @@ func (i *indexIniter) index(done chan struct{}, interrupt *atomic.Int32, lastID done = current - beginID ) eta := common.CalculateETA(done, left, time.Since(start)) - i.log.Info("Indexing history", "processed", done, "left", left, "elapsed", common.PrettyDuration(time.Since(start)), "eta", common.PrettyDuration(eta)) + i.log.Debug("Indexing history", "processed", done, "left", left, "elapsed", common.PrettyDuration(time.Since(start)), "eta", common.PrettyDuration(eta)) } } i.indexed.Store(current - 1) // update indexing progress @@ -620,7 +653,7 @@ func (i *indexIniter) index(done chan struct{}, interrupt *atomic.Int32, lastID if err := batch.finish(true); err != nil { i.log.Error("Failed to flush index", "err", err) } - log.Info("State indexing interrupted") + log.Debug("State indexing interrupted") return } } @@ -628,7 +661,7 @@ func (i *indexIniter) index(done chan struct{}, interrupt *atomic.Int32, lastID if err := batch.finish(true); err != nil { i.log.Error("Failed to flush index", "err", err) } - i.log.Info("Indexed history", "from", beginID, "to", lastID, "elapsed", common.PrettyDuration(time.Since(start))) + i.log.Debug("Indexed history", "from", beginID, "to", lastID, "elapsed", common.PrettyDuration(time.Since(start))) } // recover handles unclean shutdown recovery. After an unclean shutdown, any @@ -641,35 +674,35 @@ func (i *indexIniter) index(done chan struct{}, interrupt *atomic.Int32, lastID // by chain recovery, under the assumption that the recovered histories will be // identical to the lost ones. Fork-awareness should be added in the future to // correctly handle histories affected by reorgs. -func (i *indexIniter) recover(lastID uint64) { - defer i.wg.Done() +func (i *indexIniter) recover() bool { + log.Info("History indexer is recovering", "last", i.last.Load(), "indexed", i.indexed.Load()) for { select { case signal := <-i.interrupt: newLastID := signal.newLastID - if newLastID != lastID+1 && newLastID != lastID-1 { - signal.result <- fmt.Errorf("invalid history id, last: %d, got: %d", lastID, newLastID) + oldLastID := i.last.Load() + + // The indexing limit can only be extended or shortened continuously. + if newLastID != oldLastID+1 && newLastID != oldLastID-1 { + signal.result <- fmt.Errorf("invalid history id, last: %d, got: %d", oldLastID, newLastID) continue } - // Update the last indexed flag - lastID = newLastID signal.result <- nil i.last.Store(newLastID) - i.log.Debug("Updated history index flag", "last", lastID) + i.log.Debug("Updated history index flag", "last", newLastID) // Terminate the recovery routine once the histories are fully aligned // with the index data, indicating that index initialization is complete. metadata := loadIndexMetadata(i.disk, i.typ) - if metadata != nil && metadata.Last == lastID { - close(i.done) - i.log.Info("History indexer is recovered", "last", lastID) - return + if metadata != nil && metadata.Last == newLastID { + i.log.Info("History indexer is recovered", "last", newLastID) + return false } case <-i.closed: - return + return true } } } @@ -737,10 +770,10 @@ func checkVersion(disk ethdb.KeyValueStore, typ historyType) { // newHistoryIndexer constructs the history indexer and launches the background // initer to complete the indexing of any remaining state histories. -func newHistoryIndexer(disk ethdb.KeyValueStore, freezer ethdb.AncientStore, lastHistoryID uint64, typ historyType) *historyIndexer { +func newHistoryIndexer(disk ethdb.Database, freezer ethdb.AncientStore, lastHistoryID uint64, typ historyType, noWait bool) *historyIndexer { checkVersion(disk, typ) return &historyIndexer{ - initer: newIndexIniter(disk, freezer, typ, lastHistoryID), + initer: newIndexIniter(disk, freezer, typ, lastHistoryID, noWait), typ: typ, disk: disk, freezer: freezer, diff --git a/triedb/pathdb/history_indexer_state.go b/triedb/pathdb/history_indexer_state.go new file mode 100644 index 0000000000..70514055e7 --- /dev/null +++ b/triedb/pathdb/history_indexer_state.go @@ -0,0 +1,192 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package pathdb + +import ( + "bytes" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" +) + +// state represents the syncing status of the node. +type state int + +const ( + // stateSynced indicates that the local chain head is sufficiently close to the + // network chain head, and the majority of the data has been fully synchronized. + stateSynced state = iota + + // stateSyncing indicates that the sync process is still in progress. Local node + // is actively catching up with the network chain head. + stateSyncing + + // stateStalled indicates that sync progress has stopped for a while + // with no progress. This may be caused by network instability (e.g., no peers), + // manual operation such as syncing the local chain to a specific block. + stateStalled +) + +const ( + // syncStateTimeWindow defines the maximum allowed lag behind the network + // chain head. + // + // If the local chain head falls within this threshold, the node is considered + // close to the tip and will be marked as stateSynced. + syncStateTimeWindow = 6 * time.Hour + + // syncStalledTimeout defines the maximum duration during which no sync + // progress is observed. If this timeout is exceeded, the node's status + // will be considered stalled. + syncStalledTimeout = 5 * time.Minute +) + +type initerState struct { + state state + stateLock sync.Mutex + disk ethdb.Database + term chan struct{} +} + +func newIniterState(disk ethdb.Database, noWait bool) *initerState { + // When noWait is true, initialize the state to stateSynced synchronously + // so the indexer's run loop doesn't race with update() and skip its + // first indexing attempt for an entire heartbeat interval (15s) — that + // matters for archive-mode tests with non-realtime block timestamps, + // where update() would also pick stateSynced (via its noWait branch). + initial := stateSyncing + if noWait { + initial = stateSynced + } + s := &initerState{ + state: initial, + disk: disk, + term: make(chan struct{}), + } + go s.update(noWait) + return s +} + +func (s *initerState) get() state { + s.stateLock.Lock() + defer s.stateLock.Unlock() + + return s.state +} + +func (s *initerState) is(state state) bool { + return s.get() == state +} + +func (s *initerState) set(state state) { + s.stateLock.Lock() + defer s.stateLock.Unlock() + + s.state = state +} + +func (s *initerState) update(noWait bool) { + ticker := time.NewTicker(time.Minute) + defer ticker.Stop() + + headBlock := s.readLastBlock() + if headBlock != nil && time.Since(time.Unix(int64(headBlock.Time), 0)) < syncStateTimeWindow { + s.set(stateSynced) + log.Info("Marked indexing initer as synced") + } else if noWait { + s.set(stateSynced) + log.Info("Marked indexing initer as synced forcibly") + } else { + s.set(stateSyncing) + } + + var ( + hhash = rawdb.ReadHeadHeaderHash(s.disk) + fhash = rawdb.ReadHeadFastBlockHash(s.disk) + bhash = rawdb.ReadHeadBlockHash(s.disk) + skeleton = rawdb.ReadSkeletonSyncStatus(s.disk) + lastProgress = time.Now() + ) + for { + select { + case <-ticker.C: + state := s.get() + if state == stateSynced || state == stateStalled { + continue + } + headBlock := s.readLastBlock() + if headBlock == nil { + continue + } + // State machine: stateSyncing => stateSynced + if time.Since(time.Unix(int64(headBlock.Time), 0)) < syncStateTimeWindow { + s.set(stateSynced) + log.Info("Marked indexing initer as synced") + continue + } + // State machine: stateSyncing => stateStalled + newhhash := rawdb.ReadHeadHeaderHash(s.disk) + newfhash := rawdb.ReadHeadFastBlockHash(s.disk) + newbhash := rawdb.ReadHeadBlockHash(s.disk) + newskeleton := rawdb.ReadSkeletonSyncStatus(s.disk) + hasProgress := newhhash.Cmp(hhash) != 0 || newfhash.Cmp(fhash) != 0 || newbhash.Cmp(bhash) != 0 || !bytes.Equal(newskeleton, skeleton) + + if !hasProgress && time.Since(lastProgress) > syncStalledTimeout { + s.set(stateStalled) + log.Info("Marked indexing initer as stalled") + continue + } + if hasProgress { + hhash = newhhash + fhash = newfhash + bhash = newbhash + skeleton = newskeleton + lastProgress = time.Now() + } + + case <-s.term: + return + } + } +} + +func (s *initerState) close() { + select { + case <-s.term: + default: + close(s.term) + } + return +} + +// readLastBlock returns the local chain head. +func (s *initerState) readLastBlock() *types.Header { + hash := rawdb.ReadHeadBlockHash(s.disk) + if hash == (common.Hash{}) { + return nil + } + number, exists := rawdb.ReadHeaderNumber(s.disk, hash) + if !exists { + return nil + } + return rawdb.ReadHeader(s.disk, hash, number) +} diff --git a/triedb/pathdb/history_indexer_test.go b/triedb/pathdb/history_indexer_test.go index f333d18d8b..8bb1db42da 100644 --- a/triedb/pathdb/history_indexer_test.go +++ b/triedb/pathdb/history_indexer_test.go @@ -27,7 +27,7 @@ import ( // deadlock when the indexer is active. This specifically targets the case where // signal.result must be sent to unblock the caller. func TestHistoryIndexerShortenDeadlock(t *testing.T) { - //log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true))) + // log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true))) db := rawdb.NewMemoryDatabase() freezer, _ := rawdb.NewStateFreezer(t.TempDir(), false, false) defer freezer.Close() @@ -38,7 +38,7 @@ func TestHistoryIndexerShortenDeadlock(t *testing.T) { rawdb.WriteStateHistory(freezer, uint64(i+1), h.meta.encode(), accountIndex, storageIndex, accountData, storageData) } // As a workaround, assign a future block to keep the initer running indefinitely - indexer := newHistoryIndexer(db, freezer, 200, typeStateHistory) + indexer := newHistoryIndexer(db, freezer, 200, typeStateHistory, true) defer indexer.close() done := make(chan error, 1) diff --git a/triedb/pathdb/history_inspect.go b/triedb/pathdb/history_inspect.go index 9b4eea27b4..74b8bb8df2 100644 --- a/triedb/pathdb/history_inspect.go +++ b/triedb/pathdb/history_inspect.go @@ -50,12 +50,12 @@ func sanitizeRange(start, end uint64, freezer ethdb.AncientReader) (uint64, uint if err != nil { return 0, 0, err } - last := head - 1 + last := head if end != 0 && end < last { last = end } // Make sure the range is valid - if first >= last { + if first > last { return 0, 0, fmt.Errorf("range is invalid, first: %d, last: %d", first, last) } return first, last, nil @@ -143,7 +143,7 @@ func historyRange(freezer ethdb.AncientReader) (uint64, uint64, error) { if err != nil { return 0, 0, err } - last := head - 1 + last := head fh, err := readStateHistory(freezer, first) if err != nil { diff --git a/triedb/pathdb/history_reader.go b/triedb/pathdb/history_reader.go index 1bf4cf648d..4ae1fb36cb 100644 --- a/triedb/pathdb/history_reader.go +++ b/triedb/pathdb/history_reader.go @@ -22,11 +22,17 @@ import ( "errors" "fmt" "math" + "slices" "sort" + "sync" + "sync/atomic" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" + "golang.org/x/sync/errgroup" ) // indexReaderWithLimitTag is a wrapper around indexReader that includes an @@ -40,8 +46,8 @@ type indexReaderWithLimitTag struct { } // newIndexReaderWithLimitTag constructs a index reader with indexing position. -func newIndexReaderWithLimitTag(db ethdb.KeyValueReader, state stateIdent, limit uint64) (*indexReaderWithLimitTag, error) { - r, err := newIndexReader(db, state) +func newIndexReaderWithLimitTag(db ethdb.KeyValueReader, state stateIdent, limit uint64, bitmapSize int) (*indexReaderWithLimitTag, error) { + r, err := newIndexReader(db, state, bitmapSize) if err != nil { return nil, err } @@ -99,16 +105,17 @@ func (r *indexReaderWithLimitTag) readGreaterThan(id uint64, lastID uint64) (uin return r.reader.readGreaterThan(id) } -// historyReader is the structure to access historic state data. -type historyReader struct { +// stateHistoryReader is the structure to access historic state data. +type stateHistoryReader struct { disk ethdb.KeyValueReader freezer ethdb.AncientReader readers map[string]*indexReaderWithLimitTag } -// newHistoryReader constructs the history reader with the supplied db. -func newHistoryReader(disk ethdb.KeyValueReader, freezer ethdb.AncientReader) *historyReader { - return &historyReader{ +// newStateHistoryReader constructs the history reader with the supplied db +// for accessing historical states. +func newStateHistoryReader(disk ethdb.KeyValueReader, freezer ethdb.AncientReader) *stateHistoryReader { + return &stateHistoryReader{ disk: disk, freezer: freezer, readers: make(map[string]*indexReaderWithLimitTag), @@ -117,7 +124,7 @@ func newHistoryReader(disk ethdb.KeyValueReader, freezer ethdb.AncientReader) *h // readAccountMetadata resolves the account metadata within the specified // state history. -func (r *historyReader) readAccountMetadata(address common.Address, historyID uint64) ([]byte, error) { +func (r *stateHistoryReader) readAccountMetadata(address common.Address, historyID uint64) ([]byte, error) { blob := rawdb.ReadStateAccountIndex(r.freezer, historyID) if len(blob) == 0 { return nil, fmt.Errorf("account index is truncated, historyID: %d", historyID) @@ -143,7 +150,7 @@ func (r *historyReader) readAccountMetadata(address common.Address, historyID ui // readStorageMetadata resolves the storage slot metadata within the specified // state history. -func (r *historyReader) readStorageMetadata(storageKey common.Hash, storageHash common.Hash, historyID uint64, slotOffset, slotNumber int) ([]byte, error) { +func (r *stateHistoryReader) readStorageMetadata(storageKey common.Hash, storageHash common.Hash, historyID uint64, slotOffset, slotNumber int) ([]byte, error) { data, err := rawdb.ReadStateStorageIndex(r.freezer, historyID, slotIndexSize*slotOffset, slotIndexSize*slotNumber) if err != nil { msg := fmt.Sprintf("id: %d, slot-offset: %d, slot-length: %d", historyID, slotOffset, slotNumber) @@ -178,7 +185,7 @@ func (r *historyReader) readStorageMetadata(storageKey common.Hash, storageHash } // readAccount retrieves the account data from the specified state history. -func (r *historyReader) readAccount(address common.Address, historyID uint64) ([]byte, error) { +func (r *stateHistoryReader) readAccount(address common.Address, historyID uint64) ([]byte, error) { metadata, err := r.readAccountMetadata(address, historyID) if err != nil { return nil, err @@ -194,7 +201,7 @@ func (r *historyReader) readAccount(address common.Address, historyID uint64) ([ } // readStorage retrieves the storage slot data from the specified state history. -func (r *historyReader) readStorage(address common.Address, storageKey common.Hash, storageHash common.Hash, historyID uint64) ([]byte, error) { +func (r *stateHistoryReader) readStorage(address common.Address, storageKey common.Hash, storageHash common.Hash, historyID uint64) ([]byte, error) { metadata, err := r.readAccountMetadata(address, historyID) if err != nil { return nil, err @@ -224,35 +231,16 @@ func (r *historyReader) readStorage(address common.Address, storageKey common.Ha // stateID: represents the ID of the state of the specified version; // lastID: represents the ID of the latest/newest state history; // latestValue: represents the state value at the current disk layer with ID == lastID; -func (r *historyReader) read(state stateIdentQuery, stateID uint64, lastID uint64, latestValue []byte) ([]byte, error) { - tail, err := r.freezer.Tail() +func (r *stateHistoryReader) read(state stateIdentQuery, stateID uint64, lastID uint64, latestValue []byte) ([]byte, error) { + lastIndexed, err := checkStateAvail(state.stateIdent, typeStateHistory, r.freezer, stateID, lastID, r.disk) if err != nil { return nil, err - } // firstID = tail+1 - - // stateID+1 == firstID is allowed, as all the subsequent state histories - // are present with no gap inside. - if stateID < tail { - return nil, fmt.Errorf("historical state has been pruned, first: %d, state: %d", tail+1, stateID) } - - // To serve the request, all state histories from stateID+1 to lastID - // must be indexed. It's not supposed to happen unless system is very - // wrong. - metadata := loadIndexMetadata(r.disk, toHistoryType(state.typ)) - if metadata == nil || metadata.Last < lastID { - indexed := "null" - if metadata != nil { - indexed = fmt.Sprintf("%d", metadata.Last) - } - return nil, fmt.Errorf("state history is not fully indexed, requested: %d, indexed: %s", stateID, indexed) - } - // Construct the index reader to locate the corresponding history for // state retrieval ir, ok := r.readers[state.String()] if !ok { - ir, err = newIndexReaderWithLimitTag(r.disk, state.stateIdent, metadata.Last) + ir, err = newIndexReaderWithLimitTag(r.disk, state.stateIdent, lastIndexed, 0) if err != nil { return nil, err } @@ -277,3 +265,230 @@ func (r *historyReader) read(state stateIdentQuery, stateID uint64, lastID uint6 } return r.readStorage(state.address, state.storageKey, state.storageHash, historyID) } + +// trienodeReader is the structure to access historical trienode data. +type trienodeReader struct { + disk ethdb.KeyValueReader + freezer ethdb.AncientReader + readConcurrency int // The concurrency used to load trie node data from history +} + +// newTrienodeReader constructs the history reader with the supplied db +// for accessing historical trie nodes. +func newTrienodeReader(disk ethdb.KeyValueReader, freezer ethdb.AncientReader, readConcurrency int) *trienodeReader { + return &trienodeReader{ + disk: disk, + freezer: freezer, + readConcurrency: readConcurrency, + } +} + +// readTrienode retrieves the trienode data from the specified trienode history. +func (r *trienodeReader) readTrienode(addrHash common.Hash, path string, historyID uint64) ([]byte, bool, error) { + tr := newTrienodeHistoryReader(historyID, r.freezer) + return tr.read(addrHash, path) +} + +// assembleNode takes a complete node value as the base and applies a list of +// mutation records to assemble the final node value accordingly. +func assembleNode(blob []byte, elements [][][]byte, indices [][]int) ([]byte, error) { + if len(elements) == 0 && len(indices) == 0 { + return blob, nil + } + children, err := rlp.SplitListValues(blob) + if err != nil { + return nil, err + } + for i := 0; i < len(elements); i++ { + for j, pos := range indices[i] { + children[pos] = elements[i][j] + } + } + return rlp.MergeListValues(children) +} + +type resultQueue struct { + data [][]byte + lock sync.Mutex +} + +func newResultQueue(size int) *resultQueue { + return &resultQueue{ + data: make([][]byte, size, size*2), + } +} + +func (q *resultQueue) set(data []byte, pos int) { + q.lock.Lock() + defer q.lock.Unlock() + + if pos >= len(q.data) { + newSize := pos + 1 + if cap(q.data) < newSize { + newData := make([][]byte, newSize, newSize*2) + copy(newData, q.data) + q.data = newData + } + q.data = q.data[:newSize] + } + q.data[pos] = data +} + +func (r *trienodeReader) readOptimized(state stateIdent, it HistoryIndexIterator, latestValue []byte) ([]byte, error) { + var ( + elements [][][]byte + indices [][]int + blob = latestValue + + eg errgroup.Group + seq int + term atomic.Bool + queue = newResultQueue(r.readConcurrency * 2) + ) + eg.SetLimit(r.readConcurrency) + + for { + id, pos := it.ID(), seq + seq += 1 + + eg.Go(func() error { + data, found, err := r.readTrienode(state.addressHash, state.path, id) + if err != nil { + term.Store(true) + return err + } + // In optimistic readahead mode, it is theoretically possible to encounter a + // NotFound error, where the trie node does not actually exist and the iterator + // reports a false-positive mutation record. Terminate the iterator if so, as + // all the necessary data (checkpoints and all diffs) required has already been + // fetching. + if !found { + term.Store(true) + log.Debug("Failed to read the trienode") + return nil + } + full, _, err := decodeNodeFull(data) + if err != nil { + term.Store(true) + return err + } + if full { + term.Store(true) + } + queue.set(data, pos) + return nil + }) + if term.Load() || !it.Next() { + break + } + } + if err := eg.Wait(); err != nil { + return nil, err + } + if err := it.Error(); err != nil { + return nil, err + } + for i := 0; i < seq; i++ { + isComplete, fullBlob, err := decodeNodeFull(queue.data[i]) + if err != nil { + return nil, err + } + // Terminate the loop is the node with full value has been found + if isComplete { + blob = fullBlob + break + } + // Decode the partial encoded node and keep iterating the node history + // until the node with full value being reached. + element, index, err := decodeNodeCompressed(queue.data[i]) + if err != nil { + return nil, err + } + elements, indices = append(elements, element), append(indices, index) + } + slices.Reverse(elements) + slices.Reverse(indices) + return assembleNode(blob, elements, indices) +} + +// read retrieves the trie node data associated with the stateID. +// stateID: represents the ID of the state of the specified version; +// lastID: represents the ID of the latest/newest trie node history; +// latestValue: represents the trie node value at the current disk layer with ID == lastID; +func (r *trienodeReader) read(state stateIdent, stateID uint64, lastID uint64, latestValue []byte) ([]byte, error) { + _, err := checkStateAvail(state, typeTrienodeHistory, r.freezer, stateID, lastID, r.disk) + if err != nil { + return nil, err + } + // Construct the index iterator to traverse the trienode history + var ( + scheme *indexScheme + it HistoryIndexIterator + ) + if state.addressHash == (common.Hash{}) { + scheme = accountIndexScheme + } else { + scheme = storageIndexScheme + } + if state.addressHash == (common.Hash{}) && state.path == "" { + it = newSeqIter(lastID) + } else { + chunkID, nodeID := scheme.splitPathLast(state.path) + + queryIdent := state + queryIdent.path = chunkID + ir, err := newIndexReader(r.disk, queryIdent, scheme.getBitmapSize(len(chunkID))) + if err != nil { + return nil, err + } + filter := extFilter(nodeID) + it = ir.newIterator(&filter) + } + // Move the iterator to the first element whose id is greater than + // the given number. + found := it.SeekGT(stateID) + if err := it.Error(); err != nil { + return nil, err + } + // The state was not found in the trie node histories, as it has not been + // modified since stateID. Use the data from the associated disk layer + // instead (full value node as always) + if !found { + return latestValue, nil + } + return r.readOptimized(state, it, latestValue) +} + +// checkStateAvail determines whether the requested historical state is available +// for accessing. What's more, it also returns the ID of the latest indexed history +// entry for subsequent usage. +// +// TODO(rjl493456442) it's really expensive to perform the check for every state +// retrieval, please rework this later. +func checkStateAvail(state stateIdent, exptyp historyType, freezer ethdb.AncientReader, stateID uint64, lastID uint64, db ethdb.KeyValueReader) (uint64, error) { + if toHistoryType(state.typ) != exptyp { + return 0, fmt.Errorf("unsupported history type: %d, want: %v", toHistoryType(state.typ), exptyp) + } + // firstID = tail+1 + tail, err := freezer.Tail() + if err != nil { + return 0, err + } + // stateID+1 == firstID is allowed, as all the subsequent history entries + // are present with no gap inside. + if stateID < tail { + return 0, fmt.Errorf("historical state has been pruned, first: %d, state: %d", tail+1, stateID) + } + // To serve the request, all history entries from stateID+1 to lastID + // must be indexed. It's not supposed to happen unless system is very + // wrong. + metadata := loadIndexMetadata(db, exptyp) + if metadata == nil || metadata.Last < lastID { + indexed := "null" + if metadata != nil { + indexed = fmt.Sprintf("%d", metadata.Last) + } + return 0, fmt.Errorf("history is not fully indexed, requested: %d, indexed: %s", stateID, indexed) + } + return metadata.Last, nil +} diff --git a/triedb/pathdb/history_reader_test.go b/triedb/pathdb/history_reader_test.go index dc2894e338..5a1ed28bd2 100644 --- a/triedb/pathdb/history_reader_test.go +++ b/triedb/pathdb/history_reader_test.go @@ -50,7 +50,7 @@ func stateAvail(id uint64, env *tester) bool { return id+1 >= firstID } -func checkHistoricalState(env *tester, root common.Hash, id uint64, hr *historyReader) error { +func checkHistoricalState(env *tester, root common.Hash, id uint64, hr *stateHistoryReader) error { if !stateAvail(id, env) { return nil } @@ -153,7 +153,7 @@ func testHistoryReader(t *testing.T, historyLimit uint64) { var ( roots = env.roots dl = env.db.tree.bottom() - hr = newHistoryReader(env.db.diskdb, env.db.stateFreezer) + hr = newStateHistoryReader(env.db.diskdb, env.db.stateFreezer) ) for i, root := range roots { if root == dl.rootHash() { diff --git a/triedb/pathdb/history_state.go b/triedb/pathdb/history_state.go index bc21915dba..23428b1a54 100644 --- a/triedb/pathdb/history_state.go +++ b/triedb/pathdb/history_state.go @@ -283,11 +283,11 @@ func (h *stateHistory) typ() historyType { // forEach implements the history interface, returning an iterator to traverse the // state entries in the history. -func (h *stateHistory) forEach() iter.Seq[stateIdent] { - return func(yield func(stateIdent) bool) { +func (h *stateHistory) forEach() iter.Seq[indexElem] { + return func(yield func(indexElem) bool) { for _, addr := range h.accountList { addrHash := crypto.Keccak256Hash(addr.Bytes()) - if !yield(newAccountIdent(addrHash)) { + if !yield(accountIndexElem{addrHash}) { return } for _, slotKey := range h.storageList[addr] { @@ -298,7 +298,7 @@ func (h *stateHistory) forEach() iter.Seq[stateIdent] { if h.meta.version != stateHistoryV0 { slotHash = crypto.Keccak256Hash(slotKey.Bytes()) } - if !yield(newStorageIdent(addrHash, slotHash)) { + if !yield(storageIndexElem{addrHash, slotHash}) { return } } diff --git a/triedb/pathdb/history_trienode.go b/triedb/pathdb/history_trienode.go index 3f45b41117..11d6112806 100644 --- a/triedb/pathdb/history_trienode.go +++ b/triedb/pathdb/history_trienode.go @@ -19,9 +19,11 @@ package pathdb import ( "bytes" "encoding/binary" + "errors" "fmt" "iter" "maps" + "math" "slices" "sort" "time" @@ -44,7 +46,10 @@ import ( // - block number (8 bytes) // // - a lexicographically sorted list of trie IDs -// - the corresponding offsets into the key and value sections for each trie data chunk +// - the corresponding offsets into the key and value sections for each trie +// data chunk. The offsets refer to the end position of each chunk, with +// the assumption that the key and value sections for the first data chunk +// start at offset 0. // // Although some fields (e.g., parent state root, block number) are duplicated // between the state history and the trienode history, these two histories @@ -53,19 +58,16 @@ import ( // // # Key section // The key section stores trie node keys (paths) in a compressed format. -// It also contains relative offsets into the value section for resolving -// the corresponding trie node data. Note that these offsets are relative -// to the data chunk for the trie; the chunk offset must be added to obtain -// the absolute position. +// It also contains relative offsets into the value section for locating +// the corresponding trie node data. These offsets are relative to the +// beginning of the trie data chunk, the chunk's base offset must be added +// to obtain the absolute position in the value section. // // # Value section // The value section is a concatenated byte stream of all trie node data. // Each trie node can be retrieved using the offset and length specified // by its index entry. // -// The header and key sections are sufficient for locating a trie node, -// while a partial read of the value section is enough to retrieve its data. - // Header section: // // +----------+------------------+---------------------+---------------------+-------+------------------+---------------------+---------------------| @@ -87,9 +89,9 @@ import ( // // +---- key len ----+ // / \ -// +-------+---------+-----------+---------+-----------------------+-----------------+ -// | shared (varint) | not shared (varint) | value length (varlen) | key (varlen) | -// +-----------------+---------------------+-----------------------+-----------------+ +// +-------+---------+-----------+---------+-----------------------+-----------------------+ +// | shared (varint) | not shared (varint) | value length (varlen) | unshared key (varlen) | +// +-----------------+---------------------+-----------------------+-----------------------+ // // trailer: // @@ -99,9 +101,9 @@ import ( // | restart_1 key offset | restart_1 value offset | ... | restart number (4-bytes) | // +----------------------+------------------------+-----+--------------------------+ // -// Note: Both the key offset and the value offset are relative to the start of -// the trie data chunk. To obtain the absolute offset, add the offset of the -// trie data chunk itself. +// Note: Both the key offset and the value offset are relative to the beginning +// of the trie data chunk. The chunk's base offset must be added to obtain the +// absolute position in the value section. // // Value section: // @@ -138,9 +140,12 @@ type trienodeHistory struct { // newTrienodeHistory constructs a trienode history with the provided trie nodes. func newTrienodeHistory(root common.Hash, parent common.Hash, block uint64, nodes map[common.Hash]map[string][]byte) *trienodeHistory { - nodeList := make(map[common.Hash][]string) + nodeList := make(map[common.Hash][]string, len(nodes)) for owner, subset := range nodes { - keys := sort.StringSlice(slices.Collect(maps.Keys(subset))) + keys := make(sort.StringSlice, 0, len(subset)) + for k := range subset { + keys = append(keys, k) + } keys.Sort() nodeList[owner] = keys } @@ -157,17 +162,6 @@ func newTrienodeHistory(root common.Hash, parent common.Hash, block uint64, node } } -// sharedLen returns the length of the common prefix shared by a and b. -func sharedLen(a, b []byte) int { - n := min(len(a), len(b)) - for i := range n { - if a[i] != b[i] { - return i - } - } - return n -} - // typ implements the history interface, returning the historical data type held. func (h *trienodeHistory) typ() historyType { return typeTrienodeHistory @@ -175,11 +169,35 @@ func (h *trienodeHistory) typ() historyType { // forEach implements the history interface, returning an iterator to traverse the // state entries in the history. -func (h *trienodeHistory) forEach() iter.Seq[stateIdent] { - return func(yield func(stateIdent) bool) { +func (h *trienodeHistory) forEach() iter.Seq[indexElem] { + return func(yield func(indexElem) bool) { for _, owner := range h.owners { - for _, path := range h.nodeList[owner] { - if !yield(newTrienodeIdent(owner, path)) { + var ( + scheme *indexScheme + paths = h.nodeList[owner] + indexes = make(map[string]map[uint16]struct{}) + ) + if owner == (common.Hash{}) { + scheme = accountIndexScheme + } else { + scheme = storageIndexScheme + } + for _, leaf := range findLeafPaths(paths) { + chunks, ids := scheme.splitPath(leaf) + for i := 0; i < len(chunks); i++ { + if _, exists := indexes[chunks[i]]; !exists { + indexes[chunks[i]] = make(map[uint16]struct{}) + } + indexes[chunks[i]][ids[i]] = struct{}{} + } + } + for chunk, ids := range indexes { + elem := trienodeIndexElem{ + owner: owner, + path: chunk, + data: slices.Collect(maps.Keys(ids)), + } + if !yield(elem) { return } } @@ -207,17 +225,22 @@ func (h *trienodeHistory) encode() ([]byte, []byte, []byte, error) { restarts []uint32 prefixLen int - internalKeyOffset uint32 // key offset for the trie internally - internalValOffset uint32 // value offset for the trie internally + internalKeyOffset uint32 // key offset within the trie data internally + internalValOffset uint32 // value offset within the trie data internally ) for i, path := range h.nodeList[owner] { key := []byte(path) + + // Track the internal key and value offsets at the beginning of the + // restart section. The absolute offsets within the key and value + // sections should first include the offset of the trie chunk itself + // stored in the header section. if i%trienodeDataBlockRestartLen == 0 { restarts = append(restarts, internalKeyOffset) restarts = append(restarts, internalValOffset) prefixLen = 0 } else { - prefixLen = sharedLen(prevKey, key) + prefixLen = commonPrefixLen(prevKey, key) } value := h.nodes[owner][path] @@ -256,18 +279,13 @@ func (h *trienodeHistory) encode() ([]byte, []byte, []byte, error) { } // Fill the header section with the offsets of the key and value sections. - // Note that the key/value offsets are intentionally tracked *after* encoding - // them into their respective sections, ensuring each offset refers to the end - // position. For n trie chunks, n offset pairs are sufficient to uniquely locate - // the corresponding data. - headerSection.Write(owner.Bytes()) // 32 bytes - binary.Write(&headerSection, binary.BigEndian, uint32(keySection.Len())) // 4 bytes - - // The offset to the value section is theoretically unnecessary, since the - // individual value offset is already tracked in the key section. However, - // we still keep it here for two reasons: - // - It's cheap to store (only 4 bytes for each trie). - // - It can be useful for decoding the trie data when key is not required (e.g., in hash mode). + // Note that key/value offsets are intentionally recorded *after* encoding + // into their respective sections, so each offset refers to an end position. + // For n trie chunks, n offset pairs are sufficient to uniquely locate each + // chunk's data. For example, [0, offset_0] defines the range of trie chunk 0, + // while [offset_{n-2}, offset_{n-1}] defines the range of trie chunk n-1. + headerSection.Write(owner.Bytes()) // 32 bytes + binary.Write(&headerSection, binary.BigEndian, uint32(keySection.Len())) // 4 bytes binary.Write(&headerSection, binary.BigEndian, uint32(valueSection.Len())) // 4 bytes } return headerSection.Bytes(), keySection.Bytes(), valueSection.Bytes(), nil @@ -330,32 +348,68 @@ func decodeHeader(data []byte) (*trienodeMetadata, []common.Hash, []uint32, []ui }, owners, keyOffsets, valOffsets, nil } -func decodeSingle(keySection []byte, onValue func([]byte, int, int) error) ([]string, error) { - var ( - prevKey []byte - items int - keyOffsets []uint32 - valOffsets []uint32 +// decodeKeyEntry resolves a single entry from the key section starting from +// the specified offset. +func decodeKeyEntry(keySection []byte, offset int) (uint64, uint64, []byte, int, error) { + var byteRead int - keyOff int // the key offset within the single trie data - valOff int // the value offset within the single trie data + // Resolve the length of shared key + nShared, nn := binary.Uvarint(keySection[offset:]) // key length shared (varint) + if nn <= 0 { + return 0, 0, nil, 0, fmt.Errorf("corrupted varint encoding for nShared at offset %d", offset) + } + byteRead += nn - keys []string - ) + // Resolve the length of unshared key + nUnshared, nn := binary.Uvarint(keySection[offset+byteRead:]) // key length not shared (varint) + if nn <= 0 { + return 0, 0, nil, 0, fmt.Errorf("corrupted varint encoding for nUnshared at offset %d", offset+byteRead) + } + byteRead += nn + + // Resolve the length of value + nValue, nn := binary.Uvarint(keySection[offset+byteRead:]) // value length (varint) + if nn <= 0 { + return 0, 0, nil, 0, fmt.Errorf("corrupted varint encoding for nValue at offset %d", offset+byteRead) + } + byteRead += nn + + // Validate that the values can fit in an int to prevent overflow on 32-bit systems + if nShared > uint64(math.MaxUint32) || nUnshared > uint64(math.MaxUint32) || nValue > uint64(math.MaxUint32) { + return 0, 0, nil, 0, errors.New("key/value size too large") + } + + // Resolve the unshared key + if offset+byteRead+int(nUnshared) > len(keySection) { + return 0, 0, nil, 0, fmt.Errorf("key length too long, unshared key length: %d, off: %d, section size: %d", nUnshared, offset+byteRead, len(keySection)) + } + unsharedKey := keySection[offset+byteRead : offset+byteRead+int(nUnshared)] + byteRead += int(nUnshared) + + return nShared, nValue, unsharedKey, byteRead, nil +} + +// decodeRestartTrailer resolves all the offsets recorded at the trailer. +func decodeRestartTrailer(keySection []byte) ([]uint32, []uint32, int, error) { // Decode the number of restart section if len(keySection) < 4 { - return nil, fmt.Errorf("key section too short, size: %d", len(keySection)) + return nil, nil, 0, fmt.Errorf("key section too short, size: %d", len(keySection)) } nRestarts := binary.BigEndian.Uint32(keySection[len(keySection)-4:]) + // Decode the trailer + var ( + keyOffsets = make([]uint32, 0, int(nRestarts)) + valOffsets = make([]uint32, 0, int(nRestarts)) + ) if len(keySection) < int(8*nRestarts)+4 { - return nil, fmt.Errorf("key section too short, restarts: %d, size: %d", nRestarts, len(keySection)) + return nil, nil, 0, fmt.Errorf("key section too short, restarts: %d, size: %d", nRestarts, len(keySection)) } for i := range int(nRestarts) { o := len(keySection) - 4 - (int(nRestarts)-i)*8 keyOffset := binary.BigEndian.Uint32(keySection[o : o+4]) if i != 0 && keyOffset <= keyOffsets[i-1] { - return nil, fmt.Errorf("key offset is out of order, prev: %v, cur: %v", keyOffsets[i-1], keyOffset) + return nil, nil, 0, fmt.Errorf("key offset is out of order, prev: %v, cur: %v", keyOffsets[i-1], keyOffset) } keyOffsets = append(keyOffsets, keyOffset) @@ -363,84 +417,118 @@ func decodeSingle(keySection []byte, onValue func([]byte, int, int) error) ([]st // section have zero-size value. valOffset := binary.BigEndian.Uint32(keySection[o+4 : o+8]) if i != 0 && valOffset < valOffsets[i-1] { - return nil, fmt.Errorf("value offset is out of order, prev: %v, cur: %v", valOffsets[i-1], valOffset) + return nil, nil, 0, fmt.Errorf("value offset is out of order, prev: %v, cur: %v", valOffsets[i-1], valOffset) } valOffsets = append(valOffsets, valOffset) } - keyLimit := len(keySection) - 4 - int(nRestarts)*8 + keyLimit := len(keySection) - 4 - int(nRestarts)*8 // End of key data + return keyOffsets, valOffsets, keyLimit, nil +} +// decodeRestartSection resolves all entries in a restart section. The keyData +// contains the encoded keys for the section. +// +// onValue is the callback function being invoked for each resolved entry. The +// start and limit are the offsets within the restart section, the base value +// offset of the restart section itself should be added by the caller itself. +// What's more, this function should return `aborted == true` if the entry +// resolution should be terminated. +func decodeRestartSection(keyData []byte, onValue func(key []byte, start int, limit int) (bool, error)) error { + var ( + prevKey []byte + items int + + keyOff int // the key offset within the single trie data + valOff int // the value offset within the single trie data + ) // Decode data - for keyOff < keyLimit { - // Validate the key and value offsets within the single trie data chunk - if items%trienodeDataBlockRestartLen == 0 { - restartIndex := items / trienodeDataBlockRestartLen - if restartIndex >= len(keyOffsets) { - return nil, fmt.Errorf("restart index out of range: %d, available restarts: %d", restartIndex, len(keyOffsets)) - } - if keyOff != int(keyOffsets[restartIndex]) { - return nil, fmt.Errorf("key offset is not matched, recorded: %d, want: %d", keyOffsets[restartIndex], keyOff) - } - if valOff != int(valOffsets[restartIndex]) { - return nil, fmt.Errorf("value offset is not matched, recorded: %d, want: %d", valOffsets[restartIndex], valOff) - } + for keyOff < len(keyData) { + nShared, nValue, unsharedKey, nn, err := decodeKeyEntry(keyData, keyOff) + if err != nil { + return err } - // Resolve the entry from key section - nShared, nn := binary.Uvarint(keySection[keyOff:]) // key length shared (varint) keyOff += nn - nUnshared, nn := binary.Uvarint(keySection[keyOff:]) // key length not shared (varint) - keyOff += nn - nValue, nn := binary.Uvarint(keySection[keyOff:]) // value length (varint) - keyOff += nn - - // Resolve unshared key - if keyOff+int(nUnshared) > len(keySection) { - return nil, fmt.Errorf("key length too long, unshared key length: %d, off: %d, section size: %d", nUnshared, keyOff, len(keySection)) - } - unsharedKey := keySection[keyOff : keyOff+int(nUnshared)] - keyOff += int(nUnshared) // Assemble the full key var key []byte if items%trienodeDataBlockRestartLen == 0 { if nShared != 0 { - return nil, fmt.Errorf("unexpected non-zero shared key prefix: %d", nShared) + return fmt.Errorf("unexpected non-zero shared key prefix: %d", nShared) } key = unsharedKey } else { if int(nShared) > len(prevKey) { - return nil, fmt.Errorf("unexpected shared key prefix: %d, prefix key length: %d", nShared, len(prevKey)) + return fmt.Errorf("unexpected shared key prefix: %d, prefix key length: %d", nShared, len(prevKey)) } - key = append([]byte{}, prevKey[:nShared]...) - key = append(key, unsharedKey...) + key = make([]byte, int(nShared)+len(unsharedKey)) + copy(key[:nShared], prevKey[:nShared]) + copy(key[nShared:], unsharedKey) } if items != 0 && bytes.Compare(prevKey, key) >= 0 { - return nil, fmt.Errorf("trienode paths are out of order, prev: %v, cur: %v", prevKey, key) + return fmt.Errorf("trienode paths are out of order, prev: %v, cur: %v", prevKey, key) } prevKey = key - // Resolve value - if onValue != nil { - if err := onValue(key, valOff, valOff+int(nValue)); err != nil { - return nil, err - } + valEnd := valOff + int(nValue) + abort, err := onValue(key, valOff, valEnd) + if err != nil { + return err } - valOff += int(nValue) - + if abort { + return nil + } + valOff = valEnd items++ - keys = append(keys, string(key)) } - if keyOff != keyLimit { - return nil, fmt.Errorf("excessive key data after decoding, offset: %d, size: %d", keyOff, keyLimit) + if keyOff != len(keyData) { + return fmt.Errorf("excessive key data after decoding, offset: %d, size: %d", keyOff, len(keyData)) + } + return nil +} + +// onValue is the callback function being invoked for each resolved entry. The +// start and limit are the offsets within this trie chunk, the base value +// offset of the trie chunk itself should be added by the caller itself. +func decodeSingle(keySection []byte, onValue func([]byte, int, int) error) error { + keyOffsets, valOffsets, keyLimit, err := decodeRestartTrailer(keySection) + if err != nil { + return err + } + for i := 0; i < len(keyOffsets); i++ { + var keyData []byte + if i == len(keyOffsets)-1 { + keyData = keySection[keyOffsets[i]:keyLimit] + } else { + keyData = keySection[keyOffsets[i]:keyOffsets[i+1]] + } + err := decodeRestartSection(keyData, func(key []byte, start int, limit int) (bool, error) { + valStart := int(valOffsets[i]) + start + valLimit := int(valOffsets[i]) + limit + + // Possible in tests + if onValue == nil { + return false, nil + } + if err := onValue(key, valStart, valLimit); err != nil { + return false, err + } + return false, nil // abort=false + }) + if err != nil { + return err + } } - return keys, nil + return nil } func decodeSingleWithValue(keySection []byte, valueSection []byte) ([]string, map[string][]byte, error) { var ( - offset int - nodes = make(map[string][]byte) + offset int + estimated = len(keySection) / 8 + nodes = make(map[string][]byte, estimated) + paths = make([]string, 0, estimated) ) - paths, err := decodeSingle(keySection, func(key []byte, start int, limit int) error { + err := decodeSingle(keySection, func(key []byte, start int, limit int) error { if start != offset { return fmt.Errorf("gapped value section offset: %d, want: %d", start, offset) } @@ -451,7 +539,9 @@ func decodeSingleWithValue(keySection []byte, valueSection []byte) ([]string, ma if start > len(valueSection) || limit > len(valueSection) { return fmt.Errorf("value section out of range: start: %d, limit: %d, size: %d", start, limit, len(valueSection)) } - nodes[string(key)] = valueSection[start:limit] + strkey := string(key) + paths = append(paths, strkey) + nodes[strkey] = valueSection[start:limit] offset = limit return nil @@ -477,7 +567,8 @@ func (h *trienodeHistory) decode(header []byte, keySection []byte, valueSection h.nodes = make(map[common.Hash]map[string][]byte) for i := range len(owners) { - // Resolve the boundary of key section + // Resolve the boundary of the key section, each offset referring + // to the end position of this trie chunk. var keyStart, keyLimit uint32 if i != 0 { keyStart = keyOffsets[i-1] @@ -487,7 +578,8 @@ func (h *trienodeHistory) decode(header []byte, keySection []byte, valueSection return fmt.Errorf("invalid key offsets: keyStart: %d, keyLimit: %d, size: %d", keyStart, keyLimit, len(keySection)) } - // Resolve the boundary of value section + // Resolve the boundary of the value section, each offset referring + // to the end position of this trie chunk. var valStart, valLimit uint32 if i != 0 { valStart = valueOffsets[i-1] @@ -517,14 +609,11 @@ func (ir iRange) len() uint32 { return ir.limit - ir.start } -// singleTrienodeHistoryReader provides read access to a single trie within the -// trienode history. It stores an offset to the trie's position in the history, -// along with a set of per-node offsets that can be resolved on demand. type singleTrienodeHistoryReader struct { - id uint64 - reader ethdb.AncientReader - valueRange iRange // value range within the global value section - valueInternalOffsets map[string]iRange // value offset within the single trie data + id uint64 + reader ethdb.AncientReader + keyData []byte + valueRange iRange } func newSingleTrienodeHistoryReader(id uint64, reader ethdb.AncientReader, keyRange iRange, valueRange iRange) (*singleTrienodeHistoryReader, error) { @@ -532,121 +621,173 @@ func newSingleTrienodeHistoryReader(id uint64, reader ethdb.AncientReader, keyRa if err != nil { return nil, err } - valueOffsets := make(map[string]iRange) - _, err = decodeSingle(keyData, func(key []byte, start int, limit int) error { - valueOffsets[string(key)] = iRange{ - start: uint32(start), - limit: uint32(limit), - } - return nil + return &singleTrienodeHistoryReader{ + id: id, + reader: reader, + keyData: keyData, + valueRange: valueRange, + }, nil +} + +// searchSingle searches for a specific trie node identified by the provided +// key within a single trie node chunk. +// +// It returns the node value's offset range (start and limit) within the +// trie node data. An error is returned if the node cannot be found. +func (sr *singleTrienodeHistoryReader) searchSingle(key []byte) (int, int, bool, error) { + keyOffsets, valOffsets, keyLimit, err := decodeRestartTrailer(sr.keyData) + if err != nil { + return 0, 0, false, err + } + // Binary search against the boundary keys for each restart section + var ( + boundFind bool + boundValueLen uint64 + ) + pos := sort.Search(len(keyOffsets), func(i int) bool { + _, nValue, dkey, _, derr := decodeKeyEntry(sr.keyData[keyOffsets[i]:], 0) + if derr != nil { + err = derr + return false + } + n := bytes.Compare(key, dkey) + if n == 0 { + boundFind = true + boundValueLen = nValue + } + return n <= 0 }) if err != nil { - return nil, err + return 0, 0, false, err } - return &singleTrienodeHistoryReader{ - id: id, - reader: reader, - valueRange: valueRange, - valueInternalOffsets: valueOffsets, - }, nil + // The node is found as the boundary of restart section + if boundFind { + start := valOffsets[pos] + limit := valOffsets[pos] + uint32(boundValueLen) + return int(start), int(limit), true, nil + } + // The node is not found as all others have larger key than the target + if pos == 0 { + return 0, 0, false, nil + } + // Search the target node within the restart section + var keyData []byte + if pos == len(keyOffsets) { + keyData = sr.keyData[keyOffsets[pos-1]:keyLimit] // last section + } else { + keyData = sr.keyData[keyOffsets[pos-1]:keyOffsets[pos]] // non-last section + } + var ( + nStart int + nLimit int + found bool + ) + err = decodeRestartSection(keyData, func(ikey []byte, start, limit int) (bool, error) { + if bytes.Equal(key, ikey) { + nStart = int(valOffsets[pos-1]) + start + nLimit = int(valOffsets[pos-1]) + limit + found = true + return true, nil // abort = true + } + return false, nil // abort = false + }) + if err != nil { + return 0, 0, false, err + } + if !found { + return 0, 0, false, nil + } + return nStart, nLimit, true, nil } // read retrieves the trie node data with the provided node path. -func (sr *singleTrienodeHistoryReader) read(path string) ([]byte, error) { - offset, exists := sr.valueInternalOffsets[path] - if !exists { - return nil, fmt.Errorf("trienode %v not found", []byte(path)) +func (sr *singleTrienodeHistoryReader) read(key []byte) ([]byte, bool, error) { + start, limit, found, err := sr.searchSingle(key) + if err != nil { + return nil, false, err + } + if !found { + return nil, false, nil + } + valStart := uint64(start) + uint64(sr.valueRange.start) + valLen := uint64(limit - start) + value, err := rawdb.ReadTrienodeHistoryValueSection(sr.reader, sr.id, valStart, valLen) + if err != nil { + return nil, false, err } - return rawdb.ReadTrienodeHistoryValueSection(sr.reader, sr.id, uint64(sr.valueRange.start+offset.start), uint64(offset.len())) + return value, true, nil } // trienodeHistoryReader provides read access to node data in the trie node history. // It resolves data from the underlying ancient store only when needed, minimizing // I/O overhead. type trienodeHistoryReader struct { - id uint64 // ID of the associated trienode history - reader ethdb.AncientReader // Database reader of ancient store - keyRanges map[common.Hash]iRange // Key ranges identifying trie chunks - valRanges map[common.Hash]iRange // Value ranges identifying trie chunks - iReaders map[common.Hash]*singleTrienodeHistoryReader // readers for each individual trie chunk + id uint64 // ID of the associated trienode history + reader ethdb.AncientReader // Database reader of ancient store } // newTrienodeHistoryReader constructs the reader for specific trienode history. -func newTrienodeHistoryReader(id uint64, reader ethdb.AncientReader) (*trienodeHistoryReader, error) { - r := &trienodeHistoryReader{ - id: id, - reader: reader, - keyRanges: make(map[common.Hash]iRange), - valRanges: make(map[common.Hash]iRange), - iReaders: make(map[common.Hash]*singleTrienodeHistoryReader), - } - if err := r.decodeHeader(); err != nil { - return nil, err +func newTrienodeHistoryReader(id uint64, reader ethdb.AncientReader) *trienodeHistoryReader { + return &trienodeHistoryReader{ + id: id, + reader: reader, } - return r, nil } // decodeHeader decodes the header section of trienode history. -func (r *trienodeHistoryReader) decodeHeader() error { +func (r *trienodeHistoryReader) decodeHeader(owner common.Hash) (iRange, iRange, bool, error) { header, err := rawdb.ReadTrienodeHistoryHeader(r.reader, r.id) if err != nil { - return err + return iRange{}, iRange{}, false, err } _, owners, keyOffsets, valOffsets, err := decodeHeader(header) if err != nil { - return err + return iRange{}, iRange{}, false, err } - for i, owner := range owners { - // Decode the key range for this trie chunk - var keyStart uint32 - if i != 0 { - keyStart = keyOffsets[i-1] - } - r.keyRanges[owner] = iRange{ - start: keyStart, - limit: keyOffsets[i], - } + pos := sort.Search(len(owners), func(i int) bool { + return owner.Cmp(owners[i]) <= 0 + }) + if pos == len(owners) || owners[pos] != owner { + return iRange{}, iRange{}, false, nil + } + var keyRange iRange + if pos != 0 { + keyRange.start = keyOffsets[pos-1] + } + keyRange.limit = keyOffsets[pos] - // Decode the value range for this trie chunk - var valStart uint32 - if i != 0 { - valStart = valOffsets[i-1] - } - r.valRanges[owner] = iRange{ - start: valStart, - limit: valOffsets[i], - } + var valRange iRange + if pos != 0 { + valRange.start = valOffsets[pos-1] } - return nil + valRange.limit = valOffsets[pos] + return keyRange, valRange, true, nil } // read retrieves the trie node data with the provided TrieID and node path. -func (r *trienodeHistoryReader) read(owner common.Hash, path string) ([]byte, error) { - ir, ok := r.iReaders[owner] - if !ok { - keyRange, exists := r.keyRanges[owner] - if !exists { - return nil, fmt.Errorf("trie %x is unknown", owner) - } - valRange, exists := r.valRanges[owner] - if !exists { - return nil, fmt.Errorf("trie %x is unknown", owner) - } - var err error - ir, err = newSingleTrienodeHistoryReader(r.id, r.reader, keyRange, valRange) - if err != nil { - return nil, err - } - r.iReaders[owner] = ir +func (r *trienodeHistoryReader) read(owner common.Hash, path string) ([]byte, bool, error) { + keyRange, valRange, found, err := r.decodeHeader(owner) + if err != nil { + return nil, false, err + } + if !found { + return nil, false, nil + } + ir, err := newSingleTrienodeHistoryReader(r.id, r.reader, keyRange, valRange) + if err != nil { + return nil, false, err } - return ir.read(path) + return ir.read([]byte(path)) } // writeTrienodeHistory persists the trienode history associated with the given diff layer. -// nolint:unused -func writeTrienodeHistory(writer ethdb.AncientWriter, dl *diffLayer) error { +func writeTrienodeHistory(writer ethdb.AncientWriter, dl *diffLayer, rate uint32) error { start := time.Now() - h := newTrienodeHistory(dl.rootHash(), dl.parent.rootHash(), dl.block, dl.nodes.nodeOrigin) + nodes, err := dl.nodes.encodeNodeHistory(dl.root, rate) + if err != nil { + return err + } + h := newTrienodeHistory(dl.rootHash(), dl.parent.rootHash(), dl.block, nodes) header, keySection, valueSection, err := h.encode() if err != nil { return err @@ -670,7 +811,6 @@ func writeTrienodeHistory(writer ethdb.AncientWriter, dl *diffLayer) error { } // readTrienodeMetadata resolves the metadata of the specified trienode history. -// nolint:unused func readTrienodeMetadata(reader ethdb.AncientReader, id uint64) (*trienodeMetadata, error) { header, err := rawdb.ReadTrienodeHistoryHeader(reader, id) if err != nil { diff --git a/triedb/pathdb/history_trienode_test.go b/triedb/pathdb/history_trienode_test.go index d6b80f61f5..05278c30b1 100644 --- a/triedb/pathdb/history_trienode_test.go +++ b/triedb/pathdb/history_trienode_test.go @@ -19,6 +19,7 @@ package pathdb import ( "bytes" "encoding/binary" + "fmt" "math/rand" "reflect" "testing" @@ -137,14 +138,11 @@ func TestTrienodeHistoryReader(t *testing.T) { } } for i, h := range hs { - tr, err := newTrienodeHistoryReader(uint64(i+1), freezer) - if err != nil { - t.Fatalf("Failed to construct the history reader: %v", err) - } + tr := newTrienodeHistoryReader(uint64(i+1), freezer) for _, owner := range h.owners { nodes := h.nodes[owner] for key, value := range nodes { - blob, err := tr.read(owner, key) + blob, _, err := tr.read(owner, key) if err != nil { t.Fatalf("Failed to read trienode history: %v", err) } @@ -417,23 +415,23 @@ func TestTrienodeHistoryReaderNonExistentPath(t *testing.T) { if err := rawdb.WriteTrienodeHistory(freezer, 1, header, keySection, valueSection); err != nil { t.Fatalf("Failed to write trienode history: %v", err) } - - tr, err := newTrienodeHistoryReader(1, freezer) - if err != nil { - t.Fatalf("Failed to construct history reader: %v", err) - } + tr := newTrienodeHistoryReader(1, freezer) // Try to read a non-existent path - _, err = tr.read(testrand.Hash(), "nonexistent") - if err == nil { - t.Fatal("Expected error for non-existent trie owner") + var ( + err error + found bool + ) + _, found, err = tr.read(testrand.Hash(), "nonexistent") + if found || err != nil { + t.Fatal("Expected not found for non-existent trie owner") } // Try to read from existing owner but non-existent path owner := h.owners[0] - _, err = tr.read(owner, "nonexistent-path") - if err == nil { - t.Fatal("Expected error for non-existent path") + _, found, err = tr.read(owner, "nonexistent-path") + if found || err != nil { + t.Fatal("Expected not found for non-existent path") } } @@ -457,23 +455,19 @@ func TestTrienodeHistoryReaderNilValues(t *testing.T) { if err := rawdb.WriteTrienodeHistory(freezer, 1, header, keySection, valueSection); err != nil { t.Fatalf("Failed to write trienode history: %v", err) } - - tr, err := newTrienodeHistoryReader(1, freezer) - if err != nil { - t.Fatalf("Failed to construct history reader: %v", err) - } + tr := newTrienodeHistoryReader(1, freezer) // Test reading nil values - data1, err := tr.read(owner, "nil1") - if err != nil { + data1, found, err := tr.read(owner, "nil1") + if err != nil || !found { t.Fatalf("Failed to read nil value: %v", err) } if len(data1) != 0 { t.Fatal("Expected nil data for nil value") } - data2, err := tr.read(owner, "nil2") - if err != nil { + data2, found, err := tr.read(owner, "nil2") + if err != nil || !found { t.Fatalf("Failed to read nil value: %v", err) } if len(data2) != 0 { @@ -481,8 +475,8 @@ func TestTrienodeHistoryReaderNilValues(t *testing.T) { } // Test reading non-nil value - data3, err := tr.read(owner, "data1") - if err != nil { + data3, found, err := tr.read(owner, "data1") + if err != nil || !found { t.Fatalf("Failed to read non-nil value: %v", err) } if !bytes.Equal(data3, []byte("some data")) { @@ -498,7 +492,7 @@ func TestTrienodeHistoryReaderNilKey(t *testing.T) { // Add some nil values nodes[owner][""] = []byte("some data") - nodes[owner]["data1"] = []byte("some data") + nodes[owner]["data1"] = []byte("some data1") h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes) @@ -509,14 +503,10 @@ func TestTrienodeHistoryReaderNilKey(t *testing.T) { if err := rawdb.WriteTrienodeHistory(freezer, 1, header, keySection, valueSection); err != nil { t.Fatalf("Failed to write trienode history: %v", err) } - - tr, err := newTrienodeHistoryReader(1, freezer) - if err != nil { - t.Fatalf("Failed to construct history reader: %v", err) - } + tr := newTrienodeHistoryReader(1, freezer) // Test reading nil values - data1, err := tr.read(owner, "") + data1, _, err := tr.read(owner, "") if err != nil { t.Fatalf("Failed to read nil value: %v", err) } @@ -525,63 +515,17 @@ func TestTrienodeHistoryReaderNilKey(t *testing.T) { } // Test reading non-nil value - data2, err := tr.read(owner, "data1") + data2, _, err := tr.read(owner, "data1") if err != nil { t.Fatalf("Failed to read non-nil value: %v", err) } - if !bytes.Equal(data2, []byte("some data")) { + if !bytes.Equal(data2, []byte("some data1")) { t.Fatal("Data mismatch for non-nil key") } } -// TestTrienodeHistoryReaderIterator tests the iterator functionality -func TestTrienodeHistoryReaderIterator(t *testing.T) { - h := makeTrienodeHistory() - - // Count expected entries - expectedCount := 0 - expectedNodes := make(map[stateIdent]bool) - for owner, nodeList := range h.nodeList { - expectedCount += len(nodeList) - for _, node := range nodeList { - expectedNodes[stateIdent{ - typ: typeTrienode, - addressHash: owner, - path: node, - }] = true - } - } - - // Test the iterator - actualCount := 0 - for x := range h.forEach() { - _ = x - actualCount++ - } - if actualCount != expectedCount { - t.Fatalf("Iterator count mismatch: expected %d, got %d", expectedCount, actualCount) - } - - // Test that iterator yields expected state identifiers - seen := make(map[stateIdent]bool) - for ident := range h.forEach() { - if ident.typ != typeTrienode { - t.Fatal("Iterator should only yield trienode history identifiers") - } - key := stateIdent{typ: ident.typ, addressHash: ident.addressHash, path: ident.path} - if seen[key] { - t.Fatal("Iterator yielded duplicate identifier") - } - seen[key] = true - - if !expectedNodes[key] { - t.Fatalf("Unexpected yielded identifier %v", key) - } - } -} - -// TestSharedLen tests the sharedLen helper function -func TestSharedLen(t *testing.T) { +// TestCommonPrefixLen tests the commonPrefixLen helper function +func TestCommonPrefixLen(t *testing.T) { tests := []struct { a, b []byte expected int @@ -610,13 +554,13 @@ func TestSharedLen(t *testing.T) { } for i, test := range tests { - result := sharedLen(test.a, test.b) + result := commonPrefixLen(test.a, test.b) if result != test.expected { t.Errorf("Test %d: sharedLen(%q, %q) = %d, expected %d", i, test.a, test.b, result, test.expected) } // Test commutativity - resultReverse := sharedLen(test.b, test.a) + resultReverse := commonPrefixLen(test.b, test.a) if result != resultReverse { t.Errorf("Test %d: sharedLen is not commutative: sharedLen(a,b)=%d, sharedLen(b,a)=%d", i, result, resultReverse) @@ -678,14 +622,14 @@ func TestDecodeSingleCorruptedData(t *testing.T) { _, keySection, _, _ := h.encode() // Test with empty key section - _, err := decodeSingle([]byte{}, nil) + err := decodeSingle([]byte{}, nil) if err == nil { t.Fatal("Expected error for empty key section") } // Test with key section too small for trailer if len(keySection) > 0 { - _, err := decodeSingle(keySection[:3], nil) // Less than 4 bytes for trailer + err := decodeSingle(keySection[:3], nil) // Less than 4 bytes for trailer if err == nil { t.Fatal("Expected error for key section too small for trailer") } @@ -694,8 +638,11 @@ func TestDecodeSingleCorruptedData(t *testing.T) { // Test with corrupted varint in key section corrupted := make([]byte, len(keySection)) copy(corrupted, keySection) - corrupted[5] = 0xFF // Corrupt varint - _, err = decodeSingle(corrupted, nil) + // Fill first 10 bytes with 0xFF to create a varint overflow (>64 bits) + for i := range 10 { + corrupted[i] = 0xFF + } + err = decodeSingle(corrupted, nil) if err == nil { t.Fatal("Expected error for corrupted varint") } @@ -705,7 +652,7 @@ func TestDecodeSingleCorruptedData(t *testing.T) { copy(corrupted, keySection) // Set restart count to something too large binary.BigEndian.PutUint32(corrupted[len(corrupted)-4:], 10000) - _, err = decodeSingle(corrupted, nil) + err = decodeSingle(corrupted, nil) if err == nil { t.Fatal("Expected error for invalid restart count") } @@ -734,3 +681,57 @@ func testEncodeDecode(t *testing.T, h *trienodeHistory) { t.Fatal("Trienode content mismatch") } } + +func TestSearchSingle(t *testing.T) { + nodes := make(map[common.Hash]map[string][]byte) + ownerA, ownerB := testrand.Hash(), testrand.Hash() + nodes[ownerA] = make(map[string][]byte) + nodes[ownerB] = make(map[string][]byte) + + for i := 0; i < trienodeDataBlockRestartLen*2; i++ { + nodes[ownerA][fmt.Sprintf("%d", 2*i+1)] = testrand.Bytes(rand.Intn(5)) + nodes[ownerB][fmt.Sprintf("%d", 2*i+1)] = testrand.Bytes(rand.Intn(5)) + } + h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes) + + var freezer, _ = rawdb.NewTrienodeFreezer(t.TempDir(), false, false) + defer freezer.Close() + + header, keySection, valueSection, _ := h.encode() + if err := rawdb.WriteTrienodeHistory(freezer, 1, header, keySection, valueSection); err != nil { + t.Fatalf("Failed to write trienode history: %v", err) + } + tr := newTrienodeHistoryReader(1, freezer) + + // Test reading non-existent entry + keys := []string{ + "0", + "2", + "30", + "32", + "64", + "1000", + } + for _, key := range keys { + _, found, err := tr.read(ownerA, key) + if err != nil || found { + t.Fatalf("Expected non-existent entry %v", err) + } + _, found, err = tr.read(ownerB, key) + if err != nil || found { + t.Fatalf("Expected non-existent entry %v", err) + } + } + + for owner, subnodes := range nodes { + for key, value := range subnodes { + got, found, err := tr.read(owner, key) + if err != nil || !found { + t.Fatal("Failed to read trienode") + } + if bytes.Compare(got, value) != 0 { + t.Fatalf("Unexpected value for key %v, got %v, expected %v", []byte(key), got, value) + } + } + } +} diff --git a/triedb/pathdb/history_trienode_utils.go b/triedb/pathdb/history_trienode_utils.go new file mode 100644 index 0000000000..11107494bb --- /dev/null +++ b/triedb/pathdb/history_trienode_utils.go @@ -0,0 +1,344 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package pathdb + +import ( + "encoding/binary" + "fmt" + "math/bits" + "slices" + "strings" +) + +// commonPrefixLen returns the length of the common prefix shared by a and b. +func commonPrefixLen(a, b []byte) int { + n := min(len(a), len(b)) + for i := range n { + if a[i] != b[i] { + return i + } + } + return n +} + +// findLeafPaths scans a lexicographically sorted list of paths and returns +// the subset of paths that represent leaves. +// +// A path is considered a leaf if: +// - it is the last element in the list, or +// - the next path does not have the current path as its prefix. +// +// In other words, a leaf is a path that has no children extending it. +// +// Example: +// +// Input: ["a", "ab", "abc", "b", "ba"] +// Output: ["abc", "ba"] +// +// The input must be sorted; otherwise the result is undefined. +func findLeafPaths(paths []string) []string { + var leaves []string + for i := 0; i < len(paths); i++ { + if i == len(paths)-1 || !strings.HasPrefix(paths[i+1], paths[i]) { + leaves = append(leaves, paths[i]) + } + } + return leaves +} + +// hexPathNodeID computes a numeric node ID from the given path. The path is +// interpreted as a sequence of base-16 digits, where each byte of the input +// is treated as one hexadecimal digit in a big-endian number. +// +// The resulting node ID is constructed as: +// +// ID = 1 + 16 + 16^2 + ... + 16^(n-1) + value +// +// where n is the number of bytes in the path, and `value` is the base-16 +// interpretation of the byte sequence. +// +// The offset (1 + 16 + 16^2 + ... + 16^(n-1)) ensures that all IDs of shorter +// paths occupy a lower numeric range, preserving lexicographic ordering between +// differently-length paths. +// +// The numeric node ID is represented by the uint16 with the assumption the length +// of path won't be greater than 3. +func hexPathNodeID(path string) uint16 { + var ( + offset = uint16(0) + pow = uint16(1) + value = uint16(0) + bytes = []byte(path) + ) + for i := 0; i < len(bytes); i++ { + offset += pow + pow *= 16 + } + for i := 0; i < len(bytes); i++ { + value = value*16 + uint16(bytes[i]) + } + return offset + value +} + +// bitmapSize computes the number of bytes required for the marker bitmap +// corresponding to the remaining portion of a path after a cut point. +// The marker is a bitmap where each bit represents the presence of a +// possible element in the remaining path segment. +func bitmapSize(levels int) int { + // Compute: total = 1 + 16 + 16^2 + ... + 16^(segLen-1) + var ( + bits = 0 + pow = 1 + ) + for i := 0; i < levels; i++ { + bits += pow + pow *= 16 + } + // A small adjustment is applied to exclude the root element of this path + // segment, since any existing element would already imply the mutation of + // the root element. This trick can save us 1 byte for each bitmap which is + // non-trivial. + bits -= 1 + return bits / 8 +} + +// indexScheme defines how trie nodes are split into chunks and index them +// at chunk level. +// +// skipRoot indicates whether the root node should be excluded from indexing. +// cutPoints specifies the key length of chunks (in nibbles) extracted from +// each path. +type indexScheme struct { + // skipRoot indicates whether the root node should be excluded from indexing. + // In the account trie, the root is mutated on every state transition, so + // indexing it provides no value. + skipRoot bool + + // cutPoints defines the key lengths of chunks at different positions. + // A single trie node path may span multiple chunks vertically. + cutPoints []int + + // bitmaps specifies the required bitmap size for each chunk. The key is the + // chunk key length, and the value is the corresponding bitmap size. + bitmaps map[int]int +} + +var ( + // Account trie is split into chunks like this: + // + // - root node is excluded from indexing + // - nodes at level1 to level2 are grouped as 16 chunks + // - all other nodes are grouped 3 levels per chunk + // + // Level1 [0] ... [f] 16 chunks + // Level3 [000] ... [fff] 4096 chunks + // Level6 [000000] ... [fffffff] 16777216 chunks + // + // For the chunks at level1, there are 17 nodes per chunk. + // + // chunk-level 0 [ 0 ] 1 node + // chunk-level 1 [ 1 ] … [ 16 ] 16 nodes + // + // For the non-level1 chunks, there are 273 nodes per chunk, + // regardless of the chunk's depth in the trie. + // + // chunk-level 0 [ 0 ] 1 node + // chunk-level 1 [ 1 ] … [ 16 ] 16 nodes + // chunk-level 2 [ 17 ] … … [ 272 ] 256 nodes + accountIndexScheme = newIndexScheme(true) + + // Storage trie is split into chunks like this: (3 levels per chunk) + // + // Level0 [ ROOT ] 1 chunk + // Level3 [000] ... [fff] 4096 chunks + // Level6 [000000] ... [fffffff] 16777216 chunks + // + // Within each chunk, there are 273 nodes in total, regardless of + // the chunk's depth in the trie. + // + // chunk-level 0 [ 0 ] 1 node + // chunk-level 1 [ 1 ] … [ 16 ] 16 nodes + // chunk-level 2 [ 17 ] … … [ 272 ] 256 nodes + storageIndexScheme = newIndexScheme(false) +) + +// newIndexScheme initializes the index scheme. +func newIndexScheme(skipRoot bool) *indexScheme { + var ( + cuts []int + bitmaps = make(map[int]int) + ) + for v := 0; v <= 64; v += 3 { + var ( + levels int + length int + ) + if v == 0 && skipRoot { + length = 1 + levels = 2 + } else { + length = v + levels = 3 + } + cuts = append(cuts, length) + bitmaps[length] = bitmapSize(levels) + } + return &indexScheme{ + skipRoot: skipRoot, + cutPoints: cuts, + bitmaps: bitmaps, + } +} + +// getBitmapSize returns the required bytes for bitmap with chunk's position. +func (s *indexScheme) getBitmapSize(pathLen int) int { + return s.bitmaps[pathLen] +} + +// chunkSpan returns how many chunks should be spanned with the given path. +func (s *indexScheme) chunkSpan(length int) int { + var n int + for _, cut := range s.cutPoints { + if length >= cut { + n++ + continue + } + } + return n +} + +// splitPath applies the indexScheme to the given path and returns two lists: +// +// - chunkIDs: the progressive chunk IDs cuts defined by the scheme +// - innerIDs: the computed node ID for the path segment following each cut +// +// The scheme defines a set of cut points that partition the path. For each cut: +// +// - chunkIDs[i] is path[:cutPoints[i]] +// - innerIDs[i] is the node ID of the segment path[cutPoints[i] : nextCut-1] +func (s *indexScheme) splitPath(path string) ([]string, []uint16) { + // Special case: the root node of the account trie is mutated in every + // state transition, so its mutation records can be ignored. + n := len(path) + if n == 0 && s.skipRoot { + return nil, nil + } + var ( + // Determine how many chunks are spanned by the path + chunks = s.chunkSpan(n) + chunkIDs = make([]string, 0, chunks) + nodeIDs = make([]uint16, 0, chunks) + ) + for i := 0; i < chunks; i++ { + position := s.cutPoints[i] + chunkIDs = append(chunkIDs, path[:position]) + + var limit int + if i != chunks-1 { + limit = s.cutPoints[i+1] - 1 + } else { + limit = len(path) + } + nodeIDs = append(nodeIDs, hexPathNodeID(path[position:limit])) + } + return chunkIDs, nodeIDs +} + +// splitPathLast returns the path prefix of the deepest chunk spanned by the +// given path, along with its corresponding internal node ID. If the path +// spans no chunks, it returns an empty prefix and 0. +// +// nolint:unused +func (s *indexScheme) splitPathLast(path string) (string, uint16) { + chunkIDs, nodeIDs := s.splitPath(path) + if len(chunkIDs) == 0 { + return "", 0 + } + n := len(chunkIDs) + return chunkIDs[n-1], nodeIDs[n-1] +} + +// encodeIDs sorts the given list of uint16 IDs and encodes them into a +// compact byte slice using variable-length unsigned integer encoding. +func encodeIDs(ids []uint16) []byte { + slices.Sort(ids) + buf := make([]byte, 0, len(ids)) + for _, id := range ids { + buf = binary.AppendUvarint(buf, uint64(id)) + } + return buf +} + +// decodeIDs decodes a sequence of variable-length encoded uint16 IDs from the +// given byte slice and returns them as a set. +// +// Returns an error if the input buffer does not contain a complete Uvarint value. +func decodeIDs(buf []byte) ([]uint16, error) { + var res []uint16 + for len(buf) > 0 { + id, n := binary.Uvarint(buf) + if n <= 0 { + return nil, fmt.Errorf("too short for decoding node id, %v", buf) + } + buf = buf[n:] + res = append(res, uint16(id)) + } + return res, nil +} + +// isAncestor reports whether node x is the ancestor of node y. +func isAncestor(x, y uint16) bool { + for y > x { + y = (y - 1) / 16 // parentID(y) = (y - 1) / 16 + if y == x { + return true + } + } + return false +} + +// isBitSet reports whether the bit at `index` in the byte slice `b` is set. +func isBitSet(b []byte, index int) bool { + return b[index/8]&(1<<(7-index%8)) != 0 +} + +// setBit sets the bit at `index` in the byte slice `b` to 1. +func setBit(b []byte, index int) { + b[index/8] |= 1 << (7 - index%8) +} + +// bitPosTwoBytes returns the positions of set bits in a 2-byte bitmap. +// +// The bitmap is interpreted as a big-endian uint16. Bit positions are +// numbered from 0 to 15, where position 0 corresponds to the most +// significant bit of b[0], and position 15 corresponds to the least +// significant bit of b[1]. +func bitPosTwoBytes(b []byte) []int { + if len(b) != 2 { + panic("expect 2 bytes") + } + var ( + pos []int + mask = binary.BigEndian.Uint16(b) + ) + for mask != 0 { + p := bits.LeadingZeros16(mask) + pos = append(pos, p) + mask &^= 1 << (15 - p) + } + return pos +} diff --git a/triedb/pathdb/history_trienode_utils_test.go b/triedb/pathdb/history_trienode_utils_test.go new file mode 100644 index 0000000000..32bd91166d --- /dev/null +++ b/triedb/pathdb/history_trienode_utils_test.go @@ -0,0 +1,584 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package pathdb + +import ( + "bytes" + "reflect" + "testing" +) + +func TestHexPathNodeID(t *testing.T) { + t.Parallel() + + var suites = []struct { + input string + exp uint16 + }{ + { + input: "", + exp: 0, + }, + { + input: string([]byte{0x0}), + exp: 1, + }, + { + input: string([]byte{0xf}), + exp: 16, + }, + { + input: string([]byte{0x0, 0x0}), + exp: 17, + }, + { + input: string([]byte{0x0, 0xf}), + exp: 32, + }, + { + input: string([]byte{0x1, 0x0}), + exp: 33, + }, + { + input: string([]byte{0x1, 0xf}), + exp: 48, + }, + { + input: string([]byte{0xf, 0xf}), + exp: 272, + }, + { + input: string([]byte{0xf, 0xf, 0xf}), + exp: 4368, + }, + } + for _, suite := range suites { + got := hexPathNodeID(suite.input) + if got != suite.exp { + t.Fatalf("Unexpected node ID for %v: got %d, want %d", suite.input, got, suite.exp) + } + } +} + +func TestFindLeafPaths(t *testing.T) { + t.Parallel() + + tests := []struct { + input []string + expect []string + }{ + { + input: nil, + expect: nil, + }, + { + input: []string{"a"}, + expect: []string{"a"}, + }, + { + input: []string{"", "0", "00", "01", "1"}, + expect: []string{ + "00", + "01", + "1", + }, + }, + { + input: []string{"10", "100", "11", "2"}, + expect: []string{ + "100", + "11", + "2", + }, + }, + { + input: []string{"10", "100000000", "11", "111111111", "2"}, + expect: []string{ + "100000000", + "111111111", + "2", + }, + }, + } + for _, test := range tests { + res := findLeafPaths(test.input) + if !reflect.DeepEqual(res, test.expect) { + t.Fatalf("Unexpected result: %v, expected %v", res, test.expect) + } + } +} + +func TestSplitAccountPath(t *testing.T) { + t.Parallel() + + var suites = []struct { + input string + expPrefix []string + expID []uint16 + }{ + // Length = 0 + { + "", nil, nil, + }, + // Length = 1 + { + string([]byte{0x0}), + []string{ + string([]byte{0x0}), + }, + []uint16{ + 0, + }, + }, + { + string([]byte{0x1}), + []string{ + string([]byte{0x1}), + }, + []uint16{ + 0, + }, + }, + { + string([]byte{0xf}), + []string{ + string([]byte{0xf}), + }, + []uint16{ + 0, + }, + }, + // Length = 2 + { + string([]byte{0x0, 0x0}), + []string{ + string([]byte{0x0}), + }, + []uint16{ + 1, + }, + }, + { + string([]byte{0x0, 0x1}), + []string{ + string([]byte{0x0}), + }, + []uint16{ + 2, + }, + }, + { + string([]byte{0x0, 0xf}), + []string{ + string([]byte{0x0}), + }, + []uint16{ + 16, + }, + }, + { + string([]byte{0xf, 0xf}), + []string{ + string([]byte{0xf}), + }, + []uint16{ + 16, + }, + }, + // Length = 3 + { + string([]byte{0x0, 0x0, 0x0}), + []string{ + string([]byte{0x0}), + string([]byte{0x0, 0x0, 0x0}), + }, + []uint16{ + 1, 0, + }, + }, + // Length = 3 + { + string([]byte{0xf, 0xf, 0xf}), + []string{ + string([]byte{0xf}), + string([]byte{0xf, 0xf, 0xf}), + }, + []uint16{ + 16, 0, + }, + }, + // Length = 4 + { + string([]byte{0x0, 0x0, 0x0, 0x0}), + []string{ + string([]byte{0x0}), + string([]byte{0x0, 0x0, 0x0}), + }, + []uint16{ + 1, 1, + }, + }, + { + string([]byte{0xf, 0xf, 0xf, 0xf}), + []string{ + string([]byte{0xf}), + string([]byte{0xf, 0xf, 0xf}), + }, + []uint16{ + 16, 16, + }, + }, + // Length = 5 + { + string([]byte{0x0, 0x0, 0x0, 0x0, 0x0}), + []string{ + string([]byte{0x0}), + string([]byte{0x0, 0x0, 0x0}), + }, + []uint16{ + 1, 17, + }, + }, + { + string([]byte{0xf, 0xf, 0xf, 0xf, 0xf}), + []string{ + string([]byte{0xf}), + string([]byte{0xf, 0xf, 0xf}), + }, + []uint16{ + 16, 272, + }, + }, + // Length = 6 + { + string([]byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0}), + []string{ + string([]byte{0x0}), + string([]byte{0x0, 0x0, 0x0}), + string([]byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0}), + }, + []uint16{ + 1, 17, 0, + }, + }, + { + string([]byte{0xf, 0xf, 0xf, 0xf, 0xf, 0xf}), + []string{ + string([]byte{0xf}), + string([]byte{0xf, 0xf, 0xf}), + string([]byte{0xf, 0xf, 0xf, 0xf, 0xf, 0xf}), + }, + []uint16{ + 16, 272, 0, + }, + }, + } + for _, suite := range suites { + prefix, id := accountIndexScheme.splitPath(suite.input) + if !reflect.DeepEqual(prefix, suite.expPrefix) { + t.Fatalf("Unexpected prefix for %v: got %v, want %v", suite.input, prefix, suite.expPrefix) + } + if !reflect.DeepEqual(id, suite.expID) { + t.Fatalf("Unexpected ID for %v: got %v, want %v", suite.input, id, suite.expID) + } + } +} + +func TestSplitStoragePath(t *testing.T) { + t.Parallel() + + var suites = []struct { + input string + expPrefix []string + expID []uint16 + }{ + // Length = 0 + { + "", + []string{ + string([]byte{}), + }, + []uint16{ + 0, + }, + }, + // Length = 1 + { + string([]byte{0x0}), + []string{ + string([]byte{}), + }, + []uint16{ + 1, + }, + }, + { + string([]byte{0x1}), + []string{ + string([]byte{}), + }, + []uint16{ + 2, + }, + }, + { + string([]byte{0xf}), + []string{ + string([]byte{}), + }, + []uint16{ + 16, + }, + }, + // Length = 2 + { + string([]byte{0x0, 0x0}), + []string{ + string([]byte{}), + }, + []uint16{ + 17, + }, + }, + { + string([]byte{0x0, 0x1}), + []string{ + string([]byte{}), + }, + []uint16{ + 18, + }, + }, + { + string([]byte{0x0, 0xf}), + []string{ + string([]byte{}), + }, + []uint16{ + 32, + }, + }, + { + string([]byte{0xf, 0xf}), + []string{ + string([]byte{}), + }, + []uint16{ + 272, + }, + }, + // Length = 3 + { + string([]byte{0x0, 0x0, 0x0}), + []string{ + string([]byte{}), + string([]byte{0x0, 0x0, 0x0}), + }, + []uint16{ + 17, 0, + }, + }, + // Length = 3 + { + string([]byte{0xf, 0xf, 0xf}), + []string{ + string([]byte{}), + string([]byte{0xf, 0xf, 0xf}), + }, + []uint16{ + 272, 0, + }, + }, + // Length = 4 + { + string([]byte{0x0, 0x0, 0x0, 0x0}), + []string{ + string([]byte{}), + string([]byte{0x0, 0x0, 0x0}), + }, + []uint16{ + 17, 1, + }, + }, + { + string([]byte{0xf, 0xf, 0xf, 0xf}), + []string{ + string([]byte{}), + string([]byte{0xf, 0xf, 0xf}), + }, + []uint16{ + 272, 16, + }, + }, + // Length = 5 + { + string([]byte{0x0, 0x0, 0x0, 0x0, 0x0}), + []string{ + string([]byte{}), + string([]byte{0x0, 0x0, 0x0}), + }, + []uint16{ + 17, 17, + }, + }, + { + string([]byte{0xf, 0xf, 0xf, 0xf, 0xf}), + []string{ + string([]byte{}), + string([]byte{0xf, 0xf, 0xf}), + }, + []uint16{ + 272, 272, + }, + }, + // Length = 6 + { + string([]byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0}), + []string{ + string([]byte{}), + string([]byte{0x0, 0x0, 0x0}), + string([]byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0}), + }, + []uint16{ + 17, 17, 0, + }, + }, + { + string([]byte{0xf, 0xf, 0xf, 0xf, 0xf, 0xf}), + []string{ + string([]byte{}), + string([]byte{0xf, 0xf, 0xf}), + string([]byte{0xf, 0xf, 0xf, 0xf, 0xf, 0xf}), + }, + []uint16{ + 272, 272, 0, + }, + }, + } + for i, suite := range suites { + prefix, id := storageIndexScheme.splitPath(suite.input) + if !reflect.DeepEqual(prefix, suite.expPrefix) { + t.Fatalf("Test %d, unexpected prefix for %v: got %v, want %v", i, suite.input, prefix, suite.expPrefix) + } + if !reflect.DeepEqual(id, suite.expID) { + t.Fatalf("Test %d, unexpected ID for %v: got %v, want %v", i, suite.input, id, suite.expID) + } + } +} + +func TestIsAncestor(t *testing.T) { + suites := []struct { + x, y uint16 + want bool + }{ + {0, 1, true}, + {0, 16, true}, + {0, 17, true}, + {0, 272, true}, + + {1, 0, false}, + {1, 2, false}, + {1, 17, true}, + {1, 18, true}, + {17, 273, true}, + {1, 1, false}, + } + for _, tc := range suites { + result := isAncestor(tc.x, tc.y) + if result != tc.want { + t.Fatalf("isAncestor(%d, %d) = %v, want %v", tc.x, tc.y, result, tc.want) + } + } +} + +func TestBitmapSet(t *testing.T) { + suites := []struct { + index int + expect []byte + }{ + { + 0, []byte{0b10000000, 0x0}, + }, + { + 1, []byte{0b01000000, 0x0}, + }, + { + 7, []byte{0b00000001, 0x0}, + }, + { + 8, []byte{0b00000000, 0b10000000}, + }, + { + 15, []byte{0b00000000, 0b00000001}, + }, + } + for _, tc := range suites { + var buf [2]byte + setBit(buf[:], tc.index) + + if !bytes.Equal(buf[:], tc.expect) { + t.Fatalf("bitmap = %v, want %v", buf, tc.expect) + } + if !isBitSet(buf[:], tc.index) { + t.Fatal("bit is not set") + } + } +} + +func TestBitPositions(t *testing.T) { + suites := []struct { + input []byte + expect []int + }{ + { + []byte{0b10000000, 0x0}, []int{0}, + }, + { + []byte{0b01000000, 0x0}, []int{1}, + }, + { + []byte{0b00000001, 0x0}, []int{7}, + }, + { + []byte{0b00000000, 0b10000000}, []int{8}, + }, + { + []byte{0b00000000, 0b00000001}, []int{15}, + }, + { + []byte{0b10000000, 0b00000001}, []int{0, 15}, + }, + { + []byte{0b10000001, 0b00000001}, []int{0, 7, 15}, + }, + { + []byte{0b10000001, 0b10000001}, []int{0, 7, 8, 15}, + }, + { + []byte{0b11111111, 0b11111111}, []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + }, + { + []byte{0x0, 0x0}, nil, + }, + } + for _, tc := range suites { + got := bitPosTwoBytes(tc.input) + if !reflect.DeepEqual(got, tc.expect) { + t.Fatalf("Unexpected position set, want: %v, got: %v", tc.expect, got) + } + } +} diff --git a/triedb/pathdb/journal.go b/triedb/pathdb/journal.go index 02bdef5d34..efcc3f2549 100644 --- a/triedb/pathdb/journal.go +++ b/triedb/pathdb/journal.go @@ -338,10 +338,8 @@ func (db *Database) Journal(root common.Hash) error { // but the ancient store is not properly closed, resulting in recent writes // being lost. After a restart, the ancient store would then be misaligned // with the disk layer, causing data corruption. - if db.stateFreezer != nil { - if err := db.stateFreezer.SyncAncient(); err != nil { - return err - } + if err := syncHistory(db.stateFreezer, db.trienodeFreezer); err != nil { + return err } // Store the journal into the database and return var ( diff --git a/triedb/pathdb/lookup.go b/triedb/pathdb/lookup.go index 8b092730f8..719546f410 100644 --- a/triedb/pathdb/lookup.go +++ b/triedb/pathdb/lookup.go @@ -33,6 +33,13 @@ func storageKey(accountHash common.Hash, slotHash common.Hash) [64]byte { return key } +// storageKeySlice returns a key for uniquely identifying the storage slot in +// the slice format. +func storageKeySlice(accountHash common.Hash, slotHash common.Hash) []byte { + key := storageKey(accountHash, slotHash) + return key[:] +} + // lookup is an internal structure used to efficiently determine the layer in // which a state entry resides. type lookup struct { diff --git a/triedb/pathdb/metrics.go b/triedb/pathdb/metrics.go index 31c40053fc..a0a626f9b5 100644 --- a/triedb/pathdb/metrics.go +++ b/triedb/pathdb/metrics.go @@ -73,11 +73,8 @@ var ( stateHistoryDataBytesMeter = metrics.NewRegisteredMeter("pathdb/history/state/bytes/data", nil) stateHistoryIndexBytesMeter = metrics.NewRegisteredMeter("pathdb/history/state/bytes/index", nil) - //nolint:unused - trienodeHistoryBuildTimeMeter = metrics.NewRegisteredResettingTimer("pathdb/history/trienode/time", nil) - //nolint:unused - trienodeHistoryDataBytesMeter = metrics.NewRegisteredMeter("pathdb/history/trienode/bytes/data", nil) - //nolint:unused + trienodeHistoryBuildTimeMeter = metrics.NewRegisteredResettingTimer("pathdb/history/trienode/time", nil) + trienodeHistoryDataBytesMeter = metrics.NewRegisteredMeter("pathdb/history/trienode/bytes/data", nil) trienodeHistoryIndexBytesMeter = metrics.NewRegisteredMeter("pathdb/history/trienode/bytes/index", nil) stateIndexHistoryTimer = metrics.NewRegisteredResettingTimer("pathdb/history/state/index/time", nil) @@ -88,8 +85,9 @@ var ( lookupAddLayerTimer = metrics.NewRegisteredResettingTimer("pathdb/lookup/add/time", nil) lookupRemoveLayerTimer = metrics.NewRegisteredResettingTimer("pathdb/lookup/remove/time", nil) - historicalAccountReadTimer = metrics.NewRegisteredResettingTimer("pathdb/history/account/reads", nil) - historicalStorageReadTimer = metrics.NewRegisteredResettingTimer("pathdb/history/storage/reads", nil) + historicalAccountReadTimer = metrics.NewRegisteredResettingTimer("pathdb/history/account/reads", nil) + historicalStorageReadTimer = metrics.NewRegisteredResettingTimer("pathdb/history/storage/reads", nil) + historicalTrienodeReadTimer = metrics.NewRegisteredResettingTimer("pathdb/history/trienode/reads", nil) ) // Metrics in generation diff --git a/triedb/pathdb/nodes.go b/triedb/pathdb/nodes.go index c6f9e7aece..62c72c1953 100644 --- a/triedb/pathdb/nodes.go +++ b/triedb/pathdb/nodes.go @@ -14,12 +14,14 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +// nolint:unused package pathdb import ( "bytes" "errors" "fmt" + "hash/fnv" "io" "maps" @@ -30,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" ) @@ -424,3 +427,265 @@ func (s *nodeSetWithOrigin) decode(r *rlp.Stream) error { s.computeSize() return nil } + +// encodeNodeCompressed encodes the trie node differences between two consecutive +// versions into byte stream. The format is as below: +// +// - metadata byte layout (1 byte): +// +// ┌──── Bits (from MSB to LSB) ───┐ +// │ 7 │ 6 │ 5 │ 4 │ 3 │ 2 │ 1 │ 0 │ +// └───────────────────────────────┘ +// │ │ │ │ │ │ │ └─ FlagA: set if value is encoded in compressed format +// │ │ │ │ │ │ └───── FlagB: set if no extended bitmap is present after the metadata byte +// │ │ │ │ │ └───────── FlagC: bitmap for node (only used when flagB == 1) +// │ │ │ │ └───────────── FlagD: bitmap for node (only used when flagB == 1) +// │ │ │ └───────────────── FlagE: reserved (marks the presence of the 16th child in a full node) +// │ │ └───────────────────── FlagF: reserved +// │ └───────────────────────── FlagG: reserved +// └───────────────────────────── FlagH: reserved +// +// Note: +// - If flagB is 1, the node refers to a shortNode; +// - flagC indicates whether the key of the shortNode is recorded. +// - flagD indicates whether the value of the shortNode is recorded. +// +// - If flagB is 0, the node refers to a fullNode; +// - each bit in extended bitmap indicates whether the corresponding +// child have been modified. +// +// Example: +// +// 0b_0000_1011 +// +// Bit0=1, Bit1=1 -> node in compressed format, no extended bitmap +// Bit2=0, Bit3=1 -> the key of a short node is not stored; its value is stored. +// +// - 2 bytes extended bitmap (only if the flagB in metadata is 0), each bit +// represents a corresponding child; +// +// - concatenation of original value of modified children along with its size; +func encodeNodeCompressed(addExtension bool, elements [][]byte, indices []int) []byte { + var ( + enc []byte + flag = byte(1) // The compression format indicator + ) + // Pre-allocate the byte slice for the node encoder + size := 1 + if addExtension { + size += 2 + } + for _, element := range elements { + size += len(element) + 1 + } + enc = make([]byte, 0, size) + + if !addExtension { + flag |= 2 // The embedded bitmap indicator + + // Embedded bitmap + for _, pos := range indices { + flag |= 1 << (pos + 2) + } + enc = append(enc, flag) + } else { + // Extended bitmap + bitmap := make([]byte, 2) // bitmaps for at most 16 children + for _, pos := range indices { + // Children[16] is only theoretically possible in the Merkle-Patricia-trie, + // in practice this field is never used in the Ethereum case. If it occurs, + // use the FlagE for marking the presence. + if pos >= 16 { + log.Warn("Unexpected 16th child encountered in a full node") + flag |= 1 << 4 // Use the reserved flagE + continue + } + setBit(bitmap, pos) + } + enc = append(enc, flag) + enc = append(enc, bitmap...) + } + for _, element := range elements { + enc = append(enc, byte(len(element))) // 1 byte is sufficient for element size + enc = append(enc, element...) + } + return enc +} + +// encodeNodeFull encodes the full trie node value into byte stream. The format is +// as below: +// +// - metadata byte layout (1 byte): 0b0 +// - node value +// +// TODO(rjl493456442) it's not allocation efficient, please improve it. +func encodeNodeFull(value []byte) []byte { + enc := make([]byte, len(value)+1) + copy(enc[1:], value) + return enc +} + +// decodeNodeCompressed decodes the byte stream of compressed trie node +// back to the original elements and their indices. +// +// It assumes the byte stream contains a compressed format node. +func decodeNodeCompressed(data []byte) ([][]byte, []int, error) { + if len(data) < 1 { + return nil, nil, errors.New("invalid data: too short") + } + flag := data[0] + if flag&byte(1) == 0 { + return nil, nil, errors.New("invalid data: full node value") + } + noExtend := flag&byte(2) != 0 + + // Reconstruct indices from bitmap + var indices []int + if noExtend { + if flag&byte(4) != 0 { // flagC + indices = append(indices, 0) + } + if flag&byte(8) != 0 { // flagD + indices = append(indices, 1) + } + data = data[1:] + } else { + if len(data) < 3 { + return nil, nil, errors.New("invalid data: too short") + } + bitmap := data[1:3] + indices = bitPosTwoBytes(bitmap) + if flag&byte(16) != 0 { // flagE + indices = append(indices, 16) + log.Info("Unexpected 16th child encountered in a full node") + } + data = data[3:] + } + // Reconstruct elements + elements := make([][]byte, 0, len(indices)) + for i := 0; i < len(indices); i++ { + if len(data) == 0 { + return nil, nil, errors.New("invalid data: missing size byte") + } + // Read element size + size := int(data[0]) + data = data[1:] + + // Check if we have enough data for the element + if len(data) < size { + return nil, nil, fmt.Errorf("invalid data: expected %d bytes, got %d", size, len(data)) + } + // Extract element + if size == 0 { + elements = append(elements, nil) + + // The zero-size element is practically unexpected, for node deletion + // the rlp.EmptyString is still expected. Log loudly for the potential + // programming error. + log.Error("Empty element from compressed node, please open an issue", "raw", data) + } else { + element := make([]byte, size) + copy(element, data[:size]) + data = data[size:] + elements = append(elements, element) + } + } + // Check if all data is consumed + if len(data) != 0 { + return nil, nil, errors.New("invalid data: trailing bytes") + } + return elements, indices, nil +} + +// decodeNodeFull decodes the byte stream of full value trie node. +func decodeNodeFull(data []byte) (bool, []byte, error) { + if len(data) < 1 { + return false, nil, errors.New("invalid data: too short") + } + flag := data[0] + if flag != byte(0) { + return false, nil, nil + } + return true, data[1:], nil +} + +// encodeNodeHistory encodes the history of a node. Typically, the original values +// of dirty nodes serve as the history, but this can lead to significant storage +// overhead. +// +// For full nodes, which often see only a few modified children during state +// transitions, recording the entire child set (up to 16 children at 32 bytes +// each) is inefficient. For short nodes, which often see only the value is +// modified during the state transition, recording the key part is also unnecessary. +// To compress size, we instead record the diff of the node, rather than the +// full value. It's vital to compress the overall trienode history. +// +// However, recovering a node from a series of diffs requires applying multiple +// history records, which is computationally and IO intensive. To mitigate this, we +// periodically record the full value of a node as a checkpoint. The frequency of +// these checkpoints is a tradeoff between the compression rate and read overhead. +func (s *nodeSetWithOrigin) encodeNodeHistory(root common.Hash, rate uint32) (map[common.Hash]map[string][]byte, error) { + var ( + // the set of all encoded node history elements + nodes = make(map[common.Hash]map[string][]byte) + + // encodeFullValue determines whether a node should be encoded + // in full format with a pseudo-random probabilistic algorithm. + encodeFullValue = func(owner common.Hash, path string) bool { + // For trie nodes at the first two levels of the account trie, it is very + // likely that all children are modified within a single state transition. + // In such cases, do not use diff mode. + if owner == (common.Hash{}) && len(path) < 2 { + return true + } + h := fnv.New32a() + h.Write(root.Bytes()) + h.Write(owner.Bytes()) + h.Write([]byte(path)) + return h.Sum32()%rate == 0 + } + ) + for owner, origins := range s.nodeOrigin { + var posts map[string]*trienode.Node + if owner == (common.Hash{}) { + posts = s.nodeSet.accountNodes + } else { + posts = s.nodeSet.storageNodes[owner] + } + nodes[owner] = make(map[string][]byte) + + for path, oldvalue := range origins { + n, exists := posts[path] + if !exists { + // something not expected + return nil, fmt.Errorf("node with origin is not found, %x-%v", owner, []byte(path)) + } + encodeFull := encodeFullValue(owner, path) + if !encodeFull { + // TODO(rjl493456442) the diff-mode reencoding can take non-trivial + // time, like 1-2ms per block, is there any way to mitigate the overhead? + + // Partial encoding is required, try to find the node diffs and + // fallback to the full-value encoding if fails. + // + // The partial encoding will be failed in these certain cases: + // - the node is deleted or was not-existent; + // - the node type has been changed (e.g, from short to full) + nElem, indices, diffs, err := trie.NodeDifference(oldvalue, n.Blob) + if err != nil { + encodeFull = true // fallback to the full node encoding + } else { + // Encode the node difference as the history element + addExt := nElem != 2 // fullNode + blob := encodeNodeCompressed(addExt, diffs, indices) + nodes[owner][path] = blob + } + } + if encodeFull { + // Encode the entire original value as the history element + nodes[owner][path] = encodeNodeFull(oldvalue) + } + } + } + return nodes, nil +} diff --git a/triedb/pathdb/nodes_test.go b/triedb/pathdb/nodes_test.go index 483dc4b1a6..131d0ab012 100644 --- a/triedb/pathdb/nodes_test.go +++ b/triedb/pathdb/nodes_test.go @@ -18,11 +18,13 @@ package pathdb import ( "bytes" + "math/rand" "reflect" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/testrand" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie/trienode" ) @@ -126,3 +128,49 @@ func TestNodeSetWithOriginEncode(t *testing.T) { t.Fatalf("Unexpected data size, got: %d, want: %d", dec2.size, s.size) } } + +func TestEncodeFullNodeCompressed(t *testing.T) { + var ( + elements [][]byte + indices []int + ) + for i := 0; i <= 16; i++ { + if rand.Intn(2) == 0 { + elements = append(elements, testrand.Bytes(20)) + indices = append(indices, i) + } + } + enc := encodeNodeCompressed(true, elements, indices) + decElements, decIndices, err := decodeNodeCompressed(enc) + if err != nil { + t.Fatalf("Failed to decode node compressed, %v", err) + } + if !reflect.DeepEqual(elements, decElements) { + t.Fatalf("Elements are not matched") + } + if !reflect.DeepEqual(indices, decIndices) { + t.Fatalf("Indices are not matched") + } +} + +func TestEncodeShortNodeCompressed(t *testing.T) { + var ( + elements [][]byte + indices []int + ) + for i := 0; i < 2; i++ { + elements = append(elements, testrand.Bytes(20)) + indices = append(indices, i) + } + enc := encodeNodeCompressed(false, elements, indices) + decElements, decIndices, err := decodeNodeCompressed(enc) + if err != nil { + t.Fatalf("Failed to decode node compressed, %v", err) + } + if !reflect.DeepEqual(elements, decElements) { + t.Fatalf("Elements are not matched") + } + if !reflect.DeepEqual(indices, decIndices) { + t.Fatalf("Indices are not matched") + } +} diff --git a/triedb/pathdb/reader.go b/triedb/pathdb/reader.go index 842ac0972e..e3cfbcba8a 100644 --- a/triedb/pathdb/reader.go +++ b/triedb/pathdb/reader.go @@ -47,7 +47,7 @@ type nodeLoc struct { } // string returns the string representation of node location. -func (loc *nodeLoc) string() string { +func (loc nodeLoc) string() string { return fmt.Sprintf("loc: %s, depth: %d", loc.loc, loc.depth) } @@ -200,7 +200,7 @@ func (db *Database) StateReader(root common.Hash) (database.StateReader, error) // historical state. type HistoricalStateReader struct { db *Database - reader *historyReader + reader *stateHistoryReader id uint64 } @@ -234,7 +234,7 @@ func (db *Database) HistoricReader(root common.Hash) (*HistoricalStateReader, er return &HistoricalStateReader{ id: *id, db: db, - reader: newHistoryReader(db.diskdb, db.stateFreezer), + reader: newStateHistoryReader(db.diskdb, db.stateFreezer), }, nil } @@ -318,3 +318,90 @@ func (r *HistoricalStateReader) Storage(address common.Address, key common.Hash) } return r.reader.read(newStorageIdentQuery(address, addrHash, key, keyHash), r.id, dl.stateID(), latest) } + +// HistoricalNodeReader is a wrapper over history reader, providing access to +// historical trie node data. +type HistoricalNodeReader struct { + db *Database + reader *trienodeReader + id uint64 +} + +// HistoricNodeReader constructs a reader for accessing the requested historic state. +func (db *Database) HistoricNodeReader(root common.Hash) (*HistoricalNodeReader, error) { + // Bail out if the state history hasn't been fully indexed + if db.trienodeIndexer == nil || db.trienodeFreezer == nil { + return nil, fmt.Errorf("historical state %x is not available", root) + } + if !db.trienodeIndexer.inited() { + return nil, errors.New("trienode histories haven't been fully indexed yet") + } + // - States at the current disk layer or above are directly accessible + // via `db.NodeReader`. + // + // - States older than the current disk layer (including the disk layer + // itself) are available via `db.HistoricalNodeReader`. + id := rawdb.ReadStateID(db.diskdb, root) + if id == nil { + return nil, fmt.Errorf("state %#x is not available", root) + } + // Ensure the requested trienode history is canonical, states on side chain + // are not accessible. + meta, err := readTrienodeMetadata(db.trienodeFreezer, *id+1) + if err != nil { + return nil, fmt.Errorf("state %#x is not available", root) // e.g., the referred trienode history has been pruned + } + if meta.parent != root { + return nil, fmt.Errorf("state %#x is not canonincal", root) + } + return &HistoricalNodeReader{ + id: *id, + db: db, + reader: newTrienodeReader(db.diskdb, db.trienodeFreezer, int(db.config.FullValueCheckpoint)), + }, nil +} + +// Node directly retrieves the trie node data associated with a particular path, +// within a particular account. An error will be returned if the read operation +// exits abnormally. Specifically, if the layer is already stale. +// +// Note: +// - the returned trie node data is not a copy, please don't modify it. +// - an error will be returned if the requested trie node is not found in database. +func (r *HistoricalNodeReader) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) { + defer func(start time.Time) { + historicalTrienodeReadTimer.UpdateSince(start) + }(time.Now()) + + // TODO(rjl493456442): Theoretically, the obtained disk layer could become stale + // within a very short time window. + // + // While reading the account data while holding `db.tree.lock` can resolve + // this issue, but it will introduce a heavy contention over the lock. + // + // Let's optimistically assume the situation is very unlikely to happen, + // and try to define a low granularity lock if the current approach doesn't + // work later. + dl := r.db.tree.bottom() + latest, h, _, err := dl.node(owner, path, 0) + if err != nil { + return nil, err + } + if h == hash { + return latest, nil + } + blob, err := r.reader.read(newTrienodeIdent(owner, string(path)), r.id, dl.stateID(), latest) + if err != nil { + return nil, err + } + // Error out if the local one is inconsistent with the target. + if crypto.Keccak256Hash(blob) != hash { + blobHex := "nil" + if len(blob) > 0 { + blobHex = hexutil.Encode(blob) + } + log.Error("Unexpected historical trie node", "owner", owner.Hex(), "path", path, "blob", blobHex) + return nil, fmt.Errorf("unexpected historical trie node: (%x %v), blob: %s", owner, path, blobHex) + } + return blob, nil +} diff --git a/triedb/pathdb/states.go b/triedb/pathdb/states.go index dc737c3b53..c54d8b1136 100644 --- a/triedb/pathdb/states.go +++ b/triedb/pathdb/states.go @@ -170,12 +170,13 @@ func (s *stateSet) accountList() []common.Hash { if list != nil { return list } - // No old sorted account list exists, generate a new one. It's possible that - // multiple threads waiting for the write lock may regenerate the list - // multiple times, which is acceptable. s.listLock.Lock() defer s.listLock.Unlock() + // Double check after acquiring the write lock + if list = s.accountListSorted; list != nil { + return list + } list = slices.SortedFunc(maps.Keys(s.accountData), common.Hash.Cmp) s.accountListSorted = list return list @@ -200,12 +201,13 @@ func (s *stateSet) storageList(accountHash common.Hash) []common.Hash { } s.listLock.RUnlock() - // No old sorted account list exists, generate a new one. It's possible that - // multiple threads waiting for the write lock may regenerate the list - // multiple times, which is acceptable. s.listLock.Lock() defer s.listLock.Unlock() + // Double check after acquiring the write lock + if list := s.storageListSorted[accountHash]; list != nil { + return list + } list := slices.SortedFunc(maps.Keys(s.storageData[accountHash]), common.Hash.Cmp) s.storageListSorted[accountHash] = list return list @@ -340,7 +342,10 @@ func (s *stateSet) encode(w io.Writer) error { AddrHashes []common.Hash Accounts [][]byte } - var enc accounts + enc := accounts{ + AddrHashes: make([]common.Hash, 0, len(s.accountData)), + Accounts: make([][]byte, 0, len(s.accountData)), + } for addrHash, blob := range s.accountData { enc.AddrHashes = append(enc.AddrHashes, addrHash) enc.Accounts = append(enc.Accounts, blob) @@ -503,7 +508,10 @@ func (s *StateSetWithOrigin) encode(w io.Writer) error { Addresses []common.Address Accounts [][]byte } - var accounts Accounts + accounts := Accounts{ + Addresses: make([]common.Address, 0, len(s.accountOrigin)), + Accounts: make([][]byte, 0, len(s.accountOrigin)), + } for address, blob := range s.accountOrigin { accounts.Addresses = append(accounts.Addresses, address) accounts.Accounts = append(accounts.Accounts, blob) diff --git a/version/version.go b/version/version.go index dd944470ea..a0e2fe3065 100644 --- a/version/version.go +++ b/version/version.go @@ -18,7 +18,7 @@ package version const ( Major = 1 // Major version component of the current release - Minor = 16 // Minor version component of the current release - Patch = 8 // Patch version component of the current release + Minor = 17 // Minor version component of the current release + Patch = 2 // Patch version component of the current release Meta = "stable" // Version metadata to append to the version string )