diff --git a/bin/bitcoin-rs/src/config.rs b/bin/bitcoin-rs/src/config.rs index 7df86af40..2f7aaea28 100644 --- a/bin/bitcoin-rs/src/config.rs +++ b/bin/bitcoin-rs/src/config.rs @@ -71,6 +71,26 @@ macro_rules! emit_cli { pub(crate) config: Option, #[arg(long = "bitcoin-conf")] pub(crate) bitcoin_conf: Option, + /// Single regular Core-framed archive file for foreground offline import. + /// + /// Meta flag: never reaches `UserConfig`. All three import flags + /// must be present together to run the offline import; otherwise + /// the daemon runs. The file must be the custody archive pin + /// whose bytes equal the indexed logical stream by parity. + #[arg(long)] + pub(crate) corpus: Option, + /// Archive manifest binding the corpus identity and stop tip. + /// + /// Meta flag: never reaches `UserConfig`. Required with + /// `--corpus` and `--state` for the offline import. + #[arg(long)] + pub(crate) manifest: Option, + /// Output certified-state JSON for the offline import. + /// + /// Meta flag: never reaches `UserConfig`. Required with + /// `--corpus` and `--manifest` for the offline import. + #[arg(long)] + pub(crate) state: Option, $( $(#[$fdoc])* $($fcli)* diff --git a/bin/bitcoin-rs/src/main.rs b/bin/bitcoin-rs/src/main.rs index 09d2c86af..8f3c2a85a 100644 --- a/bin/bitcoin-rs/src/main.rs +++ b/bin/bitcoin-rs/src/main.rs @@ -27,8 +27,22 @@ fn main() -> ExitCode { Ok(cli) => cli, Err(error) => error.exit(), }; - let result = config::resolve(cli, std::env::vars_os()) - .and_then(|config| bitcoin_rs_node::run(config, bitcoin_rs_node::RuntimeInputs::default())); + // Foreground offline import shares the timed `{corpus_path}` contract: + // one regular file, never the indexed `sources` directory. All three + // flags must appear together; a partial set is a caller error, not a + // daemon run. + let import = (cli.corpus.clone(), cli.manifest.clone(), cli.state.clone()); + let result = config::resolve(cli, std::env::vars_os()).and_then(|config| match import { + (Some(corpus), Some(manifest), Some(state)) => { + bitcoin_rs_node::offline_import::run_offline_import(config, &corpus, &manifest, &state) + } + (None, None, None) => { + bitcoin_rs_node::run(config, bitcoin_rs_node::RuntimeInputs::default()) + } + _ => Err(anyhow::anyhow!( + "--corpus, --manifest, and --state must be given together" + )), + }); match result { Ok(()) => ExitCode::SUCCESS, Err(error) => { diff --git a/crates/index/src/portable/chain.rs b/crates/index/src/portable/chain.rs index 96aeca282..5143aada6 100644 --- a/crates/index/src/portable/chain.rs +++ b/crates/index/src/portable/chain.rs @@ -348,7 +348,7 @@ impl<'a> BodyReader<'a> { } } -fn decode(bytes: &[u8], expected_len: u32) -> Located { +pub(super) fn decode(bytes: &[u8], expected_len: u32) -> Located { if usize::try_from(expected_len) != Ok(bytes.len()) { return Located::Mismatch; } diff --git a/crates/index/src/portable/export.rs b/crates/index/src/portable/export.rs index 378309d04..c992af672 100644 --- a/crates/index/src/portable/export.rs +++ b/crates/index/src/portable/export.rs @@ -31,6 +31,75 @@ use std::fs::{self, File, OpenOptions}; use std::io::{self, BufWriter, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; +/// Rows per export page. Export streams the family through bounded pages +/// over one snapshot, so export memory stays flat in family size on every +/// backend instead of materializing the whole family before emitting rows. +const EXPORT_PAGE_ROWS: usize = 1_024; +/// Cap on returned key-plus-value bytes per page after the first row. +const EXPORT_PAGE_BYTES: usize = 1 << 20; + +/// Limit for one export page scan. +fn export_page_limit() -> PrefixScanLimit { + PrefixScanLimit { + max_rows: EXPORT_PAGE_ROWS, + max_bytes: EXPORT_PAGE_BYTES, + } +} + +/// Streams every `TxConfirmed` row of one snapshot in stored-key order +/// through bounded pages, calling `emit` once per row. +/// +/// Cancellation is polled every [`CANCEL_POLL_RECORDS`] visited rows. +/// Pages concatenate to the family's stored order; a page scan that +/// returns an empty incomplete page can make no progress, so it refuses +/// rather than loops. +#[expect( + clippy::type_complexity, + reason = "the page emitter signature is the streaming interface; an alias would hide the output contract" +)] +fn stream_family_pages( + snapshot: &dyn TxIndexSnapshot, + limit: PrefixScanLimit, + cancelled: &dyn Fn() -> bool, + emit: &mut dyn FnMut(HashPrefixRow, &[u8]) -> Result<(u64, u64), IndexBundleError>, +) -> Result<(u64, u64), IndexBundleError> { + let mut start_after: Option> = None; + let mut visited = 0_u64; + let mut totals = (0_u64, 0_u64); + loop { + let page = + snapshot.scan_family_page(TxIndexFamily::TxConfirmed, start_after.as_deref(), limit)?; + let mut last_key: Option> = None; + for scan_row in page.rows { + visited += 1; + if visited.is_multiple_of(CANCEL_POLL_RECORDS) && cancelled() { + return Err(IndexBundleError::Cancelled); + } + let (records, bytes) = emit(scan_row.row, &scan_row.value)?; + totals.0 += records; + totals.1 += bytes; + last_key = Some(scan_row.row.to_db_row().to_vec()); + } + if page.complete { + return Ok(totals); + } + let Some(last) = last_key else { + // A conforming page scan always returns its first row while rows + // remain; without one there is no cursor to advance. + return Err(IndexError::from(StorageError::InvalidOperation( + "export page scan returned an empty incomplete page", + )) + .into()); + }; + // Stored keys are fixed-width: the suffixed key is the least key + // strictly above the page, so the next page resumes without overlap + // or gap. + let mut next = last; + next.push(0); + start_after = Some(next); + } +} + /// Directory fsync used by a [`Publication`]. type SyncDir = fn(&Path) -> Result<(), IndexBundleError>; @@ -322,75 +391,6 @@ impl<'a> Captured<'a> { } } -/// Rows per export page. Export streams the family through bounded pages -/// over one snapshot, so export memory stays flat in family size on every -/// backend instead of materializing the whole family before emitting rows. -const EXPORT_PAGE_ROWS: usize = 1_024; -/// Cap on returned key-plus-value bytes per page after the first row. -const EXPORT_PAGE_BYTES: usize = 1 << 20; - -/// Limit for one export page scan. -fn export_page_limit() -> PrefixScanLimit { - PrefixScanLimit { - max_rows: EXPORT_PAGE_ROWS, - max_bytes: EXPORT_PAGE_BYTES, - } -} - -/// Streams every `TxConfirmed` row of one snapshot in stored-key order -/// through bounded pages, calling `emit` once per row. -/// -/// Cancellation is polled every [`CANCEL_POLL_RECORDS`] visited rows. -/// Pages concatenate to the family's stored order; a page scan that -/// returns an empty incomplete page can make no progress, so it refuses -/// rather than loops. -#[expect( - clippy::type_complexity, - reason = "the page emitter signature is the streaming interface; an alias would hide the output contract" -)] -fn stream_family_pages( - snapshot: &dyn TxIndexSnapshot, - limit: PrefixScanLimit, - cancelled: &dyn Fn() -> bool, - emit: &mut dyn FnMut(HashPrefixRow, &[u8]) -> Result<(u64, u64), IndexBundleError>, -) -> Result<(u64, u64), IndexBundleError> { - let mut start_after: Option> = None; - let mut visited = 0_u64; - let mut totals = (0_u64, 0_u64); - loop { - let page = - snapshot.scan_family_page(TxIndexFamily::TxConfirmed, start_after.as_deref(), limit)?; - let mut last_key: Option> = None; - for scan_row in page.rows { - visited += 1; - if visited.is_multiple_of(CANCEL_POLL_RECORDS) && cancelled() { - return Err(IndexBundleError::Cancelled); - } - let (records, bytes) = emit(scan_row.row, &scan_row.value)?; - totals.0 += records; - totals.1 += bytes; - last_key = Some(scan_row.row.to_db_row().to_vec()); - } - if page.complete { - return Ok(totals); - } - let Some(last) = last_key else { - // A conforming page scan always returns its first row while rows - // remain; without one there is no cursor to advance. - return Err(IndexError::from(StorageError::InvalidOperation( - "export page scan returned an empty incomplete page", - )) - .into()); - }; - // Stored keys are fixed-width: the suffixed key is the least key - // strictly above the page, so the next page resumes without overlap - // or gap. - let mut next = last; - next.push(0); - start_after = Some(next); - } -} - /// Turns stored `TxConfirmed` rows into full-txid records, one row at a time. struct RowExporter<'a> { host: &'a BundleHost<'a>, @@ -505,10 +505,15 @@ impl RowExporter<'_> { } #[cfg(test)] +#[expect(clippy::expect_used, reason = "test fixtures prove their own bounds")] mod tests { use super::*; + use crate::types::ScriptHash; + use crate::{ScriptLiveScan, TxIndexPage, TxIndexScan, TxIndexScanRow}; + use bitcoin_rs_primitives::{OutPoint, Txid}; use std::cell::Cell; use std::error::Error; + use std::sync::atomic::{AtomicUsize, Ordering}; type TestResult = Result<(), Box>; @@ -596,6 +601,227 @@ mod tests { Ok(()) } + /// Small page size over a multi-page family: pages concatenate in + /// stored-key order with no overlap, gap, or repeat. + #[test] + fn paged_stream_preserves_row_order_across_small_pages() { + let fake = PagedFake::new(25); + let expected: Vec<[u8; 12]> = fake.rows.iter().map(|(row, _)| row.to_db_row()).collect(); + assert!( + expected.windows(2).all(|pair| pair[0] < pair[1]), + "fixture keys strictly increase" + ); + let limit = PrefixScanLimit { + max_rows: 7, + max_bytes: usize::MAX, + }; + let mut seen = Vec::new(); + let (records, bytes) = stream_family_pages(&fake, limit, &|| false, &mut |row, value| { + seen.push(row.to_db_row()); + let bytes = u64::try_from(value.len()).expect("test row value fits u64"); + Ok((1, bytes)) + }) + .expect("paged stream completes"); + assert_eq!(seen, expected, "pages concatenate in stored-key order"); + assert_eq!(records, 25); + assert_eq!(bytes, 25); + assert!( + fake.pages.load(Ordering::SeqCst) > 1, + "the small page size forced multiple pages" + ); + } + + /// Cancellation trips mid-stream: the poll fires every + /// `CANCEL_POLL_RECORDS` visited rows even across page boundaries. + #[test] + fn paged_stream_stops_at_cancellation() { + let fake = PagedFake::new(5_000); + let emitted = Cell::new(0_usize); + let stop = Cell::new(false); + let result = stream_family_pages( + &fake, + PrefixScanLimit { + max_rows: 7, + max_bytes: usize::MAX, + }, + &|| stop.get(), + &mut |_, _| { + emitted.set(emitted.get() + 1); + if emitted.get() == 100 { + stop.set(true); + } + Ok((1, 0)) + }, + ); + assert!( + matches!(result, Err(IndexBundleError::Cancelled)), + "cancellation aborts the page loop" + ); + assert!( + fake.pages.load(Ordering::SeqCst) > 1, + "cancellation fired after several small pages" + ); + } + + /// An empty family completes with no rows and no error. + #[test] + fn paged_stream_accepts_an_empty_family() { + let fake = PagedFake::new(0); + let (records, bytes) = stream_family_pages( + &fake, + PrefixScanLimit { + max_rows: 7, + max_bytes: usize::MAX, + }, + &|| false, + &mut |_, _| Ok((1, 0)), + ) + .expect("an empty family completes"); + assert_eq!((records, bytes), (0, 0)); + assert_eq!(fake.pages.load(Ordering::SeqCst), 1); + } + + /// A page that returns no rows without completing can make no + /// progress: the loop refuses instead of spinning. + #[test] + fn paged_stream_refuses_a_page_that_makes_no_progress() { + let fake = PagedFake { + rows: Vec::new(), + pages: AtomicUsize::new(0), + stuck: true, + }; + let result = stream_family_pages( + &fake, + PrefixScanLimit { + max_rows: 7, + max_bytes: usize::MAX, + }, + &|| false, + &mut |_, _| Ok((1, 0)), + ); + assert!( + matches!(result, Err(IndexBundleError::Index(_))), + "an empty incomplete page is a seam error, not a spin" + ); + } + + /// In-memory `TxIndexSnapshot` that honors a page limit exactly like the + /// storage seam: key order, first row always returned, byte accounting + /// after it. Row lookups are never served: the page loop under test + /// never calls them, and calling one is an explicit typed error rather + /// than a panic. + struct PagedFake { + rows: Vec<(HashPrefixRow, Vec)>, + pages: AtomicUsize, + stuck: bool, + } + + impl PagedFake { + fn new(count: usize) -> Self { + let mut rows = Vec::with_capacity(count); + for index in 0..count { + let prefix = u64::try_from(index) + .expect("test row count fits u64") + .to_be_bytes(); + let height = u32::try_from(index % 3).expect("test height residue fits u32"); + rows.push((HashPrefixRow::new(prefix, height), vec![0xA5])); + } + Self { + rows, + pages: AtomicUsize::new(0), + stuck: false, + } + } + + fn lookup_error() -> IndexError { + IndexError::from(StorageError::InvalidOperation( + "the paged export test never calls row lookups", + )) + } + } + + impl TxIndexSnapshot for PagedFake { + fn capability_watermark( + &self, + _capability: IndexCapability, + ) -> Result, IndexError> { + Ok(None) + } + + fn scan_family_page( + &self, + _family: TxIndexFamily, + start_after: Option<&[u8]>, + limit: PrefixScanLimit, + ) -> Result { + self.pages.fetch_add(1, Ordering::SeqCst); + if self.stuck { + return Ok(TxIndexPage { + rows: Vec::new(), + complete: false, + }); + } + let mut rows = Vec::new(); + let mut bytes = 0_usize; + for (row, value) in &self.rows { + let key = row.to_db_row(); + if start_after.is_some_and(|cursor| key.as_slice() <= cursor) { + continue; + } + let row_bytes = key.len() + value.len(); + if rows.len() >= limit.max_rows + || (!rows.is_empty() && bytes.saturating_add(row_bytes) > limit.max_bytes) + { + return Ok(TxIndexPage { + rows, + complete: false, + }); + } + bytes = bytes.saturating_add(row_bytes); + rows.push(TxIndexScanRow { + row: *row, + value: value.clone(), + }); + } + Ok(TxIndexPage { + rows, + complete: true, + }) + } + + fn transaction_rows( + &self, + _txid: &Txid, + _limit: PrefixScanLimit, + ) -> Result { + Err(Self::lookup_error()) + } + + fn funding_rows( + &self, + _scripthash: ScriptHash, + _limit: PrefixScanLimit, + ) -> Result { + Err(Self::lookup_error()) + } + + fn spending_rows( + &self, + _outpoint: &OutPoint, + _limit: PrefixScanLimit, + ) -> Result { + Err(Self::lookup_error()) + } + + fn live_rows( + &self, + _scripthash: ScriptHash, + _limit: PrefixScanLimit, + ) -> Result { + Err(Self::lookup_error()) + } + } + #[test] fn committed_publication_keeps_both_files_and_never_replaces_an_entry() -> TestResult { let dir = tempfile::tempdir()?; diff --git a/crates/node/Cargo.toml b/crates/node/Cargo.toml index 75acd145e..d7b1aa87b 100644 --- a/crates/node/Cargo.toml +++ b/crates/node/Cargo.toml @@ -94,6 +94,7 @@ thiserror.workspace = true anyhow.workspace = true rayon.workspace = true sha2.workspace = true +sonic-rs.workspace = true [target.'cfg(unix)'.dependencies] rustix.workspace = true diff --git a/crates/node/src/lib.rs b/crates/node/src/lib.rs index f45ab56bb..5e69c22e9 100644 --- a/crates/node/src/lib.rs +++ b/crates/node/src/lib.rs @@ -27,6 +27,8 @@ mod lifecycle; pub mod metrics; /// Node-owned mining candidate lifecycle coordinator. pub mod mining; +/// Foreground offline import for the strict full-validation comparator. +pub mod offline_import; /// Node-owned effects around authoritative chainstate reorgs. pub mod reorg; /// Top-level node runner. diff --git a/crates/node/src/offline_import.rs b/crates/node/src/offline_import.rs new file mode 100644 index 000000000..9e098edd4 --- /dev/null +++ b/crates/node/src/offline_import.rs @@ -0,0 +1,1381 @@ +//! Foreground offline import for the strict full-validation comparator. +//! +//! Consistency: chain mutation holds the exclusive chain-transition lock +//! (linearizable single-writer per transition); certified reads use the +//! stable whole-set view (snapshot isolation over the UTXO set, blocking +//! commits for the scan); the manifest plus archive pin is bound by hash +//! (a transactional outbox is unnecessary here because publication is a +//! single atomic `create_new` state file after the checkpoint, and reruns +//! either resume by tip comparison under a matching full-validation +//! provenance record or are refused, so no exactly-once delivery is +//! assumed). +//! +//! Consumes the single regular Core-framed archive file bound by +//! `{corpus_path}` (the custody archive pin, parity-bound to the indexed +//! logical stream), the manifest at `{manifest_path}`, and the output state +//! path at `{state_path}`, all alongside the resolved [`NodeConfig`]. +//! +//! Asymmetry is explicit: Bitcoin Core consumes the same file through its +//! checked-in `-loadblock` family (`bitcoin-core/src/init.cpp:529`, +//! `2128-2136` into `ImportBlocks`, `bitcoin-core/src/node/blockstorage.cpp:1270`, +//! `bitcoin-core/src/validation.cpp:4975` `LoadExternalBlockFile`); this +//! module is the bitcoin-rs counterpart. It never takes the custody +//! `sources` directory: that directory holds multiple `blkNNNNN.dat` files +//! while the timed command expects one regular corpus file. +//! +//! Guarantees: exact bytes (framing walk plus per-record manifest binding +//! plus archive digest, verified for skipped prefix records and on reopen +//! too), full validation (`--assume-valid-height 0` with the validation mode +//! pinned to `Full` instead of any inherited live mode, indexes off, no +//! stale persisted derived-index namespace, and no resumed prefix without a +//! matching full-validation provenance record), durable bodies plus undo +//! (authoritative commit path), production clean checkpoint (nonzero when +//! publication fails), foreground only (no fork, no daemonize, no spawned +//! descendants), and an eight-field certified state. Reopen against the same +//! data directory resumes the applied prefix only when the provenance record +//! this importer wrote at certification matches the applied tip; otherwise +//! the rerun is refused and the operator revalidates from a fresh directory. + +use std::fs::File; +use std::io::{Read, Write}; +use std::num::NonZeroU32; +use std::path::{Path, PathBuf}; + +use anyhow::{Context as _, Result, bail}; +use bytes::Bytes; +use sha2::Digest as _; + +use crate::NodeConfig; + +/// Accepted manifest schemas (subsets used here). +/// +/// The authoritative C150 archive producer writes +/// `bitcoin-rs-corpus-manifest` version 1 with `archive {sha256,size}`, +/// `range {start_height,stop_height}`, and `entries`; the comparator-facing +/// shape is `core-framed-archive-manifest-v1` with `archive_sha256`, +/// `archive_bytes`, `start_height`/`stop_height`, and `blocks`. Both +/// normalize into the internal [`Manifest`]; unknown schemas and +/// unsupported corpus versions are refused at the boundary. +const CORE_FRAMED_SCHEMA: &str = "core-framed-archive-manifest-v1"; +const CORPUS_SCHEMA: &str = "bitcoin-rs-corpus-manifest"; +const CORPUS_VERSION: u32 = 1; + +#[derive(Debug, serde::Deserialize)] +struct ManifestBlock { + height: u32, + hash: String, + offset: u64, + payload_length: usize, +} + +#[derive(Debug, serde::Deserialize)] +struct Manifest { + network: String, + network_magic: String, + start_height: u32, + stop_height: u32, + archive_sha256: String, + archive_bytes: u64, + blocks: Vec, +} + +#[derive(Debug, serde::Deserialize)] +struct CorpusRange { + start_height: u32, + stop_height: u32, +} + +#[derive(Debug, serde::Deserialize)] +struct CorpusArchive { + sha256: String, + size: u64, +} + +/// Typed untagged view over the two accepted on-disk shapes. Extra +/// producer members (`corpus_id`, `genesis_hash`, `manifest_sha256`, +/// `source_tip_hash`) are ignored here; pinning beyond archive identity +/// stays with the producer and the comparator. +#[derive(Debug, serde::Deserialize)] +#[serde(untagged)] +enum RawManifest { + Framed { + schema: String, + network: String, + network_magic: String, + start_height: u32, + stop_height: u32, + archive_sha256: String, + archive_bytes: u64, + blocks: Vec, + }, + Corpus { + schema: String, + version: u32, + network: String, + network_magic: String, + range: CorpusRange, + archive: CorpusArchive, + entries: Vec, + }, +} + +/// Certified state written to `{state_path}` (matches the comparator). +#[derive(Debug, serde::Serialize)] +struct CertifiedState { + height: u32, + bestblock: String, + txouts: u64, + total_amount_sat: u64, + muhash: String, + utxo_hash_serialized_3: String, + bodies_available: bool, + disconnect_ready: bool, +} + +const HEADER_LEN: usize = 80; +const MAX_PAYLOAD: usize = 4_000_000; +/// Byte bound on the manifest, enforced while reading so an oversized or +/// unbounded path (a device, a huge file) is refused without buffering it. +const MAX_MANIFEST_BYTES: u64 = 512 * 1024 * 1024; + +/// Runs the foreground offline import and writes the certified state. +/// +/// PRE: `corpus`, `manifest`, and `state` are the expanded placeholder +/// paths; `config` carries `--assume-valid-height 0` with auxiliary +/// indexes off. +/// POST: every archive record is validated and connected, bodies plus undo +/// are durable, the production clean checkpoint is published, and `state` +/// holds the eight certified fields. +/// INVARIANT: no background workers start, no child is spawned, and any +/// refusal fails closed with a nonzero exit (the caller maps `Err` to +/// `ExitCode::FAILURE`). +pub fn run_offline_import( + mut config: NodeConfig, + corpus: &Path, + manifest: &Path, + state: &Path, +) -> Result<()> { + if config.validation.assume_valid_height != 0 { + bail!("offline import requires --assume-valid-height 0 for full validation"); + } + // Offline fast-forward never inherits a live validation mode: the + // comparator contract carries no validation-mode token and the default + // mode skips scripts, so the mode is pinned here, before the chainstate + // opens, instead of flowing from operator config. + config.validation.mode = resolve_offline_validation_mode(config.validation.mode); + if config.indexes.txindex + || config.indexes.blockfilterindex + || config.indexes.script_index.is_enabled() + { + bail!("offline import requires auxiliary indexes off"); + } + refuse_stale_persisted_indexes(&config.data_dir)?; + let manifest_doc = read_manifest(manifest)?; + let expected_magic = config.network.magic(); + let manifest_magic = parse_magic_hex(&manifest_doc.network_magic)?; + if manifest_magic != expected_magic { + bail!("manifest network magic does not match the node network"); + } + check_manifest_network(&manifest_doc, config.network.identity_name())?; + check_manifest_shape(&manifest_doc)?; + + let data_dir = config.data_dir.clone(); + let node = crate::state::NodeState::open(config, None)?; + let chainstate = node.chainstate(); + + let already = chainstate.applied_tip_snapshot(); + let skip = match already.as_ref() { + None => 0usize, + Some(tip) => { + let height = tip.height; + let entry_index = block_index_for_height(manifest_doc.start_height, height)?; + let entry = manifest_doc.blocks.get(entry_index).with_context(|| { + format!("applied tip height {height} is outside the manifest range") + })?; + if entry.height != height { + bail!("applied tip height does not match the manifest entry"); + } + if !entry.hash.eq_ignore_ascii_case(&tip.hash.to_string()) { + bail!("applied tip hash does not match the manifest entry"); + } + // Fail closed on provenance: `TipSnapshot`/`DurableHead` record + // height, hash, work, and transaction count but not the + // `ValidationMode` that committed the prefix, so a prefix + // connected under `AssumeValid`/`Fast` (scripts skipped) is + // indistinguishable from a `Full` one. The skip below runs only + // when this importer's provenance record binds the same tip; + // otherwise the rerun is refused and the operator revalidates + // from a fresh data directory. + require_full_validation_provenance(&data_dir, tip)?; + if height == manifest_doc.stop_height { + // Tip already current: connect nothing, but still walk the + // archive below so the reopen re-binds every record and the + // digest instead of trusting the bytes it skips. + manifest_doc.blocks.len() + } else { + let next = height.checked_add(1).context("applied tip overflow")?; + block_index_for_height(manifest_doc.start_height, next)? + } + } + }; + + let mut stream = open_corpus(corpus)?; + let mut hasher = sha2::Sha256::new(); + let mut total: u64 = 0; + let mut payloads: Vec = Vec::new(); + for (index, entry) in manifest_doc.blocks.iter().enumerate() { + let record = read_record(&mut stream, expected_magic, entry)?; + // Per-record binding runs before the resume skip: already-applied + // prefix bytes stay bound to their indexed entry instead of coasting + // on framing plus the trailing digest. + check_payload_against_entry(entry, &record.payload)?; + hasher.update(&record.framed); + total += u64::try_from(record.framed.len()).context("record length does not fit")?; + if index < skip { + continue; + } + let payload = bitcoin_rs_primitives::BlockPayload::from_raw(Bytes::from(record.payload)) + .map_err(|error| { + anyhow::anyhow!("block at height {} does not decode: {error}", entry.height) + })?; + if !payload + .block() + .block_hash() + .to_string() + .eq_ignore_ascii_case(&entry.hash) + { + bail!("block at height {} header hash diverged", entry.height); + } + payloads.push(payload); + if payloads.len() >= 128 { + connect_batch(&node, &mut payloads)?; + } + } + if !payloads.is_empty() { + connect_batch(&node, &mut payloads)?; + } + let trailing = { + let mut one = [0u8; 1]; + match stream.file.read(&mut one) { + Ok(0) => false, + Ok(_) => true, + Err(error) => bail!("cannot check archive trailing bytes: {error}"), + } + }; + if trailing { + bail!("archive contains trailing bytes after the last record"); + } + if total != manifest_doc.archive_bytes { + bail!("archive size does not match the manifest"); + } + let digest = bitcoin_rs_primitives::encode::hex_encode(&hasher.finalize()); + if digest != manifest_doc.archive_sha256.to_ascii_lowercase() { + bail!("archive digest does not match the manifest"); + } + + verify_and_certify(&node, &data_dir, &manifest_doc, state)?; + Ok(()) +} + +struct FramedRecord { + framed: Vec, + payload: Vec, +} + +struct CorpusStream { + file: File, +} + +fn open_corpus(path: &Path) -> Result { + let file = + File::open(path).with_context(|| format!("cannot open corpus {}", path.display()))?; + let metadata = file + .metadata() + .with_context(|| format!("cannot stat corpus {}", path.display()))?; + if !metadata.is_file() { + bail!("corpus is not a regular file"); + } + Ok(CorpusStream { file }) +} + +fn read_record( + stream: &mut CorpusStream, + expected_magic: [u8; 4], + entry: &ManifestBlock, +) -> Result { + if entry.payload_length < HEADER_LEN || entry.payload_length > MAX_PAYLOAD { + bail!( + "manifest entry at height {} is outside consensus bounds", + entry.height + ); + } + let mut header = [0u8; 8]; + stream.file.read_exact(&mut header).with_context(|| { + format!( + "archive ended inside record header at height {}", + entry.height + ) + })?; + if header[..4] != expected_magic { + bail!( + "archive record at height {} has the wrong network magic", + entry.height + ); + } + let length = usize::try_from(u32::from_le_bytes([ + header[4], header[5], header[6], header[7], + ])) + .context("record length does not fit")?; + if length != entry.payload_length { + bail!("archive record at height {} length diverged", entry.height); + } + let mut payload = vec![0u8; length]; + stream.file.read_exact(&mut payload).with_context(|| { + format!( + "archive ended inside block payload at height {}", + entry.height + ) + })?; + let mut framed = Vec::with_capacity(8 + length); + framed.extend_from_slice(&header); + framed.extend_from_slice(&payload); + Ok(FramedRecord { framed, payload }) +} + +/// Connects one batch of archive blocks, relieving journal retention pressure +/// inline. +/// +/// The foreground import runs no maintenance worker, so a retention-limit +/// refusal would otherwise abort the whole import even though the refused +/// block committed nothing and the tip is unchanged. On a +/// [`bitcoin_rs_chainstate::ApplyError::JournalBackpressure`] refusal +/// carrying a retention limit, this publishes one synchronous clean +/// checkpoint — the same foreground publication the import performs at the +/// end, which compacts the journal — and retries the uncommitted block +/// exactly once. Any other failure, a skipped publication (no applied tip to +/// checkpoint), or a second refusal propagates. +/// +/// Neither the refused attempt nor the checkpoint moves the applied tip, so +/// the inline retry resumes from the same durable state. A cross-process +/// rerun over the same data directory resumes by tip comparison only under +/// a matching provenance record; without one it is refused, so the operator +/// wipes the data directory and revalidates from genesis. +fn connect_batch( + node: &crate::state::NodeState, + batch: &mut Vec, +) -> Result<()> { + let chainstate = node.chainstate(); + let followers = node.chain_followers(); + for payload in batch.drain(..) { + let hash = payload.block().block_hash(); + if let Err(error) = followers.apply_connect(&chainstate, &payload) { + let retention_limited = matches!( + &error, + crate::ConnectMutationError::NotCommitted( + bitcoin_rs_chainstate::ApplyError::JournalBackpressure(inner) + ) if matches!( + inner.as_ref(), + bitcoin_rs_storage::chainstate_journal::JournalWriterError::RetentionLimit { .. } + ) + ); + if !retention_limited { + bail!("connect at block {hash} failed: {error}"); + } + let generation = node.publish_checkpoint().with_context(|| { + format!("journal backpressure relief checkpoint failed at block {hash}") + })?; + if generation.is_none() { + bail!( + "journal backpressure relief checkpoint skipped at block {hash}: no applied tip" + ); + } + tracing::info!( + %hash, + "compacted the chainstate journal during offline import; retrying the refused block" + ); + followers + .apply_connect(&chainstate, &payload) + .map_err(|error| anyhow::anyhow!("connect at block {hash} failed: {error}"))?; + } + } + Ok(()) +} + +fn read_manifest(path: &Path) -> Result { + let raw = read_bounded(path, MAX_MANIFEST_BYTES)?; + // No duplicate-key `Value` is used here: the manifest is decoded into + // typed structs with contiguous-height and packed-offset checks, so a + // duplicated key cannot smuggle a second chain view. + let raw_doc: RawManifest = sonic_rs::from_slice(&raw) + .with_context(|| format!("manifest {} is not valid JSON", path.display()))?; + raw_doc.normalize() +} + +/// Reads at most `limit` bytes, refusing a longer input before buffering +/// more than one byte past the bound. +fn read_bounded(path: &Path, limit: u64) -> Result> { + let file = + File::open(path).with_context(|| format!("cannot read manifest {}", path.display()))?; + let mut raw = Vec::new(); + file.take(limit.saturating_add(1)) + .read_to_end(&mut raw) + .with_context(|| format!("cannot read manifest {}", path.display()))?; + if u64::try_from(raw.len()).unwrap_or(u64::MAX) > limit { + bail!("manifest exceeds its byte bound"); + } + Ok(raw) +} + +impl RawManifest { + fn normalize(self) -> Result { + match self { + Self::Framed { + schema, + network, + network_magic, + start_height, + stop_height, + archive_sha256, + archive_bytes, + blocks, + } => { + if schema != CORE_FRAMED_SCHEMA { + bail!("manifest schema must be {CORE_FRAMED_SCHEMA}"); + } + Ok(Manifest { + network, + network_magic, + start_height, + stop_height, + archive_sha256, + archive_bytes, + blocks, + }) + } + Self::Corpus { + schema, + version, + network, + network_magic, + range, + archive, + entries, + } => { + if schema != CORPUS_SCHEMA { + bail!("manifest schema must be {CORPUS_SCHEMA}"); + } + if version != CORPUS_VERSION { + bail!("manifest version {version} is not supported"); + } + Ok(Manifest { + network, + network_magic, + start_height: range.start_height, + stop_height: range.stop_height, + archive_sha256: archive.sha256, + archive_bytes: archive.size, + blocks: entries, + }) + } + } + } +} + +fn check_manifest_network(doc: &Manifest, expected_network: &str) -> Result<()> { + if doc.network != expected_network { + bail!("manifest network does not match the node network"); + } + Ok(()) +} + +/// Sidecar binding the applied tip this importer fully validated. +/// +/// The persisted tip (`TipSnapshot`) and commit point (`DurableHead`) record +/// height, hash, work, and transaction count, but not the `ValidationMode` +/// that committed each block. A prefix connected under `AssumeValid` +/// (scripts skipped below the anchor while the gate trusted it) or `Fast` +/// (scripts skipped below the header tip) is therefore indistinguishable at +/// reopen from one connected under `Full`. This record closes that gap: it is +/// written only after this importer certifies the tip under the pinned +/// `Full` mode, and a resume skips its prefix only while the record still +/// binds the same tip. +const FULL_VALIDATION_PROVENANCE_FILE: &str = "offline-full-validation-provenance.json"; +const FULL_VALIDATION_PROVENANCE_SCHEMA: &str = "offline-full-validation-provenance-v1"; + +#[derive(Debug, serde::Deserialize, serde::Serialize)] +struct FullValidationProvenance { + schema: String, + height: u32, + tip_hash: String, +} + +/// Maps an absolute block height to its position in the packed +/// `blocks`/`entries` array. Both the reopen entry lookup and the skip +/// count after a tip at `height` go through this coordinate, so a resume +/// at height `H` over a range starting at `S` reads entry `H - S` and +/// skips `H + 1 - S` records. For `S == 0` this is the old `height` / +/// `height + 1` behavior exactly. +fn block_index_for_height(start_height: u32, height: u32) -> Result { + let offset = height + .checked_sub(start_height) + .with_context(|| format!("applied tip height {height} is outside the manifest range"))?; + usize::try_from(offset).context("tip height does not fit") +} + +/// Allows the resume skip only while the provenance record binds the tip. +/// +/// Refuses when the record is absent, unreadable, the wrong schema, or names +/// a different tip: a normal live run under `AssumeValid`/`Fast` leaves no +/// such record, and a stale record (older certification, foreign data dir, +/// torn write) never matches the current tip. Failing closed here keeps the +/// skip from certifying unexecuted scripts as fully validated; the remedy is +/// a fresh data directory, which revalidates every record from genesis. +fn require_full_validation_provenance( + data_dir: &Path, + tip: &bitcoin_rs_chain::TipSnapshot, +) -> Result<()> { + match read_full_validation_provenance(data_dir) { + Ok(record) + if record.height == tip.height + && record.tip_hash.eq_ignore_ascii_case(&tip.hash.to_string()) => + { + Ok(()) + } + Ok(record) => bail!( + "offline import refuses to resume over the applied tip at height {}: the full-validation provenance record binds height {} and no longer matches this tip, so revalidate from a fresh data directory", + tip.height, + record.height + ), + Err(error) => bail!( + "offline import refuses to resume over the applied tip at height {}: no readable full-validation provenance record ({error:#}), so revalidate from a fresh data directory", + tip.height + ), + } +} + +fn provenance_path(data_dir: &Path) -> PathBuf { + data_dir.join(FULL_VALIDATION_PROVENANCE_FILE) +} + +fn read_full_validation_provenance(data_dir: &Path) -> Result { + let path = provenance_path(data_dir); + let raw = std::fs::read(&path) + .with_context(|| format!("cannot read provenance {}", path.display()))?; + if raw.len() > 4096 { + bail!("provenance {} exceeds its bound", path.display()); + } + let doc: FullValidationProvenance = sonic_rs::from_slice(&raw) + .with_context(|| format!("provenance {} is not valid JSON", path.display()))?; + if doc.schema != FULL_VALIDATION_PROVENANCE_SCHEMA { + bail!("provenance {} has an unknown schema", path.display()); + } + Ok(doc) +} + +/// Atomically publishes the full-validation provenance for `tip`. +/// +/// The tmp-file plus rename keeps a torn write from ever reading as a +/// match: a crash between write and rename leaves the previous record (or +/// none), and the resume gate refuses anything but an exact tip bind. +fn write_full_validation_provenance( + data_dir: &Path, + tip: &bitcoin_rs_chain::TipSnapshot, +) -> Result<()> { + let doc = FullValidationProvenance { + schema: FULL_VALIDATION_PROVENANCE_SCHEMA.to_owned(), + height: tip.height, + tip_hash: tip.hash.to_string(), + }; + let mut raw = sonic_rs::to_vec_pretty(&doc).context("cannot encode provenance")?; + raw.push(b'\n'); + let tmp = data_dir.join(".offline-full-validation-provenance.json.tmp"); + std::fs::write(&tmp, &raw) + .with_context(|| format!("cannot write provenance {}", tmp.display()))?; + File::open(&tmp) + .with_context(|| format!("cannot open provenance {}", tmp.display()))? + .sync_all() + .with_context(|| format!("cannot fsync provenance {}", tmp.display()))?; + std::fs::rename(&tmp, provenance_path(data_dir)).context("cannot publish provenance")?; + File::open(data_dir) + .with_context(|| format!("cannot open data dir {}", data_dir.display()))? + .sync_all() + .with_context(|| format!("cannot fsync data dir {}", data_dir.display()))?; + Ok(()) +} + +/// Pins the offline-import validation mode, refusing to inherit. +/// +/// Every incoming mode maps to `Full`: the certified state claims full +/// script validation, so fast-forward must execute every script regardless +/// of the live posture the operator config carries. There is no refusal +/// here because the comparator contract carries no validation-mode token +/// (its default resolves to `AssumeValid`); silently inheriting that +/// default would be the accident this pin prevents. +fn resolve_offline_validation_mode( + _inherited: bitcoin_rs_chainstate::ValidationMode, +) -> bitcoin_rs_chainstate::ValidationMode { + bitcoin_rs_chainstate::ValidationMode::Full +} + +/// Binds one archive record to its manifest block index. +/// +/// Framing (`read_record`) proves network magic and length only, and the +/// trailing archive digest binds bytes late. This check binds early and per +/// record: the payload's header hash must equal the indexed entry hash, +/// including records the resume cursor skips. +fn check_payload_against_entry(entry: &ManifestBlock, payload: &[u8]) -> Result<()> { + let header = payload + .get(..HEADER_LEN) + .with_context(|| format!("block at height {} is shorter than a header", entry.height))?; + // The same computation `Block::block_hash` performs, in `Display` order + // to match the indexed entry hash. + let hash = bitcoin_rs_primitives::encode::double_sha256(header); + if !hash.to_string().eq_ignore_ascii_case(&entry.hash) { + bail!("block at height {} header hash diverged", entry.height); + } + Ok(()) +} + +/// Refuses when the data dir carries persisted derived-index content that +/// this index-off import will neither maintain nor revalidate. +/// +/// [`crate::state::NodeState::open`] leaves the derived-index host disabled +/// under the posture gate above, so a `/txindex` namespace left by +/// an earlier index-enabled run would sit untouched while the chainstate tip +/// moves underneath it. The documented revalidation path is a normal indexed +/// start, whose worker startup reconciles (rewind or rebuild from retained +/// canonical chainstate) the namespace back to the applied tip; this +/// foreground import runs no workers, so it refuses instead. Remove or +/// quarantine the namespace, or revalidate it first, then re-run. +fn refuse_stale_persisted_indexes(data_dir: &Path) -> Result<()> { + let namespace = data_dir.join("txindex"); + if !namespace_has_content(&namespace) { + return Ok(()); + } + bail!( + "stale persisted derived index at {}: remove it or revalidate with an indexed start, then re-run", + namespace.display() + ); +} + +/// True when `path` holds persisted content: a missing path or an empty +/// directory holds nothing stale; anything else (rows, a stray file, a +/// symlink, or an unreadable directory) is state this import will not +/// revalidate. +fn namespace_has_content(path: &Path) -> bool { + match std::fs::read_dir(path) { + Ok(mut entries) => entries.next().is_some(), + Err(_) => std::fs::symlink_metadata(path).is_ok(), + } +} + +fn parse_magic_hex(text: &str) -> Result<[u8; 4]> { + if text.len() != 8 || !text.bytes().all(|byte| byte.is_ascii_hexdigit()) { + bail!("network magic must be 8 hex characters"); + } + let mut out = [0u8; 4]; + for (index, chunk) in out.iter_mut().enumerate() { + let byte: u8 = u8::from_str_radix(&text[index * 2..index * 2 + 2], 16) + .with_context(|| "network magic is not hex")?; + *chunk = byte; + } + Ok(out) +} + +fn check_manifest_shape(doc: &Manifest) -> Result<()> { + if doc.blocks.is_empty() { + bail!("manifest blocks must list every height exactly once"); + } + if doc.stop_height < doc.start_height { + bail!("manifest stop is before start"); + } + let expected = usize::try_from( + doc.stop_height + .checked_sub(doc.start_height) + .context("manifest stop is before start")? + .checked_add(1) + .context("manifest range overflow")?, + ) + .context("manifest range does not fit")?; + if doc.blocks.len() != expected || doc.blocks.len() > 2_000_000 { + bail!("manifest blocks must list every height exactly once"); + } + let mut cursor: u64 = 0; + for (index, entry) in doc.blocks.iter().enumerate() { + let index_height = u32::try_from(index).context("manifest index does not fit")?; + if entry.height + != doc + .start_height + .checked_add(index_height) + .context("manifest height overflow")? + { + bail!("manifest heights must be contiguous from start_height"); + } + if entry.payload_length < HEADER_LEN || entry.payload_length > MAX_PAYLOAD { + bail!("manifest entry payload is outside consensus bounds"); + } + if entry.hash.len() != 64 { + bail!("manifest entry hash must be 64 hex characters"); + } + if entry.offset != cursor { + bail!("manifest entry offset is not packed"); + } + // Linearizable cursor advance: single-writer offsets are packed, so a + // failed add refuses rather than wrapping to a second view. + cursor = cursor + .checked_add(8) + .context("manifest cursor overflow")? + .checked_add( + u64::try_from(entry.payload_length).context("payload length does not fit")?, + ) + .context("manifest cursor overflow")?; + if cursor > doc.archive_bytes { + bail!("manifest entry overruns the archive"); + } + } + if cursor != doc.archive_bytes { + bail!("manifest does not consume the archive exactly"); + } + Ok(()) +} + +fn verify_and_certify( + node: &crate::state::NodeState, + data_dir: &Path, + manifest: &Manifest, + state_path: &Path, +) -> Result<()> { + let chainstate = node.chainstate(); + let tip = chainstate + .applied_tip_snapshot() + .context("offline import produced no applied tip")?; + let stop_entry = manifest + .blocks + .last() + .context("manifest has no stop entry")?; + if tip.height != manifest.stop_height { + bail!("applied tip height does not equal the manifest stop"); + } + if tip.height != stop_entry.height { + bail!("applied tip height does not equal the manifest stop entry"); + } + if !tip.hash.to_string().eq_ignore_ascii_case(&stop_entry.hash) { + bail!("applied tip hash does not equal the manifest stop hash"); + } + let body_store = chainstate + .block_body_store_handle() + .context("running node has no block body store")?; + let body = body_store + .load_block_body(tip.height, tip.hash) + .map_err(|error| anyhow::anyhow!("body proof failed: {error}"))?; + if body.is_none() { + bail!("tip body is not available"); + } + let request = bitcoin_rs_chainstate::verify::VerifyRequest { + level: bitcoin_rs_chainstate::verify::CheckLevel::CheckUndoAndCommitments, + depth: NonZeroU32::new(1), + cancellation: None, + }; + match bitcoin_rs_chainstate::verify::verify_recent_blocks(&chainstate, request) { + Ok(bitcoin_rs_chainstate::verify::VerifyOutcome::Passed { .. }) => {} + Ok(bitcoin_rs_chainstate::verify::VerifyOutcome::SkippedNoBlockData { height }) => { + bail!("verification skipped data this node should hold at height {height}") + } + Err(error) => bail!("verification of the applied tip failed: {error}"), + } + let (txouts, total_amount_sat, muhash, set_hash) = + node.chainstate().utxo_reader().with_stable_view(|view| { + let stats = bitcoin_rs_utxo::stats::scan_coin_stats(view, tip.height, true) + .map_err(|error| anyhow::anyhow!("coin stats scan failed: {error}"))?; + let set_hash = view + .hash_serialized_3() + .map_err(|error| anyhow::anyhow!("hash_serialized_3 failed: {error}"))?; + // Snapshot-isolated read: the stable view blocks commits for the + // scan, so txout counts and hashes belong to one tip. No + // exactly-once delivery is assumed; reruns resume by tip + // comparison under a matching provenance record, and are refused + // without one. + Ok::<_, anyhow::Error>(( + u64::try_from(view.len()) + .map_err(|error| anyhow::anyhow!("txout count does not fit: {error}"))?, + stats.total_amount, + stats.muhash.finalize_hash().to_string(), + set_hash.to_string(), + )) + })?; + let certified = CertifiedState { + height: tip.height, + bestblock: tip.hash.to_string(), + txouts, + total_amount_sat, + muhash, + utxo_hash_serialized_3: set_hash, + bodies_available: true, + disconnect_ready: true, + }; + let generation = node + .publish_checkpoint() + .map_err(|error| anyhow::anyhow!("clean checkpoint publication failed: {error}"))?; + if generation.is_none() { + bail!("clean checkpoint publication skipped: no applied tip"); + } + // Bind full-validation provenance to the certified tip before publishing + // the certified state: the tip below already passed full validation + // (every record connected under the pinned `Full` mode, bodies plus undo + // durable, tip re-verified, checkpoint published), so the record is true + // the moment it lands. Ordering it first keeps a later `write_state` + // failure (for example `create_new` refusing a reused state path) + // resumable: the rerun still matches provenance and re-certifies instead + // of forcing a from-genesis revalidation. + write_full_validation_provenance(data_dir, &tip)?; + write_state(state_path, &certified)?; + Ok(()) +} + +fn write_state(path: &Path, state: &CertifiedState) -> Result<()> { + if let Some(parent) = path.parent() { + if !parent.as_os_str().is_empty() { + std::fs::create_dir_all(parent) + .with_context(|| format!("cannot create state dir {}", parent.display()))?; + } + } + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .with_context(|| format!("cannot create state file {}", path.display()))?; + let mut raw = sonic_rs::to_vec_pretty(state).context("cannot encode certified state")?; + raw.push(b'\n'); + file.write_all(&raw) + .with_context(|| format!("cannot write state file {}", path.display()))?; + file.sync_all() + .with_context(|| format!("cannot fsync state file {}", path.display()))?; + drop(file); + if let Some(parent) = path.parent() { + if !parent.as_os_str().is_empty() { + let dir = File::open(parent) + .with_context(|| format!("cannot open state dir {}", parent.display()))?; + dir.sync_all() + .with_context(|| format!("cannot fsync state dir {}", parent.display()))?; + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + #![expect(clippy::expect_used, reason = "test fixtures panic on setup failure")] + + use super::*; + + fn entry(height: u32, hash: &str, offset: u64, len: usize) -> ManifestBlock { + ManifestBlock { + height, + hash: hash.to_owned(), + offset, + payload_length: len, + } + } + + #[test] + fn manifest_shape_rejects_gaps() { + let doc = Manifest { + network: "mainnet".to_owned(), + network_magic: "f9beb4d9".to_owned(), + start_height: 0, + stop_height: 1, + archive_sha256: "ab".repeat(32), + archive_bytes: 200, + blocks: vec![ + entry(0, &"ab".repeat(32), 0, 90), + entry(0, &"ab".repeat(32), 98, 90), + ], + }; + assert!(check_manifest_shape(&doc).is_err()); + } + + #[test] + fn manifest_shape_rejects_unpacked_offset() { + let doc = Manifest { + network: "mainnet".to_owned(), + network_magic: "f9beb4d9".to_owned(), + start_height: 0, + stop_height: 0, + archive_sha256: "ab".repeat(32), + archive_bytes: 98, + blocks: vec![entry(0, &"ab".repeat(32), 8, 90)], + }; + assert!(check_manifest_shape(&doc).is_err()); + } + + #[test] + fn magic_hex_rejects_short_input() { + assert!(parse_magic_hex("f9be").is_err()); + assert_eq!( + parse_magic_hex("f9beb4d9").expect("valid magic"), + [0xf9, 0xbe, 0xb4, 0xd9] + ); + } + + #[test] + fn manifest_reader_rejects_non_json() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("manifest.json"); + std::fs::write(&path, b"not json").expect("write"); + assert!(read_manifest(&path).is_err()); + } + + #[test] + fn corpus_open_rejects_directory() { + let dir = tempfile::tempdir().expect("tempdir"); + assert!(open_corpus(dir.path()).is_err()); + } + + #[test] + fn state_writer_refuses_to_clobber() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("state.json"); + let state = CertifiedState { + height: 1, + bestblock: "ab".repeat(32), + txouts: 2, + total_amount_sat: 5_000_000_000, + muhash: "cd".repeat(32), + utxo_hash_serialized_3: "ef".repeat(32), + bodies_available: true, + disconnect_ready: true, + }; + write_state(&path, &state).expect("first write"); + assert!(write_state(&path, &state).is_err()); + } + + #[test] + fn connect_batch_relieve_journal_retention_pressure() { + use bitcoin_rs_chain::regtest_fixture; + + // The foreground import runs no maintenance worker, so retention + // pressure must be relieved inline: a batch connect that the journal + // refuses publishes one synchronous checkpoint (compacting the + // journal, including the pressure below) and retries the uncommitted + // block once. Without the relief this second connect would abort the + // import exactly like the proved refusal below. + let dir = tempfile::tempdir().expect("tempdir"); + let mut config = + crate::NodeConfig::default_for_network(bitcoin_rs_primitives::Network::Regtest); + config.data_dir = dir.path().join("backpressure-node"); + config.p2p.listen.clear(); + config.chainstate_journal.max_journal_mib = 1; + config.chainstate_journal.rotate_mib = 1; + + let genesis = bitcoin_rs_primitives::Network::Regtest.genesis_block(); + let state = crate::state::NodeState::open(config.clone(), None).expect("open node"); + let genesis_tip = state + .apply_block(&bitcoin_rs_primitives::BlockPayload::from_local( + genesis.clone(), + )) + .expect("genesis"); + state.publish_checkpoint().expect("baseline checkpoint"); + + // Inflate the measured journal size past the retention budget: the + // next apply is refused before any tip mutation while the pressure + // stands. The relief checkpoint compacts every superseded segment, + // so this same file is gone before the retry runs. + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open( + config + .data_dir + .join("chainstate-journal/segment-9999999999.log"), + ) + .expect("pressure file") + .set_len(1024 * 1024) + .expect("pressure size"); + + let child = + regtest_fixture::mined_regtest_child_at(genesis.block_hash(), 1).expect("mine child"); + assert!( + matches!( + state.apply_block(&bitcoin_rs_primitives::BlockPayload::from_local( + child.clone() + )), + Err(crate::ConnectMutationError::NotCommitted( + bitcoin_rs_chainstate::ApplyError::JournalBackpressure(_) + )) + ), + "the fixture must prove retention pressure refuses the child" + ); + + let mut batch = vec![bitcoin_rs_primitives::BlockPayload::from_local(child)]; + connect_batch(&state, &mut batch).expect("relief retries the refused block"); + assert!( + batch.is_empty(), + "the batch drains even through the relief path" + ); + let tip = state + .chainstate() + .applied_tip_snapshot() + .expect("relief must commit the child"); + assert_ne!( + tip.as_ref(), + &genesis_tip, + "the tip must advance past genesis" + ); + assert_eq!(tip.height, 1, "the retried child commits at height 1"); + } + + fn corpus_manifest_bytes(network: &str, magic: &str) -> Vec { + format!( + concat!( + r#"{{"archive":{{"sha256":"{archive}","size":196}},"corpus_id":"C150","entries":"#, + r#"[{{"height":100,"hash":"{h0}","offset":0,"payload_length":90}},"#, + r#"{{"height":101,"hash":"{h1}","offset":98,"payload_length":90}}],"#, + r#""genesis_hash":"{h0}","manifest_sha256":"{msha}","#, + r#""network":"{network}","network_magic":"{magic}","#, + r#""range":{{"start_height":100,"stop_height":101}},"#, + r#""schema":"bitcoin-rs-corpus-manifest","#, + r#""source_tip_hash":"{h1}","version":1}}"# + ), + archive = "ab".repeat(32), + h0 = "ab".repeat(32), + h1 = "cd".repeat(32), + msha = "00".repeat(32), + network = network, + magic = magic, + ) + .into_bytes() + } + + #[test] + fn corpus_schema_normalizes_to_internal_manifest() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("manifest.json"); + std::fs::write(&path, corpus_manifest_bytes("regtest", "fabfb5da")).expect("write"); + let doc = read_manifest(&path).expect("corpus manifest parses"); + assert_eq!(doc.network, "regtest"); + assert_eq!(doc.start_height, 100); + assert_eq!(doc.stop_height, 101); + assert_eq!(doc.archive_bytes, 196); + assert_eq!(doc.blocks.len(), 2); + assert_eq!(doc.blocks[0].height, 100); + assert_eq!(doc.blocks[1].offset, 98); + assert!(check_manifest_shape(&doc).is_ok()); + } + + #[test] + fn manifest_reader_rejects_unknown_schema() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("manifest.json"); + std::fs::write( + &path, + br#"{"schema":"unknown-manifest-v9","network":"regtest","network_magic":"fabfb5da","start_height":0,"stop_height":0,"archive_sha256":"ab","archive_bytes":0,"blocks":[]}"#, + ) + .expect("write"); + assert!(read_manifest(&path).is_err()); + } + + #[test] + fn manifest_reader_rejects_unsupported_corpus_version() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("manifest.json"); + let mut raw = corpus_manifest_bytes("regtest", "fabfb5da"); + let text = String::from_utf8(raw.clone()).expect("utf8"); + raw = text + .replace(r#""version":1"#, r#""version":2"#) + .into_bytes(); + std::fs::write(&path, raw).expect("write"); + assert!(read_manifest(&path).is_err()); + } + + #[test] + fn manifest_network_matches_configured_identity() { + let regtest = bitcoin_rs_primitives::Network::Regtest.identity_name(); + assert_eq!(regtest, "regtest"); + let doc = Manifest { + network: "regtest".to_owned(), + network_magic: "fabfb5da".to_owned(), + start_height: 100, + stop_height: 101, + archive_sha256: "ab".repeat(32), + archive_bytes: 196, + blocks: vec![ + entry(100, &"ab".repeat(32), 0, 90), + entry(101, &"cd".repeat(32), 98, 90), + ], + }; + assert!(check_manifest_network(&doc, regtest).is_ok()); + assert!(check_manifest_network(&doc, "mainnet").is_err()); + } + + #[test] + fn offline_import_pins_full_validation_mode() { + use bitcoin_rs_chainstate::ValidationMode; + // The pin refuses to inherit: every incoming live mode maps to Full, + // so an operator default (AssumeValid) or a fast sync posture can + // never leak script skipping into the certified state. + for inherited in [ + ValidationMode::Full, + ValidationMode::AssumeValid, + ValidationMode::Fast, + ] { + assert_eq!( + resolve_offline_validation_mode(inherited), + ValidationMode::Full + ); + } + } + + fn binding_entry(height: u32, hash: &str) -> ManifestBlock { + entry(height, hash, 0, HEADER_LEN) + } + + fn header_hash_hex(header: &[u8; HEADER_LEN]) -> String { + // Same computation the binding check performs, so the fixture hash + // is pinned to the indexed display order rather than the helper. + bitcoin_rs_primitives::encode::double_sha256(header).to_string() + } + + #[test] + fn connected_records_stay_bound_to_the_manifest() { + // The per-record binding covers every record the import connects, so + // a corrupted payload of the right length is still refused. + let header = [7u8; HEADER_LEN]; + let doc_hash = header_hash_hex(&header); + let good = binding_entry(3, &doc_hash); + let mut payload = header.to_vec(); + check_payload_against_entry(&good, &payload).expect("bound record"); + // Uppercase manifest hashes normalize the same way. + let upper = binding_entry(3, &doc_hash.to_ascii_uppercase()); + check_payload_against_entry(&upper, &payload).expect("case-insensitive"); + // One flipped payload bit diverges from the indexed entry. + payload[10] ^= 0x01; + assert!(check_payload_against_entry(&good, &payload).is_err()); + // A wrong entry hash refuses even with intact bytes. + let wrong = binding_entry(3, &"ab".repeat(32)); + assert!(check_payload_against_entry(&wrong, &header).is_err()); + // A runt that framing would already reject never binds. + let runt = binding_entry(3, &doc_hash); + assert!(check_payload_against_entry(&runt, &header[..HEADER_LEN - 1]).is_err()); + } + + #[test] + fn offline_import_refuses_stale_persisted_index_namespace() { + // No namespace, or an empty one, holds nothing stale. + let fresh = tempfile::tempdir().expect("tempdir"); + refuse_stale_persisted_indexes(fresh.path()).expect("absent namespace"); + std::fs::create_dir(fresh.path().join("txindex")).expect("empty namespace"); + refuse_stale_persisted_indexes(fresh.path()).expect("empty namespace"); + // Rows left by an earlier index-enabled run refuse, even though the + // config posture gate already passed with indexes off. + std::fs::write(fresh.path().join("txindex").join("CURRENT"), b"stale") + .expect("seed stale row"); + assert!(refuse_stale_persisted_indexes(fresh.path()).is_err()); + // A stray non-directory artifact is state this import will not + // revalidate either. + let stray = tempfile::tempdir().expect("tempdir"); + std::fs::write(stray.path().join("txindex"), b"stale").expect("seed stray file"); + assert!(refuse_stale_persisted_indexes(stray.path()).is_err()); + } + + /// Two real regtest blocks (genesis plus one mined child) framed as a + /// Core archive with a matching manifest, so the provenance tests run the + /// real entry point instead of asserting on helpers alone. + struct TwoBlockArchive { + blocks: Vec, + corpus: Vec, + manifest: Vec, + stop_hash: String, + } + + fn two_block_archive() -> TwoBlockArchive { + use sha2::Digest as _; + let genesis = bitcoin_rs_primitives::Network::Regtest.genesis_block(); + let child = bitcoin_rs_chain::regtest_fixture::mined_regtest_child_at( + genesis.block_hash(), + 1, + ) + .expect("mine child"); + let blocks = vec![genesis, child]; + let magic = bitcoin_rs_primitives::Network::Regtest.magic(); + let mut corpus = Vec::new(); + let mut entries = Vec::new(); + let mut offset: u64 = 0; + for (index, block) in blocks.iter().enumerate() { + let payload = + bitcoin_rs_primitives::BlockPayload::from_local(block.clone()); + let raw = payload.bytes(); + let mut framed = Vec::with_capacity(8 + raw.len()); + framed.extend_from_slice(&magic); + framed.extend_from_slice( + &u32::try_from(raw.len()).expect("payload fits").to_le_bytes(), + ); + framed.extend_from_slice(raw); + entries.push(format!( + r#"{{"height":{index},"hash":"{hash}","offset":{offset},"payload_length":{len}}}"#, + index = u32::try_from(index).expect("index fits"), + hash = block.block_hash().to_string(), + len = raw.len() + )); + offset += u64::try_from(framed.len()).expect("offset fits"); + corpus.extend_from_slice(&framed); + } + let mut hasher = sha2::Sha256::new(); + hasher.update(&corpus); + let manifest = format!( + r#"{{"schema":"core-framed-archive-manifest-v1","network":"regtest","network_magic":"fabfb5da","start_height":0,"stop_height":1,"archive_sha256":"{digest}","archive_bytes":{bytes},"blocks":[{blocks}]}}"#, + digest = bitcoin_rs_primitives::encode::hex_encode(&hasher.finalize()), + bytes = corpus.len(), + blocks = entries.join(",") + ); + let stop_hash = blocks[1].block_hash().to_string(); + TwoBlockArchive { + blocks, + corpus, + manifest: manifest.into_bytes(), + stop_hash, + } + } + + fn offline_import_config(data_dir: &std::path::Path) -> crate::NodeConfig { + let mut config = + crate::NodeConfig::default_for_network(bitcoin_rs_primitives::Network::Regtest); + config.data_dir = data_dir.to_path_buf(); + config.p2p.listen.clear(); + config + } + + fn write_archive_fixture( + dir: &std::path::Path, + archive: &TwoBlockArchive, + ) -> (std::path::PathBuf, std::path::PathBuf) { + let corpus = dir.join("corpus.bin"); + let manifest = dir.join("manifest.json"); + std::fs::write(&corpus, &archive.corpus).expect("write corpus"); + std::fs::write(&manifest, &archive.manifest).expect("write manifest"); + (corpus, manifest) + } + + #[test] + fn offline_import_fresh_dir_validates_and_records_provenance() { + // Fresh data directory: every record is connected under the pinned + // `Full` mode, the tip certifies, and the provenance record binds the + // certified tip for later resumes. + let archive = two_block_archive(); + let dir = tempfile::tempdir().expect("tempdir"); + let data_dir = dir.path().join("node"); + let (corpus, manifest) = write_archive_fixture(dir.path(), &archive); + let state = dir.path().join("state.json"); + run_offline_import(offline_import_config(&data_dir), &corpus, &manifest, &state) + .expect("fresh import succeeds"); + let record = + read_full_validation_provenance(&data_dir).expect("provenance recorded"); + assert_eq!(record.height, 1, "provenance binds the certified tip"); + assert!( + record.tip_hash.eq_ignore_ascii_case(&archive.stop_hash), + "provenance binds the certified tip hash" + ); + let text = + String::from_utf8(std::fs::read(&state).expect("read state")).expect("utf8"); + assert!(text.contains("\"height\": 1"), "state certifies height 1"); + assert!( + text.contains(&archive.stop_hash), + "state certifies the archive tip" + ); + } + + #[test] + fn offline_import_certified_dir_resumes_idempotently() { + // The provenance record this importer wrote at certification lets a + // rerun skip the already-validated prefix: the tip is current, so no + // block is reconnected, but the archive is still walked and rebound. + let archive = two_block_archive(); + let dir = tempfile::tempdir().expect("tempdir"); + let data_dir = dir.path().join("node"); + let (corpus, manifest) = write_archive_fixture(dir.path(), &archive); + run_offline_import( + offline_import_config(&data_dir), + &corpus, + &manifest, + &dir.path().join("state.json"), + ) + .expect("first import succeeds"); + let resume_state = dir.path().join("state-resume.json"); + run_offline_import( + offline_import_config(&data_dir), + &corpus, + &manifest, + &resume_state, + ) + .expect("certified directory resumes"); + let text = + String::from_utf8(std::fs::read(&resume_state).expect("read state")).expect("utf8"); + assert!(text.contains("\"height\": 1"), "resume still certifies height 1"); + assert!( + text.contains(&archive.stop_hash), + "resume still certifies the archive tip" + ); + } + + #[test] + fn offline_import_refuses_tip_without_provenance() { + // A data directory whose tip came from anywhere but this importer + // (here: blocks connected directly, as a live `AssumeValid`/`Fast` + // run would leave them) carries no provenance record, so the import + // refuses instead of skipping unexecuted scripts into a full-validation + // certificate. The corpus path is never even opened: the tip check + // runs first. + let archive = two_block_archive(); + let dir = tempfile::tempdir().expect("tempdir"); + let data_dir = dir.path().join("node"); + { + let live = + crate::state::NodeState::open(offline_import_config(&data_dir), None) + .expect("open node"); + for block in &archive.blocks { + live.apply_block(&bitcoin_rs_primitives::BlockPayload::from_local( + block.clone(), + )) + .expect("connect"); + } + // No provenance written; dropping releases the data-dir lock so + // the entry point below can reopen the directory. + } + let (corpus, manifest) = write_archive_fixture(dir.path(), &archive); + let error = run_offline_import( + offline_import_config(&data_dir), + &corpus, + &manifest, + &dir.path().join("state.json"), + ) + .expect_err("tip without provenance refuses"); + let message = error.to_string(); + assert!( + message.contains("provenance") && message.contains("fresh data directory"), + "refusal must name provenance and the remedy, got: {message}" + ); + } + + #[test] + fn network_magic_refuses_non_hex_without_panicking() { + assert_eq!( + parse_magic_hex("f9beb4d9").expect("hex"), + [0xf9, 0xbe, 0xb4, 0xd9] + ); + // Eight bytes but multibyte UTF-8: refused, not sliced off a char + // boundary. + assert!(parse_magic_hex("\u{e9}\u{e9}f9be").is_err()); + // Sign prefixes `from_str_radix` would accept are not hex digits. + assert!(parse_magic_hex("+fbeb4d9").is_err()); + assert!(parse_magic_hex("f9beb4d").is_err()); + } + + #[test] + fn manifest_read_refuses_oversized_input_without_buffering_it() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("manifest.json"); + std::fs::write(&path, [b' '; 17]).expect("manifest"); + assert_eq!(read_bounded(&path, 17).expect("at bound").len(), 17); + let error = read_bounded(&path, 16).expect_err("oversized manifest"); + assert!(error.to_string().contains("byte bound"), "{error}"); + } +} diff --git a/crates/utxo/src/snapshot.rs b/crates/utxo/src/snapshot.rs index 31506e453..a5b806fbd 100644 --- a/crates/utxo/src/snapshot.rs +++ b/crates/utxo/src/snapshot.rs @@ -832,9 +832,11 @@ impl VerifiedSnapshotArtifact { .byte_len() .checked_add(1) .ok_or(RawSeedError::Integrity)?; - // Buffer the small header/output/script reads in memory. The buffer - // sits below the tee, so the digest still covers exactly the bytes - // the decoder consumes; prefetch never reaches the hash. + // Seeding issues one tiny read per output header and script, so the + // capped file reader is buffered in memory. The buffer sits below + // the tee on the consumed-byte side, so the witnessed length and + // digest still cover exactly what the parser reads while the file + // sees bulk reads; prefetch never reaches the hash. let capped = Read::by_ref(&mut self.file).take(limit); let mut buffered = BufReader::new(capped); let mut tee = TeeReader::new(&mut buffered); diff --git a/docs/benchmarks/end-to-end-sync.md b/docs/benchmarks/end-to-end-sync.md index 814e8d95a..cf314f3b5 100644 --- a/docs/benchmarks/end-to-end-sync.md +++ b/docs/benchmarks/end-to-end-sync.md @@ -6,7 +6,7 @@ This document owns the two synchronization lanes of the target node: offline rep | Lane | Boundary | Owner path | Comparator | |---|---|---|---| -| Offline replay | spawn to durable clean exit over a hash-pinned Core-framed archive | `bounded ingress -> parse once -> contextual preparation -> parallel proof -> ordered durable commit -> coherent publication` | [`offline-full-validation.md`](offline-full-validation.md) | +| Offline replay | spawn to durable clean exit over a hash-pinned Core-framed archive (harness also proves dual packed/indexed parity, still `planned_not_executed` live) | `bounded ingress -> parse once -> contextual preparation -> parallel proof -> ordered durable commit -> coherent publication` | [`offline-full-validation.md`](offline-full-validation.md) | | Live IBD, loopback | start to pinned stop over deterministic loopback peers, no public nodes | `P2pService` download window plus the same commit spine | [`p2p-loopback.md`](p2p-loopback.md) | Both lanes run under full validation (`--assume-valid-height 0`), strict-Rust cryptography, default unpruned fjall, optional indexes off, on the final artifact. Every run records the six sample identities and the pinned stop identity. Restart and reorg recovery to each intermediate committed ancestor is exercised in the same lanes (T12, T13). diff --git a/docs/benchmarks/hot-path-ledger.toml b/docs/benchmarks/hot-path-ledger.toml index c1ab84f58..b15e57f96 100644 --- a/docs/benchmarks/hot-path-ledger.toml +++ b/docs/benchmarks/hot-path-ledger.toml @@ -304,7 +304,7 @@ domains = ["offline"] timed_region = "outside" seams = ["tools/benchmark-campaign/offline_full_validation.py"] disposition = "already_bounded" -evidence = "Outside the timed region by #44/#46. Not a node hot path. The retired mainnet_prefix_replay example is not a seam." +evidence = "Outside the timed region by #44/#46. Harness stages packed archive plus indexed blk parity (verify_index once plus exhaust iter, every blk pinned 0o400 with pre/post identity checks). Dual custody staging cost grows outside the timed region. Not a node hot path. The retired mainnet_prefix_replay example is not a seam." # --------------------------------------------------------------------------- # P2P-only diff --git a/docs/benchmarks/offline-full-validation.md b/docs/benchmarks/offline-full-validation.md index 051db7fbe..3f1b0328a 100644 --- a/docs/benchmarks/offline-full-validation.md +++ b/docs/benchmarks/offline-full-validation.md @@ -7,8 +7,10 @@ and implements the frozen parity contract from issue #46. Both Bitcoin Core 31.1 and bitcoin-rs construct chainstate from the **same** hash-pinned Core-framed archive. The comparator never reaches inside either node. A ratio is computed only after every custody and correctness gate -passes. This repository does not claim a live C150 or Cmodern campaign: CI -proves the harness with fixture nodes. +passes. The harness also accepts a dual packed/indexed input for custody +parity: the packed archive plus a producer-owned indexed blk stream over the +same verified logical bytes. This repository does not claim a live C150 or +Cmodern campaign: CI proves the harness with fixture nodes only. ## What is held identical @@ -26,23 +28,52 @@ One `offline-full-validation-config-v1` document binds every arm: (`000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f` from Core `CMainParams`), not only against synthetic fixture bytes hashed with the same algorithm. -- **Manifest**: `core-framed-archive-manifest-v1` names network, magic, - inclusive height range, archive digest and size, and one packed entry per - height (`hash`, `offset`, `payload_length`). Heights are contiguous. The - packed records must consume the archive exactly. +- **Manifest**: either `core-framed-archive-manifest-v1` (top-level + `start_height`/`stop_height`, `archive_sha256`/`archive_bytes`, `blocks`) + or the producer `bitcoin-rs-corpus-manifest` version 1 (`range` + `{start_height,stop_height}`, `archive` `{sha256,size}`, `entries`; only + version 1 is accepted and unknown schemas are refused). Both readers parse + both shapes directly with no converted or duplicate manifest. Both shapes + name network, magic, inclusive height range, archive digest and size, and + one packed entry per height (`height`, `hash`, `offset`, `payload_length`). + Heights are contiguous from `start_height`. The packed records must + consume the archive exactly. The manifest `network` must equal the + configured node network identity (`config.network.identity_name()`, so a + regtest run accepts `regtest`) and its magic must equal that network's + magic; there is no mainnet-only literal. +- **Indexed parity (harness)**: an optional `indexed` block names a + producer-owned `bitcoin-rs-corpus-index-v1` document, a directory of Core + `blkNNNNN.dat` sources, and the freeze file that binds them. At config + load the comparator runs producer `verify_index` once, then fully exhausts + `iter_indexed_framed` and requires the streamed framed bytes to equal the + packed archive digest and size. Magic must match, order and stop hash are + bound through the freeze tip plus the packed manifest tip, and any source + or index mutation refuses before any child starts. - **Pinned corpus**: after config load, the campaign copies archive and manifest to the literal custody paths `/custody/archive` and `/custody/manifest` (where `` is the `/arms` directory `main()` creates inside its `TemporaryDirectory` with `prefix="offline-full-validation-"` under the `--output` parent), - re-hashes both, and chmod's them `0o400`. Every arm reads those pins via - the `{corpus_path}` / `{manifest_path}` placeholders. Before and after - every child, the controller re-hashes both pins and checks device, inode, + re-hashes both, and chmod's them `0o400`. When `indexed` is present it + also copies the index document to `custody/index` and every declared + `blkNNNNN.dat` source into `custody/sources/` at `0o400`. Every arm reads + the pins via the `{corpus_path}` / `{manifest_path}` placeholders: + `_corpus_path_for_arm` binds `{corpus_path}` to `custody/archive`, the + single regular file whose bytes equal the indexed logical stream by the + verified parity above. The index and blk pins are custody-pinned and + re-hashed before and after every child, but they are never passed as + `{corpus_path}`: the custody `sources` directory holds multiple + `blkNNNNN.dat` files while the timed command expects one regular corpus + file. Before and after every child, the controller re-hashes every pin + (archive, manifest, index, each blk source) and checks device, inode, size, and mtime. A same-user child that chmod's and rewrites a pin fails - closed. The two pins persist across all 14 arms and are re-hashed before and - after every child, but they are transient working copies, not retained - artifacts: when the campaign finishes, `TemporaryDirectory` removes the whole - tree and only the result JSON published to `--output` survives. + closed. The pins persist across all 14 arms, but they are transient + working copies, not retained artifacts: when the campaign finishes, + `TemporaryDirectory` removes the whole tree and only the result JSON + published to `--output` survives. Dual custody transiently holds the + corpus twice (packed bytes plus indexed sources); no third copy is + materialized because parity already binds the executed bytes to the + indexed stream. - **Posture**: `assume_valid` must be false. `txindex`, `blockfilterindex`, and `coinstatsindex` must be off. Cache policy is the closed set `process-cold/page-cache-unspecified`. The sibling MuHash comparator @@ -81,15 +112,27 @@ persistence, flush, and shutdown. No stage is subtracted. CPU (`utime+stime`) and peak RSS are sampled from `/proc` and reported separately; they are not an alternate definition of speed. -The child must remain in the foreground and exit. Bitcoin Core commands -must not daemonize (`-daemon=0`) and should stop after import -(`-stopafterblockimport=1`, `-assumevalid=0`, `-stopatheight=H`). bitcoin-rs -commands must apply the same archive through height `H` with -`--assume-valid-height 0`, persist bodies and undo, publish the production -clean checkpoint, and exit nonzero if publication fails. After wait, the -process group must be empty and the comparator, running as a Linux child -subreaper, must own no leftover descendants. A fixture that forks a -sleeper and then exits 0 is refused; no result JSON is published. +The child must remain in the foreground and exit. Bitcoin Core consumes the +`{corpus_path}` file through its checked-in `-loadblock` family: flag +`bitcoin-core/src/init.cpp:529`, collection `2128-2136` (`vImportFiles`, +`ImportBlocks(chainman, vImportFiles)`, then `ActivateBestChains`), fan-out +`bitcoin-core/src/node/blockstorage.cpp:1270` (`ImportBlocks`), and +`bitcoin-core/src/validation.cpp:4975` +(`ChainstateManager::LoadExternalBlockFile` magic scan, size gate, header +hash, `AcceptBlock` without `ProcessNewBlock`). Core commands must not +daemonize (`-daemon=0`) and should stop after import +(`-stopafterblockimport=1`, `-assumevalid=0`, `-stopatheight=H`). +bitcoin-rs consumes the same single regular `{corpus_path}` file through +`crates/node/src/offline_import.rs` (`--corpus`, `--manifest`, `--state` +with `--data-dir` and `--assume-valid-height 0`): Core-framed walk with +manifest binding, full validation, durable bodies plus undo, production +clean checkpoint, and an eight-field certified state. It persists bodies +and undo, publishes the production clean checkpoint, exits nonzero when +publication or any gate fails, never daemonizes or forks, and leaves no +descendants. After wait, the process group must be empty and the +comparator, running as a Linux child subreaper, must own no leftover +descendants. A fixture that forks a sleeper and then exits 0 is refused; no +result JSON is published. ## Correctness gates, in order @@ -118,7 +161,10 @@ digests, posture, lifecycle), a `correctness` block (every gate `true` — the document cannot exist otherwise), every arm's wall/CPU/RSS, exit code, and certified state, per-role percentile summaries, and `candidate_over_core_p50_ratio`. `result_sha256` is the canonical hash of -the document without that field. +the document without that field. The indexed index digest and per-source +digests are bound through `config_sha256` (the indexed block is part of the +hashed config); the custody block keeps the packed archive/manifest shape so +the result schema is unchanged. Raw argv is never published. Public command digests use the same category-only projection as the P2P loopback comparator. @@ -132,16 +178,26 @@ python3 -m unittest test_offline_full_validation # from tools/benchmark-campai ``` Tests use a two-block Core-framed archive and deterministic fixture nodes -that read the archive and write the certified state file. They prove the -harness, not live node performance. A live seven-pair campaign still needs +that read the archive and write the certified state file. Indexed harness +tests add a dual packed/indexed case over the same two linked bytes with a +synthetic harness freeze (not C150/Cmodern): an indexed golden plus source, +truncation, index-tamper, post-load and child-time pin refusals, and a +no-second-packed-copy proof (no `custody/indexed-archive` exists, +`custody/archive` is the only custody file carrying the packed bytes, every +arm executes those bytes, and every custody pin stays `0o400`). They prove +the harness, not live node performance. A live seven-pair campaign still needs hash-pinned `bitcoind` and bitcoin-rs binaries plus an exported archive of a corpus frozen by issue #42 ([`docs/contracts/campaign-corpora.md`](../contracts/campaign-corpora.md)); -archive bytes are not stored in git. +archive bytes are not stored in git. Indexed-only arm execution (no packed +archive for arms) remains unresolved and is not claimed here. ## Limits -Campaign ceilings: archive 1 TiB, manifest 512 MiB, 2 000 000 blocks. +Campaign ceilings: archive 1 TiB, manifest 512 MiB, index 512 MiB, +2 000 000 blocks, at most 10 000 blk sources. The 1 TiB bound covers both +the packed archive and the indexed logical stream via byte-parity size +equality; the index bound covers Cmodern metadata (tens of megabytes). Those bounds admit the Cmodern corpus (genesis .. 709,635); they are not a promise to ingest the live full-tip chain (~760 GiB), which is not a product corpus under issue #42. diff --git a/docs/contracts/campaign-corpora.md b/docs/contracts/campaign-corpora.md index ddde65d24..c6fd59931 100644 --- a/docs/contracts/campaign-corpora.md +++ b/docs/contracts/campaign-corpora.md @@ -50,6 +50,14 @@ product cell. A length-prefixed diagnostic file is not a product corpus. archive the REST exporter writes. - Archive bytes are not stored in git. A cell names the certified archive by the manifest digest produced at export time. +- Indexed representation (harness parity only, not a new product corpus): + schema `bitcoin-rs-corpus-index-v1` locates the same verified logical byte + stream inside Core `blkNNNNN.dat` sources with per-source fingerprints, + exact order, logical digest/size, and stop-hash binding. Producer + `build_index_from_scan`/`IndexWriter` builds it; `verify_index` runs once + before `iter_indexed_framed` is fully exhausted. The offline comparator + harness accepts dual packed/indexed inputs and requires byte parity with + the packed archive. Product identities (C150/Cmodern pins) are unchanged. ### `CORP-03`: Validation posture @@ -127,7 +135,14 @@ that height. No Cmodern cell may close on a guessed or recalled UTXO total. - `tools/campaign-corpus/test_corpus.py` (`python3 tools/campaign-corpus/test_corpus.py`) pins both identities, the eleven specials, C150 census zeros, Cmodern all-positive specials, Core - framing, manifest digest binding, and `assume_valid_height = 0`. + framing, manifest digest binding, and `assume_valid_height = 0`. Its + `IndexedParity` cases prove indexed logical bytes match the packed archive + with same-length, header-stable, order, truncation, and unsafe-name + refusals. +- `tools/benchmark-campaign/test_offline_full_validation.py` + `IndexedParityHarnessTests` prove dual packed/indexed harness custody and + refusals (golden plus source, truncation, index-tamper, and post-load pin + refusals) without claiming a live campaign. - `bin/bitcoin-rs/tests/overhaul_reference_set.rs`: G0 pin; rejects label-only, malformed, and well-formed-but-unbound identities, checks kernel package custody, and pins `corpus_custody()` honesty. diff --git a/tools/benchmark-campaign/offline_full_validation.py b/tools/benchmark-campaign/offline_full_validation.py old mode 100644 new mode 100755 index 2bdd0e55f..3dbdc1fc4 --- a/tools/benchmark-campaign/offline_full_validation.py +++ b/tools/benchmark-campaign/offline_full_validation.py @@ -16,37 +16,99 @@ import argparse import hashlib +import importlib.util import json import os +import re import signal import stat import subprocess import sys import tempfile import time -from collections.abc import Sequence +from collections.abc import Iterator, Sequence from dataclasses import dataclass from pathlib import Path -from typing import NoReturn - -from p2p_loopback import _ChildSubreaperScope, _direct_children_pids +from types import ModuleType +from typing import NoReturn, Protocol, runtime_checkable from evidence import ( ContractError, - _is_object, + JsonObject, _array, + _is_object, + _publish_result, _text, _uint, + _validate_json_depth, canonical_bytes, canonical_sha256, - _validate_json_depth, summarize, - _publish_result, - JsonObject, ) +from p2p_loopback import _ChildSubreaperScope, _direct_children_pids + + +def _campaign_corpus_path() -> Path: + """Single path to the sibling producer-owned corpus module.""" + return Path(__file__).resolve().parent.parent / "campaign-corpus" / "corpus.py" + + +def _load_campaign_corpus_module() -> ModuleType: + """Load the sibling producer module by path once, without any handoff gate. + + Both the manifest-identity aliases and the indexed-reader handoff share + this loader so ``corpus.py`` executes at most once per process. Callers + apply their own validation: the identity reader checks the two constants + while ``_campaign_corpus`` keeps its indexed-handoff refusal. + """ + cached = sys.modules.get("campaign_corpus") + if cached is not None: + return cached + path = _campaign_corpus_path() + spec = importlib.util.spec_from_file_location("campaign_corpus", path) + if spec is None or spec.loader is None: + raise ContractError(f"cannot load campaign corpus module: {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + try: + spec.loader.exec_module(module) + except OSError as error: + del sys.modules[spec.name] + raise ContractError(f"cannot load campaign corpus module: {error}") from error + except Exception as error: + del sys.modules[spec.name] + raise ContractError( + f"campaign corpus module failed to load: {error}" + ) from error + return module + + +def _producer_manifest_identity() -> tuple[str, int]: + """Alias the producer-owned manifest schema tag and version. + + The schema is owned by the sibling ``tools/campaign-corpus`` module; the + comparator only aliases its tag and version so the two can never drift. + Key sets stay local: the comparator validates its read-only projection + with its own field names. This reader checks constants only and never + touches the indexed-reader handoff (``_campaign_corpus``). + """ + module = _load_campaign_corpus_module() + schema = getattr(module, "MANIFEST_SCHEMA", None) + version = getattr(module, "MANIFEST_VERSION", None) + if ( + not isinstance(schema, str) + or not isinstance(version, int) + or isinstance(version, bool) + ): + if sys.modules.get("campaign_corpus") is module: + del sys.modules["campaign_corpus"] + raise ContractError("campaign corpus schema has an unusable manifest identity") + return schema, version + CONFIG_SCHEMA = "offline-full-validation-config-v1" MANIFEST_SCHEMA = "core-framed-archive-manifest-v1" +PRODUCER_MANIFEST_SCHEMA, PRODUCER_MANIFEST_VERSION = _producer_manifest_identity() RESULT_SCHEMA = "offline-full-validation-result-v1" PAIR_COUNT = 7 HEADER_BYTES = 80 @@ -58,6 +120,14 @@ MAX_BINARY_BYTES = 1024 * 1024 * 1024 MAX_COMMAND_ARGS = 256 MAX_BLOCKS = 2_000_000 +# Indexed representation ceilings, rejustified for Cmodern alongside the packed +# ceilings above. The packed archive ceiling (1 TiB) still bounds the indexed +# logical byte stream via byte-parity size equality. The index document is +# metadata only (one small entry per block plus per-source fingerprints): +# Cmodern needs tens of megabytes, well within the 512 MiB manifest-scale +# bound reused here. Source count matches the producer bound. +MAX_INDEX_BYTES = 512 * 1024 * 1024 +MAX_SOURCES = 10000 MAX_ARM_TIMEOUT_NS = 4 * 60 * 60 * 1_000_000_000 MIN_ARM_TIMEOUT_NS = 1_000_000_000 CHILD_TERMINATE_GRACE_NS = 1_000_000_000 @@ -144,11 +214,13 @@ "cache_policy", } ) -_PROGRAM_KEYS = frozenset( - {"binary", "binary_sha256", "command", "reopen_command"} -) +_PROGRAM_KEYS = frozenset({"binary", "binary_sha256", "command", "reopen_command"}) _CORPUS_KEYS = frozenset({"archive", "manifest"}) _FILE_REF_KEYS = frozenset({"path", "sha256", "bytes"}) +_INDEXED_KEYS = frozenset({"index", "source_dir", "products"}) +_INDEXED_SOURCE_KEYS = frozenset({"name", "sha256", "size"}) +_SOURCE_NAME_RE = re.compile(r"blk\d{5}\.dat") +_INDEXED_CONFIG_KEY = "indexed" _MANIFEST_KEYS = frozenset( { "schema", @@ -162,6 +234,24 @@ } ) _BLOCK_ENTRY_KEYS = frozenset({"height", "hash", "offset", "payload_length"}) +_PRODUCER_MANIFEST_KEYS = frozenset( + { + "archive", + "corpus_id", + "entries", + "genesis_hash", + "manifest_sha256", + "network", + "network_magic", + "range", + "schema", + "source_tip_hash", + "version", + } +) +_PRODUCER_ARCHIVE_KEYS = frozenset({"sha256", "size"}) +_PRODUCER_RANGE_KEYS = frozenset({"start_height", "stop_height"}) +_PRODUCER_ENTRY_KEYS = frozenset({"height", "hash", "offset", "payload_length"}) _CONFIG_KEYS = frozenset( { "schema", @@ -194,12 +284,40 @@ class FileIdentity: sha256: str +@dataclass(frozen=True) +class IndexedInput: + """Dual-parity indexed inputs, distinct from the packed archive/manifest. + + The packed corpus stays required for arm execution. When present, the + indexed block adds a producer-owned index document plus a directory of + Core blk files plus the freeze file that binds them to C150/Cmodern (or + to a synthetic harness freeze). Verification runs producer + ``verify_index`` once before exhausting ``iter_indexed_framed`` and + requires byte parity with the packed archive. Custody copies the index + file and every declared blk file with FileRef discipline. + """ + + index: FileRef + source_dir: Path + products: FileRef + + +@dataclass(frozen=True) +class PinnedIndexedSource: + name: str + ref: FileRef + identity: FileIdentity + + @dataclass(frozen=True) class PinnedCorpus: archive: FileRef archive_identity: FileIdentity manifest: FileRef manifest_identity: FileIdentity + index: FileRef | None + index_identity: FileIdentity | None + sources: tuple[PinnedIndexedSource, ...] @dataclass(frozen=True) @@ -260,6 +378,7 @@ class Config: archive_identity: FileIdentity manifest: FileRef manifest_document: Manifest + indexed: IndexedInput | None expected_state: CertifiedState core: Program candidate: Program @@ -386,6 +505,156 @@ def _identity_from_stat(info: os.stat_result, digest: str) -> FileIdentity: ) +@runtime_checkable +class _IndexedFreeze(Protocol): + network_magic: bytes + + +@runtime_checkable +class _CampaignCorpusReader(Protocol): + def load_freeze(self, path: Path) -> _IndexedFreeze: ... + def verify_index( + self, freeze: _IndexedFreeze, index_path: Path, source_dir: Path + ) -> object: ... + def iter_indexed_framed( + self, freeze: _IndexedFreeze, index_path: Path, source_dir: Path + ) -> Iterator[bytes]: ... + + +_CAMPAIGN_CORPUS: _CampaignCorpusReader | None = None + + +def _campaign_corpus() -> _CampaignCorpusReader: + """Load the producer-owned indexed corpus reader without a new dependency. + + The module lives in the sibling ``tools/campaign-corpus`` directory and + is loaded by path so the comparator needs no packaging change. The result + is cached. Callers use only the public handoff (``load_freeze``, + ``verify_index``, ``iter_indexed_framed``); all refusal semantics are + converted to the comparator ``ContractError`` at the call sites. + """ + global _CAMPAIGN_CORPUS + if _CAMPAIGN_CORPUS is not None: + return _CAMPAIGN_CORPUS + module = _load_campaign_corpus_module() + if not isinstance(module, _CampaignCorpusReader): + raise ContractError("campaign corpus reader lacks the indexed handoff") + _CAMPAIGN_CORPUS = module + return module + + +def _parse_indexed(value: object) -> IndexedInput: + root = _object(value, "indexed", _INDEXED_KEYS) + index = _file_ref(root["index"], "indexed.index", MAX_INDEX_BYTES) + source_dir = Path(_text(root["source_dir"], "indexed.source_dir")).absolute() + if source_dir.is_symlink(): + raise ContractError("indexed source_dir must not be a symlink") + if not source_dir.is_dir(): + raise ContractError("indexed source_dir is not a directory") + products = _file_ref(root["products"], "indexed.products", MAX_CONFIG_BYTES) + return IndexedInput(index, source_dir, products) + + +def _check_file_ref_binding(ref: FileRef, field: str) -> None: + """Refuse when a FileRef's live bytes do not match its declared identity.""" + info, observed = _hash_regular(ref.path, field) + if info.st_size != ref.size or observed != ref.sha256: + raise ContractError(f"{field} changed before the campaign pin") + + +def _load_indexed_freeze(products: FileRef) -> _IndexedFreeze: + _check_file_ref_binding(products, "indexed products") + module = _campaign_corpus() + try: + return module.load_freeze(path=products.path) + except (ValueError, OSError) as error: + raise ContractError( + f"indexed products freeze is not usable: {error}" + ) from error + + +def _verify_indexed_parity( + archive: FileRef, + magic: bytes, + indexed: IndexedInput, +) -> None: + """Verify the indexed stream and require byte parity with the packed archive. + + Runs producer ``verify_index`` once before exhausting producer + ``iter_indexed_framed`` fully. The exhausted framed bytes are hashed + streaming and must equal the packed archive digest and size, which + preserves exact bytes, order, stop hash (via the freeze tip plus the + packed manifest tip), and mutation detection. Every producer refusal + becomes a comparator refusal with no result JSON. + """ + _check_file_ref_binding(indexed.index, "indexed index") + freeze = _load_indexed_freeze(indexed.products) + freeze_magic = freeze.network_magic + if not isinstance(freeze_magic, (bytes, bytearray)) or bytes(freeze_magic) != magic: + raise ContractError("indexed freeze network magic does not match the config") + module = _campaign_corpus() + try: + module.verify_index(freeze, indexed.index.path, indexed.source_dir) + except (ValueError, OSError) as error: + raise ContractError(f"indexed verification refused: {error}") from error + hasher = hashlib.sha256() + total = 0 + try: + framed = module.iter_indexed_framed( + freeze, indexed.index.path, indexed.source_dir + ) + for chunk in framed: + if not isinstance(chunk, (bytes, bytearray)): + raise ContractError("indexed stream yielded non-bytes") + hasher.update(chunk) + total += len(chunk) + if total > MAX_ARCHIVE_BYTES: + raise ContractError("indexed logical stream exceeds MAX_ARCHIVE_BYTES") + except (ValueError, OSError) as error: + raise ContractError(f"indexed stream refused: {error}") from error + if total != archive.size: + raise ContractError("indexed logical size does not match the packed archive") + if hasher.hexdigest() != archive.sha256: + raise ContractError("indexed logical digest does not match the packed archive") + + +def _indexed_sources_from_pinned_index( + index_path: Path, +) -> list[tuple[str, str, int]]: + """List declared blk sources from a pinned index copy for custody pinning. + + Full acceptance already ran via producer ``verify_index``; this parsing + only enumerates which files custody must copy. Entries are validated + minimally (basename shape, lowercase digest, positive size, no + duplicates, count bound) and returned in index order. + """ + raw = _load_json(index_path, "indexed index", MAX_INDEX_BYTES) + if not _is_object(raw): + raise ContractError("indexed index must be a JSON object") + sources_raw = raw.get("sources") + if not isinstance(sources_raw, list) or not sources_raw: + raise ContractError("indexed index declares no sources") + if len(sources_raw) > MAX_SOURCES: + raise ContractError("indexed index declares too many sources") + seen: set[str] = set() + declared: list[tuple[str, str, int]] = [] + for position, item in enumerate(sources_raw): + field = f"indexed sources[{position}]" + entry = _object(item, field, _INDEXED_SOURCE_KEYS) + name = _text(entry["name"], f"{field}.name") + if _SOURCE_NAME_RE.fullmatch(name) is None: + raise ContractError(f"{field}.name is not a blkNNNNN.dat basename") + if name in seen: + raise ContractError(f"duplicate indexed source {name!r}") + seen.add(name) + digest = _hash(entry["sha256"], f"{field}.sha256") + size = _uint(entry["size"], f"{field}.size", MAX_ARCHIVE_BYTES) + if size == 0: + raise ContractError(f"{field}.size must be positive") + declared.append((name, digest, size)) + return declared + + def header_hash(payload: bytes) -> str: """Display-hex of double-SHA256 of the 80-byte header. @@ -402,49 +671,97 @@ def header_hash(payload: bytes) -> str: def _parse_manifest(value: object, archive: FileRef, magic: bytes) -> Manifest: - root = _object(value, "manifest", _MANIFEST_KEYS) - if root["schema"] != MANIFEST_SCHEMA: - raise ContractError(f"manifest.schema must be {MANIFEST_SCHEMA}") - network = _text(root["network"], "manifest.network") - documented_magic = _magic(root["network_magic"], "manifest.network_magic") - if documented_magic != magic: - raise ContractError("manifest network magic does not match the config") - start = _uint(root["start_height"], "manifest.start_height") - stop = _uint(root["stop_height"], "manifest.stop_height") - if stop < start: - raise ContractError("manifest stop_height is before start_height") - archive_digest = _hash(root["archive_sha256"], "manifest.archive_sha256") - archive_bytes = _uint(root["archive_bytes"], "manifest.archive_bytes", MAX_ARCHIVE_BYTES) - if archive_digest != archive.sha256 or archive_bytes != archive.size: - raise ContractError("manifest archive identity does not match the corpus FileRef") - raw_blocks = _array(root["blocks"], "manifest.blocks") + if not _is_object(value): + raise ContractError("manifest must be a JSON object") + schema = value.get("schema") + if schema == MANIFEST_SCHEMA: + root = _object(value, "manifest", _MANIFEST_KEYS) + network = _text(root["network"], "manifest.network") + documented_magic = _magic(root["network_magic"], "manifest.network_magic") + if documented_magic != magic: + raise ContractError("manifest network magic does not match the config") + start = _uint(root["start_height"], "manifest.start_height") + stop = _uint(root["stop_height"], "manifest.stop_height") + if stop < start: + raise ContractError("manifest stop_height is before start_height") + archive_digest = _hash(root["archive_sha256"], "manifest.archive_sha256") + archive_bytes = _uint( + root["archive_bytes"], "manifest.archive_bytes", MAX_ARCHIVE_BYTES + ) + if archive_digest != archive.sha256 or archive_bytes != archive.size: + raise ContractError( + "manifest archive identity does not match the corpus FileRef" + ) + raw_blocks = _array(root["blocks"], "manifest.blocks") + blocks_field = "manifest.blocks" + entry_keys = _BLOCK_ENTRY_KEYS + elif schema == PRODUCER_MANIFEST_SCHEMA: + root = _object(value, "manifest", _PRODUCER_MANIFEST_KEYS) + version = _uint(root["version"], "manifest.version") + if version != PRODUCER_MANIFEST_VERSION: + raise ContractError(f"manifest.version must be {PRODUCER_MANIFEST_VERSION}") + network = _text(root["network"], "manifest.network") + documented_magic = _magic(root["network_magic"], "manifest.network_magic") + if documented_magic != magic: + raise ContractError("manifest network magic does not match the config") + range_obj = _object(root["range"], "manifest.range", _PRODUCER_RANGE_KEYS) + start = _uint(range_obj["start_height"], "manifest.range.start_height") + stop = _uint(range_obj["stop_height"], "manifest.range.stop_height") + if stop < start: + raise ContractError("manifest stop_height is before start_height") + archive_obj = _object( + root["archive"], "manifest.archive", _PRODUCER_ARCHIVE_KEYS + ) + archive_digest = _hash(archive_obj["sha256"], "manifest.archive.sha256") + archive_bytes = _uint( + archive_obj["size"], "manifest.archive.size", MAX_ARCHIVE_BYTES + ) + if archive_digest != archive.sha256 or archive_bytes != archive.size: + raise ContractError( + "manifest archive identity does not match the corpus FileRef" + ) + _text(root["corpus_id"], "manifest.corpus_id") + _hash(root["genesis_hash"], "manifest.genesis_hash") + _hash(root["source_tip_hash"], "manifest.source_tip_hash") + _hash(root["manifest_sha256"], "manifest.manifest_sha256") + raw_blocks = _array(root["entries"], "manifest.entries") + blocks_field = "manifest.entries" + entry_keys = _PRODUCER_ENTRY_KEYS + else: + raise ContractError( + f"manifest.schema must be {MANIFEST_SCHEMA} or {PRODUCER_MANIFEST_SCHEMA}" + ) expected = stop - start + 1 if not raw_blocks or len(raw_blocks) != expected or len(raw_blocks) > MAX_BLOCKS: - raise ContractError("manifest.blocks must list every height exactly once") + raise ContractError(f"{blocks_field} must list every height exactly once") blocks: list[BlockEntry] = [] cursor = 0 for index, raw in enumerate(raw_blocks): - item = _object(raw, f"manifest.blocks[{index}]", _BLOCK_ENTRY_KEYS) - height = _uint(item["height"], f"manifest.blocks[{index}].height") + item = _object(raw, f"{blocks_field}[{index}]", entry_keys) + height = _uint(item["height"], f"{blocks_field}[{index}].height") if height != start + index: raise ContractError("manifest heights must be contiguous from start_height") - offset = _uint(item["offset"], f"manifest.blocks[{index}].offset", MAX_ARCHIVE_BYTES) + offset = _uint( + item["offset"], f"{blocks_field}[{index}].offset", MAX_ARCHIVE_BYTES + ) payload_length = _uint( item["payload_length"], - f"manifest.blocks[{index}].payload_length", + f"{blocks_field}[{index}].payload_length", MAX_PAYLOAD_BYTES, ) if payload_length < HEADER_BYTES: - raise ContractError(f"manifest.blocks[{index}] payload is shorter than a header") + raise ContractError( + f"{blocks_field}[{index}] payload is shorter than a header" + ) if offset != cursor: - raise ContractError(f"manifest.blocks[{index}] offset is not packed") + raise ContractError(f"{blocks_field}[{index}] offset is not packed") record_length = 8 + payload_length if offset + record_length > archive.size: - raise ContractError(f"manifest.blocks[{index}] overruns the archive") + raise ContractError(f"{blocks_field}[{index}] overruns the archive") blocks.append( BlockEntry( height, - _hash(item["hash"], f"manifest.blocks[{index}].hash"), + _hash(item["hash"], f"{blocks_field}[{index}].hash"), offset, payload_length, ) @@ -479,16 +796,22 @@ def _verify_archive(archive: FileRef, manifest: Manifest, magic: bytes) -> FileI raise ContractError("archive ended inside a record header") digest.update(header) if header[:4] != magic: - raise ContractError(f"archive record {index} has the wrong network magic") + raise ContractError( + f"archive record {index} has the wrong network magic" + ) payload_length = int.from_bytes(header[4:], "little") if payload_length != entry.payload_length: - raise ContractError(f"archive record {index} length does not match the manifest") + raise ContractError( + f"archive record {index} length does not match the manifest" + ) payload = os.read(descriptor, payload_length) if len(payload) != payload_length: raise ContractError("archive ended inside a block payload") digest.update(payload) if header_hash(payload) != entry.block_hash: - raise ContractError(f"archive record {index} header hash does not match the manifest") + raise ContractError( + f"archive record {index} header hash does not match the manifest" + ) trailing = os.read(descriptor, 1) if trailing: raise ContractError("archive contains trailing bytes after the last record") @@ -535,6 +858,10 @@ def _require_pin(path: Path, expected: FileIdentity, field: str) -> None: def _assert_pins(pinned: PinnedCorpus) -> None: _require_pin(pinned.archive.path, pinned.archive_identity, "archive") _require_pin(pinned.manifest.path, pinned.manifest_identity, "manifest") + if pinned.index is not None and pinned.index_identity is not None: + _require_pin(pinned.index.path, pinned.index_identity, "indexed index") + for source in pinned.sources: + _require_pin(source.ref.path, source.identity, f"indexed source {source.name}") def _certified_state(value: object, field: str) -> CertifiedState: @@ -550,7 +877,9 @@ def _certified_state(value: object, field: str) -> CertifiedState: _boolean(item["disconnect_ready"], f"{field}.disconnect_ready"), ) if not state.bodies_available or not state.disconnect_ready: - raise ContractError(f"{field} must prove body availability and disconnect readiness") + raise ContractError( + f"{field} must prove body availability and disconnect readiness" + ) return state @@ -651,7 +980,20 @@ def _program(raw: object, role: str, reopen_required: bool) -> Program: def parse_config(value: object) -> Config: - root = _object(value, "config", _CONFIG_KEYS) + if not _is_object(value): + raise ContractError("config must be a JSON object") + actual_keys = frozenset(value) + if _INDEXED_CONFIG_KEY in actual_keys: + expected_keys = _CONFIG_KEYS | frozenset({_INDEXED_CONFIG_KEY}) + else: + expected_keys = _CONFIG_KEYS + if actual_keys != expected_keys: + raise ContractError( + "config has wrong keys; " + f"missing_count={len(expected_keys - actual_keys)}, " + f"unknown_count={len(actual_keys - expected_keys)}" + ) + root = value if root["schema"] != CONFIG_SCHEMA: raise ContractError(f"config.schema must be {CONFIG_SCHEMA}") magic = _magic(root["network_magic"], "network_magic") @@ -687,11 +1029,17 @@ def parse_config(value: object) -> Config: finally: os.close(descriptor) archive_identity = _verify_archive(archive, manifest, magic) + indexed: IndexedInput | None = None + if _INDEXED_CONFIG_KEY in root: + indexed = _parse_indexed(root[_INDEXED_CONFIG_KEY]) + _verify_indexed_parity(archive, magic, indexed) expected = _certified_state(root["expected_state"], "expected_state") if expected.height != manifest.stop_height: raise ContractError("expected_state.height must equal the manifest stop height") if expected.bestblock != manifest.blocks[-1].block_hash: - raise ContractError("expected_state.bestblock must equal the final archive block hash") + raise ContractError( + "expected_state.bestblock must equal the final archive block hash" + ) lifecycle_raw = _object(root["lifecycle"], "lifecycle", _LIFECYCLE_KEYS) mode = _text(lifecycle_raw["mode"], "lifecycle.mode") if mode not in {"fresh", "reopen"}: @@ -699,9 +1047,13 @@ def parse_config(value: object) -> Config: reopen_state_raw = lifecycle_raw["expected_reopen_state"] reopen_state: CertifiedState | None = None if mode == "reopen": - reopen_state = _certified_state(reopen_state_raw, "lifecycle.expected_reopen_state") + reopen_state = _certified_state( + reopen_state_raw, "lifecycle.expected_reopen_state" + ) if reopen_state != expected: - raise ContractError("reopen state must equal the timed-trial certified state") + raise ContractError( + "reopen state must equal the timed-trial certified state" + ) elif reopen_state_raw is not None: raise ContractError("fresh lifecycle forbids expected_reopen_state") core = _program(root["core"], "core", mode == "reopen") @@ -714,7 +1066,9 @@ def parse_config(value: object) -> Config: projected = dict(raw_value) projected["command"] = _public_argv(program.command) projected["reopen_command"] = ( - None if program.reopen_command is None else _public_argv(program.reopen_command) + None + if program.reopen_command is None + else _public_argv(program.reopen_command) ) public_root[role] = projected return Config( @@ -725,6 +1079,7 @@ def parse_config(value: object) -> Config: archive_identity, manifest_ref, manifest, + indexed, expected, core, candidate, @@ -893,6 +1248,54 @@ def _pin_corpus(config: Config, root: Path) -> PinnedCorpus: config.manifest.size, "manifest", ) + index_ref: FileRef | None = None + index_identity: FileIdentity | None = None + pinned_sources: list[PinnedIndexedSource] = [] + if config.indexed is not None: + custody_index = custody / "index" + index_identity = _copy_readonly( + config.indexed.index.path, + custody_index, + config.indexed.index.sha256, + config.indexed.index.size, + "indexed index", + ) + index_ref = FileRef( + custody_index, + config.indexed.index.sha256, + config.indexed.index.size, + ) + declared = _indexed_sources_from_pinned_index(custody_index) + sources_dir = custody / "sources" + try: + sources_dir.mkdir() + except OSError as error: + raise ContractError(f"cannot pin indexed sources: {error}") from error + for name, digest, size in declared: + original = config.indexed.source_dir / name + dest = sources_dir / name + identity = _copy_readonly( + original, + dest, + digest, + size, + f"indexed source {name}", + ) + pinned_sources.append( + PinnedIndexedSource(name, FileRef(dest, digest, size), identity) + ) + try: + sources_fd = os.open( + sources_dir, os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC + ) + except OSError as error: + raise ContractError( + f"cannot dirsync indexed sources pin: {error}" + ) from error + try: + os.fsync(sources_fd) + finally: + os.close(sources_fd) try: dirfd = os.open(custody, os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC) except OSError as error: @@ -906,6 +1309,9 @@ def _pin_corpus(config: Config, root: Path) -> PinnedCorpus: archive_identity, FileRef(manifest_path, config.manifest.sha256, config.manifest.size), manifest_identity, + index_ref, + index_identity, + tuple(pinned_sources), ) @@ -955,8 +1361,7 @@ def _wait_child( sample = _sample_child(process.pid) if sample is not None: cpu_ns, rss = sample - if rss > peak_rss: - peak_rss = rss + peak_rss = max(peak_rss, rss) remaining = deadline_ns - time.monotonic_ns() if remaining <= 0: _kill_tree(process) @@ -1081,6 +1486,21 @@ def _run_phase( return code, wall_ns, cpu_ns, peak_rss +def _corpus_path_for_arm(pinned: PinnedCorpus) -> Path: + """Resolve ``{corpus_path}`` to the packed archive pin. + + Indexed parity is verified at config load (``_verify_indexed_parity`` + exhausts the producer stream and requires its digest and size to equal + the packed archive), so the packed pin already carries the indexed + logical bytes. Arms therefore execute the packed pin directly: packed + and indexed inputs stay distinct, the index and blk pins are + custody-held and re-hashed but never executed, and no indexed-derived + second packed file is materialized. + """ + + return pinned.archive.path + + def _run_arm( config: Config, program: Program, @@ -1101,7 +1521,7 @@ def _run_arm( program.command, binary_path=binary_path, data_dir=data_dir, - corpus_path=pinned.archive.path, + corpus_path=_corpus_path_for_arm(pinned), manifest_path=pinned.manifest.path, state_path=state_path, ) @@ -1126,7 +1546,7 @@ def _run_arm( program.reopen_command, binary_path=binary_path, data_dir=data_dir, - corpus_path=pinned.archive.path, + corpus_path=_corpus_path_for_arm(pinned), manifest_path=pinned.manifest.path, state_path=reopen_path, ) @@ -1199,12 +1619,16 @@ def _require_comparable(config: Config, arms: Sequence[ArmObservation]) -> None: expected_first = "core" if pair_index % 2 == 0 else "candidate" expected_second = "candidate" if expected_first == "core" else "core" if first.role != expected_first or second.role != expected_second: - raise ContractError("pair order must alternate Core-first then candidate-first") + raise ContractError( + "pair order must alternate Core-first then candidate-first" + ) if first.pair_index != pair_index or second.pair_index != pair_index: raise ContractError("pair index is inconsistent") for arm in (first, second): if not arm.durability_ok or not arm.state_ok or arm.error is not None: - raise ContractError(f"{arm.role} arm {arm.order_index} failed its gates") + raise ContractError( + f"{arm.role} arm {arm.order_index} failed its gates" + ) if arm.binary_sha256 != ( config.core.binary_sha256 if arm.role == "core" @@ -1229,15 +1653,11 @@ def run_campaign(config: Config, output_root: Path) -> JsonObject: first = config.core if pair_index % 2 == 0 else config.candidate second = config.candidate if first is config.core else config.core arms.append( - _run_arm( - config, first, pair_index, order_index, output_root, pinned - ) + _run_arm(config, first, pair_index, order_index, output_root, pinned) ) order_index += 1 arms.append( - _run_arm( - config, second, pair_index, order_index, output_root, pinned - ) + _run_arm(config, second, pair_index, order_index, output_root, pinned) ) order_index += 1 _require_comparable(config, arms) diff --git a/tools/benchmark-campaign/test_offline_full_validation.py b/tools/benchmark-campaign/test_offline_full_validation.py old mode 100644 new mode 100755 index 7aede3e15..ef1e8914a --- a/tools/benchmark-campaign/test_offline_full_validation.py +++ b/tools/benchmark-campaign/test_offline_full_validation.py @@ -8,6 +8,7 @@ import json import shutil import stat +import struct import sys import tempfile import unittest @@ -15,7 +16,9 @@ from typing import TypeIs sys.path.insert(0, str(Path(__file__).parent)) +sys.path.insert(0, str(Path(__file__).parent.parent / "campaign-corpus")) +import corpus as campaign_corpus # ty: ignore[unresolved-import] # pyright: ignore[reportMissingImports] import offline_full_validation as offline from evidence import ( ContractError, @@ -136,7 +139,9 @@ def _command(*, reopen: bool = False) -> list[str]: return command -def _build_workspace(root: Path, *, reopen: bool = False) -> tuple[Path, CertifiedState]: +def _build_workspace( + root: Path, *, reopen: bool = False +) -> tuple[Path, CertifiedState]: payloads = (_payload(1), _payload(2)) records = [_record(payload) for payload in payloads] archive_bytes = b"".join(records) @@ -217,6 +222,177 @@ def program() -> JsonObject: return config_path, expected +HARNESS_CORPUS_ID = "HARNESS" + + +def _linked_payloads() -> tuple[bytes, bytes]: + """Two linked 81-byte payloads for dual packed/indexed parity. + + Headers wire prev linkage so producer ``verify_index`` passes, while the + trailing suffix byte keeps the packed two-block shape. Harness-only and + not a C150/Cmodern identity. + """ + header_one = ( + struct.pack(" Path: + document = { + "schema": "bitcoin-rs-campaign-corpora-v1", + "network": "mainnet", + "network_magic": MAGIC, + "genesis_hash": genesis, + "validation": {"assume_valid_height": 0}, + "census_specials": ["taproot_key_path_spends"], + "core_oracle": { + "hash_type": "muhash", + "implementation": "Bitcoin Core 31.1", + "use_index": True, + }, + "products": { + HARNESS_CORPUS_ID: { + "stop_height": 1, + "stop_hash": stop, + "block_count": 2, + "census": {"specials": "none"}, + "state": { + "txouts": 1, + "total_amount_sat": 50, + "muhash": "ab" * 32, + "bestblock": stop, + "oracle": "core_gettxoutsetinfo_muhash_at_stop", + }, + } + }, + } + path = root / "harness-products.json" + path.write_bytes(json.dumps(document, indent=2).encode("utf-8")) + return path + + +def _build_dual_workspace( + root: Path, *, reopen: bool = False +) -> tuple[Path, CertifiedState]: + """Packed archive plus indexed blk stream over the same linked bytes. + + Proves harness custody and refusal mechanics only, not live C150/Cmodern + performance and not a live campaign. + """ + payloads = _linked_payloads() + records = [_record(payload) for payload in payloads] + archive_bytes = b"".join(records) + archive_path = root / "blocks.dat" + archive_path.write_bytes(archive_bytes) + hashes = [header_hash(payload) for payload in payloads] + offset = 0 + entries: list[JsonObject] = [] + for height, (payload, digest, record) in enumerate( + zip(payloads, hashes, records, strict=True) + ): + entries.append( + { + "height": height, + "hash": digest, + "offset": offset, + "payload_length": len(payload), + } + ) + offset += len(record) + manifest = { + "schema": MANIFEST_SCHEMA, + "network": "mainnet", + "network_magic": MAGIC, + "start_height": 0, + "stop_height": 1, + "archive_sha256": hashlib.sha256(archive_bytes).hexdigest(), + "archive_bytes": len(archive_bytes), + "blocks": entries, + } + manifest_path = root / "manifest.json" + manifest_path.write_bytes(canonical_bytes(manifest) + b"\n") + expected = _state(height=1, bestblock=hashes[-1]) + node = _write_executable(root / "node.py", _node_source(expected, reopen=reopen)) + digest = _sha256_file(node) + + def program() -> JsonObject: + return { + "binary": str(node), + "binary_sha256": digest, + "command": _command(), + "reopen_command": _command(reopen=True) if reopen else None, + } + + products_path = _write_harness_products(root, hashes[0], hashes[-1]) + freeze = campaign_corpus.load_freeze(path=products_path) + source_dir = root / "blks" + source_dir.mkdir() + (source_dir / "blk00000.dat").write_bytes(records[0]) + (source_dir / "blk00001.dat").write_bytes(records[1]) + index_path = root / "harness.index" + campaign_corpus.build_index_from_scan( + freeze, HARNESS_CORPUS_ID, source_dir, index_path + ) + config = { + "schema": CONFIG_SCHEMA, + "network_magic": MAGIC, + "arm_timeout_ns": 10_000_000_000, + "posture": { + "assume_valid": False, + "txindex": False, + "blockfilterindex": False, + "coinstatsindex": False, + "cache_policy": "process-cold/page-cache-unspecified", + }, + "corpus": { + "archive": { + "path": str(archive_path), + "sha256": hashlib.sha256(archive_bytes).hexdigest(), + "bytes": len(archive_bytes), + }, + "manifest": { + "path": str(manifest_path), + "sha256": _sha256_file(manifest_path), + "bytes": manifest_path.stat().st_size, + }, + }, + "indexed": { + "index": { + "path": str(index_path), + "sha256": _sha256_file(index_path), + "bytes": index_path.stat().st_size, + }, + "source_dir": str(source_dir), + "products": { + "path": str(products_path), + "sha256": _sha256_file(products_path), + "bytes": products_path.stat().st_size, + }, + }, + "expected_state": state_json(expected), + "core": program(), + "candidate": program(), + "lifecycle": { + "mode": "reopen" if reopen else "fresh", + "expected_reopen_state": state_json(expected) if reopen else None, + }, + } + config_path = root / "config.json" + config_path.write_bytes(canonical_bytes(config) + b"\n") + return config_path, expected + + class ArchiveContractTests(unittest.TestCase): def test_header_hash_is_reversed_double_sha256(self) -> None: payload = _payload(9) @@ -397,7 +573,9 @@ def test_seven_pairs_publish_a_gated_result(self) -> None: self.addCleanup(lambda: shutil.rmtree(root, ignore_errors=True)) config_path, expected = _build_workspace(root) output = root / OUTPUT_NAME - self.assertEqual(main(["--config", str(config_path), "--output", str(output)]), 0) + self.assertEqual( + main(["--config", str(config_path), "--output", str(output)]), 0 + ) result = json.loads(output.read_bytes()) self.assertEqual(result["schema"], RESULT_SCHEMA) self.assertEqual(result["pair_count"], PAIR_COUNT) @@ -407,7 +585,7 @@ def test_seven_pairs_publish_a_gated_result(self) -> None: arms = result["arms"] assert isinstance(arms, list) self.assertEqual(len(arms), 14) - roles = [ _object(arm)["role"] for arm in arms ] + roles = [_object(arm)["role"] for arm in arms] self.assertEqual( roles, ["core", "candidate", "candidate", "core"] * 3 + ["core", "candidate"], @@ -415,7 +593,10 @@ def test_seven_pairs_publish_a_gated_result(self) -> None: for arm in arms: record = _object(arm) self.assertEqual(record["exit_code"], 0) - self.assertGreater(record["wall_ns"], 0) + wall_ns = record["wall_ns"] + if not isinstance(wall_ns, int): + self.fail(f"wall_ns must be an integer, got {type(wall_ns).__name__}") + self.assertGreater(wall_ns, 0) self.assertEqual(record["final_state"], state_json(expected)) body = {key: value for key, value in result.items() if key != "result_sha256"} self.assertEqual(result["result_sha256"], canonical_sha256(body)) @@ -427,7 +608,9 @@ def test_reopen_lifecycle_is_untimed_and_gated(self) -> None: self.addCleanup(lambda: shutil.rmtree(root, ignore_errors=True)) config_path, expected = _build_workspace(root, reopen=True) output = root / OUTPUT_NAME - self.assertEqual(main(["--config", str(config_path), "--output", str(output)]), 0) + self.assertEqual( + main(["--config", str(config_path), "--output", str(output)]), 0 + ) result = json.loads(output.read_bytes()) for arm in result["arms"]: record = _object(arm) @@ -458,7 +641,9 @@ def test_nonzero_exit_refuses_publication(self) -> None: config_path, expected = _build_workspace(root) failing = _write_executable( root / "fail.py", - _node_source(expected).replace("raise SystemExit(0)", "raise SystemExit(9)"), + _node_source(expected).replace( + "raise SystemExit(0)", "raise SystemExit(9)" + ), ) config = json.loads(config_path.read_text()) config["core"]["binary"] = str(failing) @@ -545,5 +730,270 @@ def test_summarize_uses_nearest_rank(self) -> None: self.assertEqual(summary["max_ns"], 70) +class IndexedParityHarnessTests(unittest.TestCase): + """Dual packed/indexed parity over a synthetic two-block stream. + + Proves harness custody and refusal mechanics only, not live C150/Cmodern + performance and not a live campaign. + """ + + def test_indexed_parity_publishes_a_gated_result(self) -> None: + root = Path(tempfile.mkdtemp(prefix="offline-indexed-")) + self.addCleanup(lambda: shutil.rmtree(root, ignore_errors=True)) + config_path, expected = _build_dual_workspace(root) + output = root / OUTPUT_NAME + self.assertEqual( + main(["--config", str(config_path), "--output", str(output)]), 0 + ) + result = json.loads(output.read_bytes()) + self.assertEqual(result["schema"], RESULT_SCHEMA) + self.assertEqual(result["pair_count"], PAIR_COUNT) + self.assertEqual(result["arm_count"], 14) + correctness = _object(result["correctness"]) + self.assertTrue(all(correctness.values())) + for arm in result["arms"]: + record = _object(arm) + self.assertEqual(record["exit_code"], 0) + self.assertEqual(record["final_state"], state_json(expected)) + + def test_indexed_campaign_materializes_no_second_packed_copy(self) -> None: + root = Path(tempfile.mkdtemp(prefix="offline-indexed-nodup-")) + self.addCleanup(lambda: shutil.rmtree(root, ignore_errors=True)) + config_path, _ = _build_dual_workspace(root) + arms_root = root / "arms" + result = run_campaign(load_config(config_path), arms_root) + self.assertEqual(result["arm_count"], 14) + custody = arms_root / "custody" + # No indexed-derived second packed file is materialized. + self.assertFalse((custody / "indexed-archive").exists()) + archive_bytes = (root / "blocks.dat").read_bytes() + archive_digest = hashlib.sha256(archive_bytes).hexdigest() + matches: list[str] = [] + for path in sorted(custody.rglob("*")): + if path.is_symlink() or not path.is_file(): + continue + if path.stat().st_size != len(archive_bytes): + continue + if hashlib.sha256(path.read_bytes()).hexdigest() == archive_digest: + matches.append(path.name) + self.assertEqual(matches, ["archive"]) + # Every arm executed those exact packed bytes through custody/archive. + arm_dirs = sorted( + child + for child in arms_root.iterdir() + if child.is_dir() and child.name != "custody" + ) + self.assertEqual(len(arm_dirs), 14) + for arm_dir in arm_dirs: + imported = arm_dir / "data" / "imported" + self.assertTrue(imported.is_file()) + self.assertEqual( + hashlib.sha256(imported.read_bytes()).hexdigest(), archive_digest + ) + # Indexed inputs stay distinct, custody-pinned, and read-only. + sources = sorted((custody / "sources").iterdir()) + self.assertEqual( + [path.name for path in sources], ["blk00000.dat", "blk00001.dat"] + ) + for path in [custody / "archive", custody / "index", *sources]: + self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o400) + + def test_indexed_source_mutation_is_refused(self) -> None: + root = Path(tempfile.mkdtemp(prefix="offline-indexed-mut-")) + self.addCleanup(lambda: shutil.rmtree(root, ignore_errors=True)) + config_path, _ = _build_dual_workspace(root) + victim = root / "blks" / "blk00001.dat" + raw = bytearray(victim.read_bytes()) + raw[10] ^= 0x01 + victim.write_bytes(bytes(raw)) + with self.assertRaises(ContractError): + load_config(config_path) + + def test_indexed_truncation_is_refused(self) -> None: + root = Path(tempfile.mkdtemp(prefix="offline-indexed-trunc-")) + self.addCleanup(lambda: shutil.rmtree(root, ignore_errors=True)) + config_path, _ = _build_dual_workspace(root) + victim = root / "blks" / "blk00001.dat" + data = victim.read_bytes() + victim.write_bytes(data[:-1]) + with self.assertRaises(ContractError): + load_config(config_path) + + def test_indexed_index_tamper_is_refused(self) -> None: + root = Path(tempfile.mkdtemp(prefix="offline-indexed-idx-")) + self.addCleanup(lambda: shutil.rmtree(root, ignore_errors=True)) + config_path, _ = _build_dual_workspace(root) + index_path = root / "harness.index" + document = json.loads(index_path.read_text(encoding="utf-8")) + entries = document["entries"] + assert isinstance(entries, list) + second = entries[1] + assert isinstance(second, dict) + second["hash"] = HASH64 + index_path.write_bytes(canonical_bytes(document) + b"\n") + config = json.loads(config_path.read_text(encoding="utf-8")) + config["indexed"]["index"]["sha256"] = _sha256_file(index_path) + config["indexed"]["index"]["bytes"] = index_path.stat().st_size + config_path.write_bytes(canonical_bytes(config) + b"\n") + with self.assertRaises(ContractError): + load_config(config_path) + + def test_indexed_source_mutation_after_load_is_refused(self) -> None: + root = Path(tempfile.mkdtemp(prefix="offline-indexed-pin-")) + self.addCleanup(lambda: shutil.rmtree(root, ignore_errors=True)) + config_path, _ = _build_dual_workspace(root) + config = load_config(config_path) + victim = root / "blks" / "blk00000.dat" + raw = bytearray(victim.read_bytes()) + raw[20] ^= 0xFF + victim.write_bytes(bytes(raw)) + with self.assertRaises(ContractError): + run_campaign(config, root / "arms") + + def test_child_custody_source_mutation_is_refused(self) -> None: + root = Path(tempfile.mkdtemp(prefix="offline-indexed-srcmut-")) + self.addCleanup(lambda: shutil.rmtree(root, ignore_errors=True)) + config_path, expected = _build_dual_workspace(root) + mutator = _write_executable( + root / "mutate_src.py", + _node_source(expected).replace( + "raise SystemExit(0)\n", + "target = __import__('pathlib').Path(args.corpus).parent / 'sources' / 'blk00000.dat'\n" + "__import__('os').chmod(target, 0o600)\n" + "target.write_bytes(target.read_bytes()[::-1])\n" + "raise SystemExit(0)\n", + ), + ) + config = json.loads(config_path.read_text()) + config["candidate"]["binary"] = str(mutator) + config["candidate"]["binary_sha256"] = _sha256_file(mutator) + config_path.write_bytes(canonical_bytes(config) + b"\n") + output = root / OUTPUT_NAME + with self.assertRaises(ContractError): + main(["--config", str(config_path), "--output", str(output)]) + self.assertFalse(output.exists()) + + +class ProducerManifestTests(unittest.TestCase): + """Producer-shaped manifests map to the core-framed representation. + + The producer schema is read-only here: no converter file is written and + every FileRef, magic, contiguity, packing, and exact-consumption check + still applies. + """ + + def _archive_fixture(self, root: Path) -> tuple[bytes, Path, list[str]]: + payloads = (_payload(1), _payload(2)) + records = [_record(payload) for payload in payloads] + archive_bytes = b"".join(records) + archive_path = root / "blocks.dat" + archive_path.write_bytes(archive_bytes) + hashes = [header_hash(payload) for payload in payloads] + return archive_bytes, archive_path, hashes + + def _core_doc(self, archive_bytes: bytes, hashes: list[str]) -> JsonObject: + offset = 0 + entries: list[JsonObject] = [] + for height, digest in enumerate(hashes): + entries.append( + { + "height": height, + "hash": digest, + "offset": offset, + "payload_length": 81, + } + ) + offset += 8 + 81 + return { + "schema": MANIFEST_SCHEMA, + "network": "mainnet", + "network_magic": MAGIC, + "start_height": 0, + "stop_height": 1, + "archive_sha256": hashlib.sha256(archive_bytes).hexdigest(), + "archive_bytes": len(archive_bytes), + "blocks": entries, + } + + def _producer_doc(self, archive_bytes: bytes, hashes: list[str]) -> JsonObject: + offset = 0 + entries: list[JsonObject] = [] + for height, digest in enumerate(hashes): + entries.append( + { + "height": height, + "hash": digest, + "offset": offset, + "payload_length": 81, + } + ) + offset += 8 + 81 + return { + "schema": "bitcoin-rs-corpus-manifest", + "version": 1, + "corpus_id": "C150", + "network": "mainnet", + "network_magic": MAGIC, + "genesis_hash": hashes[0], + "range": {"start_height": 0, "stop_height": 1}, + "source_tip_hash": hashes[-1], + "archive": { + "size": len(archive_bytes), + "sha256": hashlib.sha256(archive_bytes).hexdigest(), + }, + "entries": entries, + "manifest_sha256": HASH64, + } + + def test_producer_manifest_matches_core_manifest(self) -> None: + root = Path(tempfile.mkdtemp(prefix="offline-producer-")) + self.addCleanup(lambda: shutil.rmtree(root, ignore_errors=True)) + archive_bytes, archive_path, hashes = self._archive_fixture(root) + ref = offline.FileRef( + archive_path, + hashlib.sha256(archive_bytes).hexdigest(), + len(archive_bytes), + ) + magic = bytes.fromhex(MAGIC) + core_parsed = offline._parse_manifest( + self._core_doc(archive_bytes, hashes), ref, magic + ) + producer_parsed = offline._parse_manifest( + self._producer_doc(archive_bytes, hashes), ref, magic + ) + self.assertEqual(producer_parsed, core_parsed) + offline._verify_archive(ref, producer_parsed, magic) + + def test_producer_version_mismatch_is_refused(self) -> None: + root = Path(tempfile.mkdtemp(prefix="offline-producer-ver-")) + self.addCleanup(lambda: shutil.rmtree(root, ignore_errors=True)) + archive_bytes, archive_path, hashes = self._archive_fixture(root) + ref = offline.FileRef( + archive_path, + hashlib.sha256(archive_bytes).hexdigest(), + len(archive_bytes), + ) + document = self._producer_doc(archive_bytes, hashes) + assert isinstance(document, dict) + document["version"] = 2 + with self.assertRaises(ContractError): + offline._parse_manifest(document, ref, bytes.fromhex(MAGIC)) + + def test_unknown_schema_is_refused(self) -> None: + root = Path(tempfile.mkdtemp(prefix="offline-producer-schema-")) + self.addCleanup(lambda: shutil.rmtree(root, ignore_errors=True)) + archive_bytes, archive_path, hashes = self._archive_fixture(root) + ref = offline.FileRef( + archive_path, + hashlib.sha256(archive_bytes).hexdigest(), + len(archive_bytes), + ) + document = self._core_doc(archive_bytes, hashes) + assert isinstance(document, dict) + document["schema"] = "unknown-manifest-v9" + with self.assertRaises(ContractError): + offline._parse_manifest(document, ref, bytes.fromhex(MAGIC)) + + if __name__ == "__main__": unittest.main() diff --git a/tools/campaign-corpus/corpus.py b/tools/campaign-corpus/corpus.py index 6d081d814..8e6adc3d8 100755 --- a/tools/campaign-corpus/corpus.py +++ b/tools/campaign-corpus/corpus.py @@ -10,10 +10,14 @@ from __future__ import annotations import argparse +import contextlib +import errno import hashlib import http.client import json import os +import re +import stat import struct import sys import uuid @@ -59,6 +63,30 @@ _MAX_INT_DIGITS = len(str(_MAX_ARCHIVE_SIZE)) _HEX = frozenset("0123456789abcdefABCDEF") +INDEX_SCHEMA = "bitcoin-rs-corpus-index-v1" +INDEX_VERSION = 1 +_MAX_SOURCES = 10000 +_SOURCE_NAME_RE = re.compile(r"blk\d{5}\.dat") +_INDEX_ROOT_KEYS = frozenset( + { + "corpus_id", + "entries", + "genesis_hash", + "index_sha256", + "logical", + "network", + "network_magic", + "range", + "schema", + "source_tip_hash", + "sources", + "version", + } +) +_INDEX_SOURCE_KEYS = frozenset({"name", "sha256", "size"}) +_INDEX_ENTRY_KEYS = frozenset({"hash", "height", "length", "offset", "source"}) +_INDEX_LOGICAL_KEYS = frozenset({"sha256", "size"}) + Fetch = Callable[[str], bytes] @@ -117,6 +145,31 @@ class CorpusSummary: manifest_file_sha256: str +@dataclass(frozen=True) +class IndexSource: + name: str + size: int + sha256: str + + +@dataclass(frozen=True) +class IndexEntry: + height: int + hash: str + source: int + offset: int + length: int + + +@dataclass(frozen=True) +class IndexSummary: + count: int + logical_size: int + logical_sha256: str + index_sha256: str + index_file_sha256: str + + def load_freeze(path: Path = PRODUCTS_PATH) -> Freeze: raw = _load_json(path) if raw.get("schema") != "bitcoin-rs-campaign-corpora-v1": @@ -191,18 +244,52 @@ def _read_exact(source: BinaryIO, count: int, what: str) -> bytes: return b"".join(chunks) +def _require_zero_tail(source: BinaryIO, offset: int) -> None: + """Consume zero-padding past `offset`; refuse when non-zero bytes follow. + + Core preallocates blk files with zeros. Padding is legal only as an + all-zero tail: any non-zero byte after the first all-zero header means + the file was over-trimmed or spliced, never a clean EOF. + """ + while True: + chunk = source.read(1 << 20) + if not chunk: + return + if chunk.strip(b"\x00"): + raise ContractError( + f"Core blk tail has non-zero bytes after zero padding at offset" + f" {offset}: trim the file to the last frame end" + ) + + def iter_frames( source: BinaryIO, magic: bytes = MAINNET_MAGIC ) -> Iterator[tuple[FrameMeta, bytes]]: - """Stream Core-framed payloads. Clean EOF is legal only between frames.""" + """Stream Core-framed payloads. Clean EOF is legal only between frames. + + A trailing all-zero region is Core blk preallocation padding, not a + frame: an 8-byte all-zero header followed only by zero bytes ends the + stream. Raw blk files are never XOR-obfuscated; a magic mismatch at + offset 0 fails closed instead of attempting de-obfuscation. + """ offset = 0 while True: head = source.read(1) if not head: return header = head + _read_exact(source, HEADER_LEN - 1, "Core frame header") + if header == b"\x00" * HEADER_LEN: + _require_zero_tail(source, offset) + return if header[:4] != magic: - raise ContractError("Core frame magic mismatch") + if offset == 0: + raise ContractError( + "Core frame magic mismatch at offset 0: not a raw" + " blkNNNNN.dat record (raw blk files are never" + " XOR-obfuscated; do not point at an obfuscated store" + " or LevelDB directory)" + ) + raise ContractError(f"Core frame magic mismatch at offset {offset}") length = struct.unpack_from(" MAX_PAYLOAD: raise ContractError("Core frame payload exceeds the consensus maximum") @@ -1013,6 +1100,945 @@ def core_oracle_params(freeze: Freeze, corpus_id: str) -> list[object]: return [oracle["hash_type"], chosen.stop_height, oracle["use_index"]] +_MAX_INDEX_BYTES = _MAX_BLOCK_COUNT * 256 + (1 << 20) + + +@dataclass(frozen=True) +class SourceOffset: + source: int + offset: int + + +@dataclass(frozen=True) +class IndexDocument: + corpus_id: str + network: str + network_magic: str + genesis_hash: str + start_height: int + stop_height: int + source_tip_hash: str + sources: tuple[IndexSource, ...] + entries: tuple[IndexEntry, ...] + logical_size: int + logical_sha256: str + index_sha256: str + + +@dataclass(frozen=True) +class _ScanRecord: + prev: str + source: int + offset: int + length: int + + +def _check_source_name(name: object) -> str: + """Validate an indexed source file name. + + Only Core block-file basenames (``blkNNNNN.dat``) are accepted. This + rejects absolute paths, directory traversal, separators, and drive + prefixes by construction. Consistency: callers hold snapshot isolation + over the source directory (quiescent blk files, no concurrent writer); + verification is idempotent and safe to retry. + """ + if not isinstance(name, str) or not name: + raise ContractError("index source name must be a non-empty string") + if len(name) > 32: + raise ContractError("index source name exceeds its length bound") + if _SOURCE_NAME_RE.fullmatch(name) is None: + raise ContractError( + f"index source name {name!r} is not a blkNNNNN.dat basename" + ) + return name + + +def _lower_hex(value: object, label: str, digits: int) -> str: + text = _text(value, label) + if len(text) != digits: + raise ContractError(f"{label} is not a {digits}-character hex value") + if text != text.lower(): + raise ContractError(f"{label} must be lowercase hex") + if any(char not in "0123456789abcdef" for char in text): + raise ContractError(f"{label} is not a hex value") + return text + + +def _open_source_nofollow(path: Path) -> BinaryIO: + """Open an indexed source without following symlinks. + + `_source_path` validation and `open()` are otherwise a TOCTOU pair: a + symlink swapped in between would be followed and hashed. `O_NOFOLLOW` + makes the open itself authoritative: a symlink fails with `ELOOP` + instead of being traversed, and `fstat` pins regular-file identity + before the descriptor is wrapped. + """ + try: + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + except OSError as err: + if err.errno == errno.ELOOP: + raise ContractError(f"source {path.name!r} must not be a symlink") from err + raise ContractError(f"cannot open source file {path}: {err}") from err + try: + mode = os.fstat(fd).st_mode + except OSError as err: + os.close(fd) + raise ContractError(f"cannot stat source file {path}: {err}") from err + if not stat.S_ISREG(mode): + os.close(fd) + raise ContractError(f"source {path.name!r} is not a regular file") + return os.fdopen(fd, "rb") + + +def _hash_file(path: Path) -> tuple[int, str]: + """Stream a source file, returning its size and SHA-256 hex digest. + + The open never follows symlinks (see `_open_source_nofollow`), so a + symlink swapped in after `_source_path` validation fails closed here + instead of hashing attacker-chosen bytes. + """ + hasher = hashlib.sha256() + size = 0 + handle = _open_source_nofollow(path) + try: + with handle: + while True: + chunk = handle.read(1 << 20) + if not chunk: + break + hasher.update(chunk) + size += len(chunk) + except OSError as err: + raise ContractError(f"cannot read source file {path}: {err}") from err + return size, hasher.hexdigest() + + +def _source_path(source_dir: Path, name: str) -> Path: + if not source_dir.is_dir(): + raise ContractError(f"source directory {source_dir} is not a directory") + path = source_dir / name + if path.is_symlink(): + raise ContractError(f"source {name!r} must not be a symlink") + if not path.is_file(): + raise ContractError(f"source {name!r} is not a regular file") + return path + + +class _SingleSourceReader: + """Bounded indexed-source reader holding at most one open file. + + The handle stays open while consecutive entries share a source and closes + on source switch or ``close()``. All indexed paths (build, verify, iterate) + read through this helper, so frame policy cannot diverge. Callers must call + ``close()``; the reader never opens more than one file. + """ + + def __init__( + self, source_dir: Path, sources: Sequence[IndexSource], magic: bytes + ) -> None: + self._source_dir = source_dir + self._sources = sources + self._magic = magic + self._current: int | None = None + self._handle: BinaryIO | None = None + self._stack = contextlib.ExitStack() + + def read(self, entry: IndexEntry) -> bytes: + if entry.source < 0 or entry.source >= len(self._sources): + raise ContractError( + f"entries[{entry.height}].source is outside the sources array" + ) + if entry.offset < 0 or entry.length < MIN_HEADER or entry.length > MAX_PAYLOAD: + raise ContractError( + f"entries[{entry.height}] has an out-of-range offset/length" + ) + recorded = self._sources[entry.source] + if entry.offset + HEADER_LEN + entry.length > recorded.size: + raise ContractError( + f"entries[{entry.height}] range exceeds its source size" + ) + if self._current != entry.source: + self.close() + path = _source_path(self._source_dir, recorded.name) + self._handle = self._stack.enter_context(_open_source_nofollow(path)) + self._current = entry.source + handle = self._handle + if handle is None: + raise ContractError("indexed source reader lost its handle") + try: + handle.seek(entry.offset) + except OSError as err: + raise ContractError( + f"cannot seek entries[{entry.height}] in {recorded.name}: {err}" + ) from err + header = _read_exact(handle, HEADER_LEN, "indexed Core frame header") + if header[:4] != self._magic: + raise ContractError(f"entries[{entry.height}] magic does not match mainnet") + if struct.unpack_from(" None: + try: + self._stack.close() + except OSError as err: + print(f"warning: cannot close source handle: {err}", file=sys.stderr) + self._handle = None + self._current = None + + +def _verify_entry_hash( + freeze: Freeze, + chosen: Product, + entry: IndexEntry, + height: int, + payload: bytes, + last_hash: str | None, +) -> str: + """Check one payload against its entry, genesis, linkage, and frozen tip.""" + block_hash = header_block_hash(payload) + if block_hash != entry.hash: + raise ContractError(f"entries[{height}] hash does not match the payload header") + if height == 0: + if block_hash != freeze.genesis_hash: + raise ContractError("indexed genesis is not mainnet genesis") + else: + if last_hash is None: + raise ContractError("indexed chain lost its linkage cursor") + if not _prev_field_matches(payload, last_hash): + raise ContractError("indexed chain fails previous-header linkage") + if height == chosen.stop_height and block_hash != chosen.stop_hash: + raise ContractError( + f"{chosen.corpus_id} indexed tip is not the frozen stop hash" + ) + return block_hash + + +def _index_entry_chunk(entry: IndexEntry) -> bytes: + return canonical_json( + { + "hash": entry.hash, + "height": entry.height, + "length": entry.length, + "offset": entry.offset, + "source": entry.source, + } + ) + + +def _index_source_chunk(source: IndexSource) -> bytes: + return canonical_json( + {"name": source.name, "sha256": source.sha256, "size": source.size} + ) + + +def _index_metadata( + freeze: Freeze, + corpus_id: str, + sources: Sequence[IndexSource], + logical_size: int, + logical_sha: str, + index_sha: str, +) -> dict[str, object]: + chosen = product(freeze, corpus_id) + return { + "corpus_id": chosen.corpus_id, + "genesis_hash": freeze.genesis_hash, + "logical": {"sha256": logical_sha, "size": logical_size}, + "network": freeze.network, + "network_magic": freeze.network_magic.hex(), + "range": {"start_height": 0, "stop_height": chosen.stop_height}, + "schema": INDEX_SCHEMA, + "source_tip_hash": chosen.stop_hash, + "version": INDEX_VERSION, + } + + +def _index_chunks( + metadata: Mapping[str, object], + source_chunks: Iterable[bytes], + entry_chunks: Iterable[bytes], + index_sha: str, +) -> Iterator[bytes]: + """Emit canonical index-v1 bytes, splicing sources/entries without materializing arrays.""" + logical = _object(metadata["logical"], "logical") + yield b'{"corpus_id":' + canonical_json(metadata["corpus_id"]) + yield b',"entries":[' + first = True + for chunk in entry_chunks: + if not first: + yield b"," + first = False + yield chunk + yield b"]" + yield b',"genesis_hash":' + canonical_json(metadata["genesis_hash"]) + yield b',"index_sha256":' + canonical_json(index_sha) + yield b',"logical":' + canonical_json( + {"sha256": logical["sha256"], "size": logical["size"]} + ) + yield b',"network":' + canonical_json(metadata["network"]) + yield b',"network_magic":' + canonical_json(metadata["network_magic"]) + yield b',"range":' + canonical_json( + { + "start_height": _object(metadata["range"], "range")["start_height"], + "stop_height": _object(metadata["range"], "range")["stop_height"], + } + ) + yield b',"schema":' + canonical_json(metadata["schema"]) + yield b',"source_tip_hash":' + canonical_json(metadata["source_tip_hash"]) + yield b',"sources":[' + first = True + for chunk in source_chunks: + if not first: + yield b"," + first = False + yield chunk + yield b"]" + yield b',"version":' + canonical_json(metadata["version"]) + yield b"}" + + +def _load_index(path: Path) -> IndexDocument: + """Parse and structurally validate an index file. No source I/O is performed.""" + try: + with open(path, "rb") as handle: + data = handle.read(_MAX_INDEX_BYTES + 1) + except OSError as err: + raise ContractError(f"cannot read index file {path}: {err}") from err + if len(data) > _MAX_INDEX_BYTES: + raise ContractError("index file exceeds its size bound") + try: + raw = json.loads(data.decode("utf-8"), object_pairs_hook=_reject_duplicates) + except ValueError as err: + raise ContractError(f"index file {path} is not valid JSON: {err}") from err + if not isinstance(raw, dict): + raise ContractError("index root must be a JSON object") + if frozenset(raw.keys()) != _INDEX_ROOT_KEYS: + raise ContractError("index root fields do not match index-v1") + if _text(raw["schema"], "schema") != INDEX_SCHEMA: + raise ContractError("index schema is not bitcoin-rs-corpus-index-v1") + if _u32(raw["version"], "version") != INDEX_VERSION: + raise ContractError("index version is not 1") + corpus_id = _text(raw["corpus_id"], "corpus_id") + network = _text(raw["network"], "network") + network_magic = _lower_hex(raw["network_magic"], "network_magic", 8) + genesis_hash = _lower_hex(raw["genesis_hash"], "genesis_hash", 64) + source_tip_hash = _lower_hex(raw["source_tip_hash"], "source_tip_hash", 64) + range_obj = _object(raw["range"], "range") + if frozenset(range_obj.keys()) != _RANGE_KEYS: + raise ContractError("index range fields do not match v1") + start_height = _u32(range_obj["start_height"], "range.start_height") + stop_height = _u32(range_obj["stop_height"], "range.stop_height") + if start_height != 0: + raise ContractError("index range must start at genesis") + logical_obj = _object(raw["logical"], "logical") + if frozenset(logical_obj.keys()) != _INDEX_LOGICAL_KEYS: + raise ContractError("index logical fields do not match v1") + logical_sha256 = _lower_hex(logical_obj["sha256"], "logical.sha256", 64) + logical_size = _u64(logical_obj["size"], "logical.size") + index_sha256 = _lower_hex(raw["index_sha256"], "index_sha256", 64) + raw_sources = _array(raw["sources"], "sources") + if len(raw_sources) == 0 or len(raw_sources) > _MAX_SOURCES: + raise ContractError("index sources are outside their count bound") + sources: list[IndexSource] = [] + seen_names: set[str] = set() + for position, item in enumerate(raw_sources): + body = _object(item, f"sources[{position}]") + if frozenset(body.keys()) != _INDEX_SOURCE_KEYS: + raise ContractError(f"sources[{position}] fields do not match v1") + name = _check_source_name(body["name"]) + if name in seen_names: + raise ContractError(f"duplicate index source {name!r}") + seen_names.add(name) + size = _u64(body["size"], f"sources[{position}].size") + if size > _MAX_ARCHIVE_SIZE: + raise ContractError(f"sources[{position}].size exceeds its bound") + sha256 = _lower_hex(body["sha256"], f"sources[{position}].sha256", 64) + sources.append(IndexSource(name=name, size=size, sha256=sha256)) + raw_entries = _array(raw["entries"], "entries") + if len(raw_entries) == 0 or len(raw_entries) > _MAX_BLOCK_COUNT: + raise ContractError("index entries are outside their count bound") + entries: list[IndexEntry] = [] + for position, item in enumerate(raw_entries): + body = _object(item, f"entries[{position}]") + if frozenset(body.keys()) != _INDEX_ENTRY_KEYS: + raise ContractError(f"entries[{position}] fields do not match v1") + height = _u64(body["height"], f"entries[{position}].height") + block_hash = _lower_hex(body["hash"], f"entries[{position}].hash", 64) + source = _u64(body["source"], f"entries[{position}].source") + offset = _u64(body["offset"], f"entries[{position}].offset") + length = _u64(body["length"], f"entries[{position}].length") + if source >= len(sources): + raise ContractError( + f"entries[{position}].source is outside the sources array" + ) + if length < MIN_HEADER or length > MAX_PAYLOAD: + raise ContractError( + f"entries[{position}].length is outside consensus bounds" + ) + if offset + HEADER_LEN + length > sources[source].size: + raise ContractError(f"entries[{position}] range exceeds its source size") + entries.append( + IndexEntry( + height=height, + hash=block_hash, + source=source, + offset=offset, + length=length, + ) + ) + for position, entry in enumerate(entries): + if entry.height != position: + raise ContractError("index entries must be contiguous heights from genesis") + return IndexDocument( + corpus_id=corpus_id, + network=network, + network_magic=network_magic, + genesis_hash=genesis_hash, + start_height=start_height, + stop_height=stop_height, + source_tip_hash=source_tip_hash, + sources=tuple(sources), + entries=tuple(entries), + logical_size=logical_size, + logical_sha256=logical_sha256, + index_sha256=index_sha256, + ) + + +def _check_index_digest(doc: IndexDocument) -> None: + metadata = { + "corpus_id": doc.corpus_id, + "genesis_hash": doc.genesis_hash, + "logical": {"sha256": doc.logical_sha256, "size": doc.logical_size}, + "network": doc.network, + "network_magic": doc.network_magic, + "range": {"start_height": doc.start_height, "stop_height": doc.stop_height}, + "schema": INDEX_SCHEMA, + "source_tip_hash": doc.source_tip_hash, + "version": INDEX_VERSION, + } + preimage = hashlib.sha256() + for chunk in _index_chunks( + metadata, + [_index_source_chunk(source) for source in doc.sources], + [_index_entry_chunk(entry) for entry in doc.entries], + ZERO_SHA256, + ): + preimage.update(chunk) + if preimage.hexdigest() != doc.index_sha256: + raise ContractError("index_sha256 does not match the canonical preimage") + + +def _check_index_against_freeze(freeze: Freeze, doc: IndexDocument) -> Product: + chosen = product(freeze, doc.corpus_id) + if doc.network != freeze.network: + raise ContractError("index network is not mainnet") + if doc.network_magic != freeze.network_magic.hex(): + raise ContractError("index network magic is not mainnet") + if doc.genesis_hash != freeze.genesis_hash: + raise ContractError("index genesis hash is not mainnet genesis") + if doc.start_height != 0 or doc.stop_height != chosen.stop_height: + raise ContractError( + f"{chosen.corpus_id} index range is not genesis through the frozen tip" + ) + if doc.source_tip_hash != chosen.stop_hash: + raise ContractError(f"{chosen.corpus_id} index tip is not the frozen stop hash") + if len(doc.entries) != chosen.block_count: + raise ContractError( + f"{chosen.corpus_id} index entry count is not the frozen block count" + ) + return chosen + + +class IndexWriter: + """Streaming indexed-corpus builder over caller-supplied payloads and locators. + + The writer owns append invariants: contiguous height, frozen capacity, + previous-header linkage, the rolling logical digest, and per-frame range + checks against registered source sizes. Source digests are registered by + the caller (see ``build_index_from_scan`` for custody-derived values); + ``verify_index`` recomputes them from disk before any frame is accepted. + Outputs are published atomically with no-clobber semantics; inputs are + never pruned, truncated, or mutated. Consistency: build on a quiescent + source directory (snapshot isolation); finishing is idempotent only by + retrying with a fresh writer and a fresh destination. + """ + + def __init__(self, freeze: Freeze, corpus_id: str) -> None: + self._freeze = freeze + self._chosen = product(freeze, corpus_id) + self._sources: list[IndexSource] = [] + self._entries: list[IndexEntry] = [] + self._count = 0 + self._logical_size = 0 + self._logical_sha = hashlib.sha256() + self._last_hash: str | None = None + + def add_source(self, source: IndexSource) -> int: + checked_name = _check_source_name(source.name) + if source.size > _MAX_ARCHIVE_SIZE: + raise ContractError("index source size exceeds its bound") + _lower_hex(source.sha256, "source.sha256", 64) + for existing in self._sources: + if existing.name == checked_name: + raise ContractError(f"duplicate index source {checked_name!r}") + if len(self._sources) >= _MAX_SOURCES: + raise ContractError("index sources exceed their count bound") + index = len(self._sources) + self._sources.append( + IndexSource(name=checked_name, size=source.size, sha256=source.sha256) + ) + return index + + def append( + self, + payload: bytes, + at: SourceOffset, + expected_hash: str | None = None, + ) -> IndexEntry: + if self._count >= self._chosen.block_count: + raise ContractError("index append exceeds the frozen block count") + if len(payload) > MAX_PAYLOAD or len(payload) < MIN_HEADER: + raise ContractError("index payload is outside consensus bounds") + block_hash = header_block_hash(payload) + if expected_hash is not None and block_hash != _lower_hex( + expected_hash, "expected_hash", 64 + ): + raise ContractError("advertised header hash does not match the payload") + height = self._count + if height == 0: + if block_hash != self._freeze.genesis_hash: + raise ContractError("index genesis hash is not mainnet genesis") + else: + if self._last_hash is None: + raise ContractError("index writer lost its linkage cursor") + if not _prev_field_matches(payload, self._last_hash): + raise ContractError("indexed block does not link to its predecessor") + if ( + height == self._chosen.stop_height + and block_hash != self._chosen.stop_hash + ): + raise ContractError( + f"{self._chosen.corpus_id} index tip is not the frozen stop hash" + ) + if at.source < 0 or at.source >= len(self._sources): + raise ContractError("indexed frame source is outside the sources array") + if at.offset < 0: + raise ContractError("indexed frame offset is negative") + recorded = self._sources[at.source] + if at.offset + HEADER_LEN + len(payload) > recorded.size: + raise ContractError("indexed frame range exceeds its source size") + entry = IndexEntry( + height=height, + hash=block_hash, + source=at.source, + offset=at.offset, + length=len(payload), + ) + self._entries.append(entry) + self._logical_sha.update( + self._freeze.network_magic + struct.pack(" IndexSummary: + if self._count != self._chosen.block_count: + raise ContractError( + f"{self._chosen.corpus_id} index has {self._count} blocks, expected {self._chosen.block_count}" + ) + if self._last_hash is None: + raise ContractError("index writer has no committed tip") + if self._last_hash != self._chosen.stop_hash: + raise ContractError( + f"{self._chosen.corpus_id} index tip is not the frozen stop hash" + ) + intervals: dict[int, list[tuple[int, int]]] = {} + for entry in self._entries: + span = (entry.offset, entry.offset + HEADER_LEN + entry.length) + intervals.setdefault(entry.source, []).append(span) + for source, spans in intervals.items(): + spans.sort() + previous_end: int | None = None + for start, end in spans: + if previous_end is not None and start < previous_end: + raise ContractError( + f"index entries overlap in source {self._sources[source].name!r}" + ) + previous_end = end + logical_sha = self._logical_sha.hexdigest() + metadata = _index_metadata( + self._freeze, + self._chosen.corpus_id, + self._sources, + self._logical_size, + logical_sha, + ZERO_SHA256, + ) + preimage = hashlib.sha256() + for chunk in _index_chunks( + metadata, + [_index_source_chunk(source) for source in self._sources], + [_index_entry_chunk(entry) for entry in self._entries], + ZERO_SHA256, + ): + preimage.update(chunk) + index_sha = preimage.hexdigest() + file_sha = hashlib.sha256() + for chunk in _index_chunks( + metadata, + [_index_source_chunk(source) for source in self._sources], + [_index_entry_chunk(entry) for entry in self._entries], + index_sha, + ): + file_sha.update(chunk) + _write_all(index, chunk) + _write_all(index, b"\n") + file_sha.update(b"\n") + return IndexSummary( + count=self._count, + logical_size=self._logical_size, + logical_sha256=logical_sha, + index_sha256=index_sha, + index_file_sha256=file_sha.hexdigest(), + ) + + +def build_index_from_scan( + freeze: Freeze, corpus_id: str, source_dir: Path, index_path: Path +) -> IndexSummary: + """Scan Core blk files, select the canonical chain, and publish an index. + + Selection walks backward from the frozen stop hash through previous-header + links, so side-branch blocks in the same files are ignored. Every source + is fully hashed (streaming) before any frame is accepted, so same-length + source replacement is rejected. Frames are re-read in height order to bind + the logical digest, and every source is re-hashed after the re-read: a + file mutated between the pre-hash, the scan, and the re-read fails the + build instead of publishing an index over mixed generations. Full proof-of-work, merkle-root recomputation, and + coinbase consensus rules are not reimplemented here: they remain owned by + the Rust validation engines that consume the identical logical byte stream + (the same bytes the packed archive carries), plus ``classify_census`` for + script census. This keeps one authoritative consensus owner and avoids a + divergent second implementation. Consistency: snapshot isolation over a + quiescent ``source_dir``; the build is idempotent by retrying with a fresh + destination (existing destinations are never clobbered). + """ + chosen = product(freeze, corpus_id) + if not source_dir.is_dir(): + raise ContractError(f"source directory {source_dir} is not a directory") + _refuse_existing(index_path) + try: + names = sorted(entry.name for entry in source_dir.iterdir()) + except OSError as err: + raise ContractError( + f"cannot list source directory {source_dir}: {err}" + ) from err + blk_names = [name for name in names if _SOURCE_NAME_RE.fullmatch(name) is not None] + if len(blk_names) == 0: + raise ContractError( + f"source directory {source_dir} holds no blkNNNNN.dat files" + ) + if len(blk_names) > _MAX_SOURCES: + raise ContractError("source file count exceeds its bound") + for name in blk_names: + _check_source_name(name) + sources: list[IndexSource] = [] + for name in blk_names: + validated = _source_path(source_dir, name) + size, digest = _hash_file(validated) + sources.append(IndexSource(name=name, size=size, sha256=digest)) + by_hash: dict[str, _ScanRecord] = {} + for position, source in enumerate(sources): + path = _source_path(source_dir, source.name) + try: + with _open_source_nofollow(path) as handle: + for meta, payload in iter_frames(handle, freeze.network_magic): + block_hash = header_block_hash(payload) + prev_display = payload[4:36][::-1].hex() + if block_hash in by_hash: + raise ContractError( + f"duplicate block hash {block_hash} across sources" + ) + by_hash[block_hash] = _ScanRecord( + prev=prev_display, + source=position, + offset=meta.offset, + length=meta.payload_length, + ) + except OSError as err: + raise ContractError(f"cannot scan source {source.name}: {err}") from err + tip = chosen.stop_hash + if tip not in by_hash: + raise ContractError(f"{corpus_id} stop hash is absent from the sources") + reversed_hashes: list[str] = [tip] + seen: set[str] = {tip} + cursor = tip + while cursor != freeze.genesis_hash: + record = by_hash.get(cursor) + if record is None: + raise ContractError(f"canonical chain has a gap at {cursor}") + cursor = record.prev + if cursor in seen: + raise ContractError("canonical chain contains a linkage cycle") + seen.add(cursor) + reversed_hashes.append(cursor) + if len(reversed_hashes) > chosen.block_count: + raise ContractError( + f"{corpus_id} canonical chain exceeds the frozen block count" + ) + if len(reversed_hashes) != chosen.block_count: + raise ContractError( + f"{corpus_id} canonical chain length is not the frozen block count" + ) + ordered = list(reversed(reversed_hashes)) + if ordered[0] != freeze.genesis_hash: + raise ContractError("canonical chain does not start at genesis") + writer = IndexWriter(freeze, corpus_id) + for source in sources: + writer.add_source(source) + reader = _SingleSourceReader(source_dir, sources, freeze.network_magic) + try: + for height, block_hash in enumerate(ordered): + record = by_hash.get(block_hash) + if record is None: + raise ContractError(f"canonical chain has a gap at height {height}") + entry = IndexEntry( + height=height, + hash=block_hash, + source=record.source, + offset=record.offset, + length=record.length, + ) + payload = reader.read(entry) + writer.append( + payload, + SourceOffset(source=record.source, offset=record.offset), + expected_hash=block_hash, + ) + finally: + reader.close() + _reverify_sources(source_dir, sources) + scratch = _unique_scratch(index_path) + try: + with open(scratch, "xb") as index_fh: + summary = writer.finish(index_fh) + _synced(index_fh) + except OSError as err: + raise ContractError(f"cannot write index scratch file: {err}") from err + try: + _commit_scratch(scratch, index_path) + finally: + try: + scratch.unlink(missing_ok=True) + except OSError as err: + print(f"warning: cannot remove index scratch file: {err}", file=sys.stderr) + return summary + + +def _reverify_sources(source_dir: Path, sources: Sequence[IndexSource]) -> None: + """Re-hash every source after the scan/re-read and refuse on drift. + + The pre-hash, the frame scan, and the height-ordered re-read are three + separate opens. Without this gate a file replaced mid-build could mix + generations: the scan would locate frames in one generation while the + re-read binds digests from another. Re-hashing here binds all three + passes to one generation before `writer.finish` runs. + """ + for source in sources: + path = _source_path(source_dir, source.name) + size, digest = _hash_file(path) + if size != source.size or digest != source.sha256: + raise ContractError( + f"source {source.name} changed during the scan;" + " rebuild on a quiescent directory" + ) + + +def _actual_frame_spans( + source_dir: Path, sources: Sequence[IndexSource], magic: bytes +) -> list[set[tuple[int, int]]]: + """Pre-parse every source with the `iter_frames` owner. + + Returns per-source sets of exact top-level ``(offset, length)`` spans. + Only frame metadata is retained; payload bytes are discarded at once, so + no second packed copy is held. `ContractError` from `iter_frames` (magic + mismatch, truncation, over-limit length) propagates unchanged; `OSError` + becomes `ContractError` to preserve indexed-path refusal behavior. + """ + spans: list[set[tuple[int, int]]] = [] + for source in sources: + path = _source_path(source_dir, source.name) + actual: set[tuple[int, int]] = set() + try: + with _open_source_nofollow(path) as handle: + for meta, _payload in iter_frames(handle, magic): + actual.add((meta.offset, meta.payload_length)) + except OSError as err: + raise ContractError(f"cannot scan source {source.name}: {err}") from err + spans.append(actual) + return spans + + +def verify_index(freeze: Freeze, index_path: Path, source_dir: Path) -> Product: + """Fully verify an index against its sources. Pre-hashes every source before accepting frames.""" + doc = _load_index(index_path) + _check_index_digest(doc) + chosen = _check_index_against_freeze(freeze, doc) + for source in doc.sources: + path = _source_path(source_dir, source.name) + try: + actual_size = path.stat().st_size + except OSError as err: + raise ContractError(f"cannot stat source {source.name}: {err}") from err + if actual_size != source.size: + raise ContractError(f"source {source.name} size does not match the index") + _, digest = _hash_file(path) + if digest != source.sha256: + raise ContractError(f"source {source.name} digest does not match the index") + actual_spans = _actual_frame_spans(source_dir, doc.sources, freeze.network_magic) + logical_sha = hashlib.sha256() + logical_size = 0 + last_hash: str | None = None + intervals: dict[int, list[tuple[int, int]]] = {} + reader = _SingleSourceReader(source_dir, doc.sources, freeze.network_magic) + try: + for height, entry in enumerate(doc.entries): + if entry.height != height: + raise ContractError("index entries are not in height order") + if (entry.offset, entry.length) not in actual_spans[entry.source]: + raise ContractError( + f"entries[{height}] is not a top-level frame in source " + f"{doc.sources[entry.source].name!r}" + ) + payload = reader.read(entry) + last_hash = _verify_entry_hash( + freeze, chosen, entry, height, payload, last_hash + ) + logical_sha.update(freeze.network_magic + struct.pack(" Iterator[tuple[int, str, bytes]]: + """Yield verified (height, hash, payload) triples in height order. + + Every source is fully hashed (streaming) before the first payload is + yielded, so same-length replacement is rejected up front. Actual top-level + frames are pre-parsed with `iter_frames` before the first payload is + yielded, so embedded or misaligned spans are rejected even when their + magic and length decode. Each payload is hash- and linkage-checked before + it is yielded. The caller must exhaust the + iterator: the trailing logical-digest check binds stream identity for the + comparator, and an abandoned prefix must not be accepted. Retrying is + idempotent. + """ + doc = _load_index(index_path) + _check_index_digest(doc) + chosen = _check_index_against_freeze(freeze, doc) + for source in doc.sources: + path = _source_path(source_dir, source.name) + try: + actual_size = path.stat().st_size + except OSError as err: + raise ContractError(f"cannot stat source {source.name}: {err}") from err + if actual_size != source.size: + raise ContractError(f"source {source.name} size does not match the index") + _, digest = _hash_file(path) + if digest != source.sha256: + raise ContractError(f"source {source.name} digest does not match the index") + actual_spans = _actual_frame_spans(source_dir, doc.sources, freeze.network_magic) + logical_sha = hashlib.sha256() + logical_size = 0 + last_hash: str | None = None + intervals: dict[int, list[tuple[int, int]]] = {} + reader = _SingleSourceReader(source_dir, doc.sources, freeze.network_magic) + try: + for height, entry in enumerate(doc.entries): + if entry.height != height: + raise ContractError("index entries are not in height order") + if (entry.offset, entry.length) not in actual_spans[entry.source]: + raise ContractError( + f"entries[{height}] is not a top-level frame in source " + f"{doc.sources[entry.source].name!r}" + ) + payload = reader.read(entry) + block_hash = _verify_entry_hash( + freeze, chosen, entry, height, payload, last_hash + ) + logical_sha.update(freeze.network_magic + struct.pack(" Iterator[bytes]: + """Yield verified Core-framed records identical to the packed archive bytes.""" + for _, _, payload in iter_indexed_payloads(freeze, index_path, source_dir): + yield write_frame(payload, freeze.network_magic) + + def _unique_scratch(dest: Path) -> Path: return dest.parent / f".{dest.name}.tmp.{os.getpid()}.{uuid.uuid4().hex}" @@ -1043,13 +2069,58 @@ def _synced(fh: BinaryIO) -> None: os.fsync(fh.fileno()) +def build_archive_from_index( + freeze: Freeze, + index_path: Path, + source_dir: Path, + archive_path: Path, + manifest_path: Path, +) -> CorpusSummary: + """Pack a byte-identical archive from a verified index. + + The payload stream fully verifies its sources while yielding (source + pre-hash, top-level span gate, per-payload hash/linkage, trailing + logical digest), so the packed bytes are the logical stream the index + fingerprinted. The published manifest digest and size must equal the + index logical digest and size; a mismatch unpublishes both outputs and + fails instead of leaving divergent digests behind. + """ + doc = _load_index(index_path) + _check_index_digest(doc) + chosen = _check_index_against_freeze(freeze, doc) + summary = _run_writer( + freeze, + chosen.corpus_id, + archive_path, + manifest_path, + ( + (block_hash, payload) + for _, block_hash, payload in iter_indexed_payloads( + freeze, index_path, source_dir + ) + ), + ) + if ( + summary.archive_sha256 != doc.logical_sha256 + or summary.archive_size != doc.logical_size + ): + with contextlib.suppress(OSError): + archive_path.unlink() + with contextlib.suppress(OSError): + manifest_path.unlink() + raise ContractError( + "packed archive digest does not match the index logical digest" + ) + return summary + + def _run_writer( freeze: Freeze, corpus_id: str, archive_path: Path, manifest_path: Path, items: Iterable[tuple[str | None, bytes]], -) -> None: +) -> CorpusSummary: if archive_path.resolve() == manifest_path.resolve(): raise ContractError("archive and manifest paths must differ") _refuse_existing(archive_path) @@ -1058,7 +2129,10 @@ def _run_writer( manifest_path.parent.mkdir(parents=True, exist_ok=True) archive_scratch = _unique_scratch(archive_path) manifest_scratch = _unique_scratch(manifest_path) - spool_scratch = _unique_scratch(archive_path) + # The spool never commits, but it gets its own scratch namespace + # (`.spool.tmp.*`) instead of sharing the archive's, so the three live + # scratch paths are pairwise distinct by construction, not just by UUID. + spool_scratch = _unique_scratch(archive_path.parent / "spool") try: with ( open(archive_scratch, "xb") as archive_fh, @@ -1080,6 +2154,7 @@ def _run_writer( print(f"wrote {archive_path} ({summary.archive_size} bytes)") print(f"wrote {manifest_path}") print(f"archive sha256 {summary.archive_sha256}") + return summary def main(argv: Sequence[str] | None = None) -> int: @@ -1108,6 +2183,29 @@ def main(argv: Sequence[str] | None = None) -> int: classify_cmd.add_argument("--contract", required=True, choices=("C150", "Cmodern")) classify_cmd.add_argument("--counters", type=Path, required=True) + build_index_cmd = sub.add_parser( + "build-index", help="build an indexed corpus over Core blk files" + ) + build_index_cmd.add_argument( + "--corpus-id", required=True, choices=("C150", "Cmodern") + ) + build_index_cmd.add_argument("--source-dir", type=Path, required=True) + build_index_cmd.add_argument("--index", type=Path, required=True) + + verify_index_cmd = sub.add_parser( + "verify-index", help="verify an indexed corpus against its sources" + ) + verify_index_cmd.add_argument("--index", type=Path, required=True) + verify_index_cmd.add_argument("--source-dir", type=Path, required=True) + + build_archive_cmd = sub.add_parser( + "build-archive", help="pack a byte-identical archive from a verified index" + ) + build_archive_cmd.add_argument("--index", type=Path, required=True) + build_archive_cmd.add_argument("--source-dir", type=Path, required=True) + build_archive_cmd.add_argument("--archive", type=Path, required=True) + build_archive_cmd.add_argument("--manifest", type=Path, required=True) + args = parser.parse_args(argv) freeze = load_freeze() if args.command == "export": @@ -1148,6 +2246,22 @@ def main(argv: Sequence[str] | None = None) -> int: result = classify_census(freeze, args.contract, _load_json(args.counters)) json.dump(result, sys.stdout, indent=2, sort_keys=True) sys.stdout.write("\n") + elif args.command == "build-index": + summary = build_index_from_scan( + freeze, args.corpus_id, args.source_dir, args.index + ) + print(f"wrote {args.index} ({summary.logical_size} bytes logical)") + print(f"logical sha256 {summary.logical_sha256}") + print(f"index sha256 {summary.index_sha256}") + elif args.command == "verify-index": + verify_index(freeze, args.index, args.source_dir) + print(f"verified {args.index}") + elif args.command == "build-archive": + summary = build_archive_from_index( + freeze, args.index, args.source_dir, args.archive, args.manifest + ) + print(f"wrote {args.archive} ({summary.archive_size} bytes)") + print(f"archive sha256 {summary.archive_sha256}") return 0 diff --git a/tools/campaign-corpus/test_corpus.py b/tools/campaign-corpus/test_corpus.py index a35149598..2cdbd7482 100755 --- a/tools/campaign-corpus/test_corpus.py +++ b/tools/campaign-corpus/test_corpus.py @@ -24,7 +24,8 @@ from dataclasses import replace from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path -from typing import Protocol +from typing import BinaryIO, Protocol, TypeGuard +from unittest import mock import corpus from corpus import ContractError @@ -70,6 +71,24 @@ def write(self, data: bytes) -> int: ... # --------------------------------------------------------------------------- +def _is_binary_stream(value: object) -> TypeGuard[BinaryIO]: + return ( + hasattr(value, "read") + and hasattr(value, "write") + and hasattr(value, "seek") + and hasattr(value, "tell") + and hasattr(value, "truncate") + and hasattr(value, "readline") + and hasattr(value, "flush") + ) + + +def _checked_binary(stream: object) -> BinaryIO: + """Boundary parse: prove a caller-owned double exposes the binary surface.""" + assert _is_binary_stream(stream) + return stream + + def _dsha(data: bytes) -> bytes: return hashlib.sha256(hashlib.sha256(data).digest()).digest() @@ -233,18 +252,16 @@ def _isolated_products(): root = Path(raw) products = _write_products(root) original_loader = corpus.load_freeze - corpus.PRODUCTS_PATH = products - def _load_isolated(path: Path = products) -> corpus.Freeze: + def _isolated_load(path: Path = products) -> corpus.Freeze: return original_loader(path) - # Use the module dictionary so ty does not treat the replacement - # function as an incompatible attribute assignment. - vars(corpus)["load_freeze"] = _load_isolated + corpus.PRODUCTS_PATH = products + try: - yield root + with mock.patch.object(corpus, "load_freeze", _isolated_load): + yield root finally: - corpus.load_freeze = original_loader corpus.PRODUCTS_PATH = _ORIGINAL_PRODUCTS_PATH @@ -580,12 +597,18 @@ def test_iter_frames_streams_frames_and_offsets(self) -> None: def test_iter_frames_survives_short_reads(self) -> None: for chunk in (1, 3, 7): with self.subTest(chunk=chunk): - frames = list(corpus.iter_frames(_ShortReads(_GOLDEN_ARCHIVE, chunk))) + frames = list( + corpus.iter_frames( + _checked_binary(_ShortReads(_GOLDEN_ARCHIVE, chunk)) + ) + ) self.assertEqual([payload for _, payload in frames], list(_LINKED)) def test_iter_frames_clean_eof_ends_iteration(self) -> None: self.assertEqual(list(corpus.iter_frames(io.BytesIO(b""))), []) - self.assertEqual(list(corpus.iter_frames(_ShortReads(b"", 4))), []) + self.assertEqual( + list(corpus.iter_frames(_checked_binary(_ShortReads(b"", 4)))), [] + ) def test_iter_frames_rejects_wrong_magic(self) -> None: archive = _frame(_GENESIS, magic=b"TEST") @@ -607,13 +630,13 @@ def test_iter_frames_rejects_oversize_length_before_payload_read(self) -> None: oversized = MAGIC + struct.pack(" None: stream = _RecordingReads(_GOLDEN_ARCHIVE) - frames = list(corpus.iter_frames(stream)) + frames = list(corpus.iter_frames(_checked_binary(stream))) self.assertEqual(len(frames), 3) # No request may exceed one frame header (8) or one declared payload # (80); a whole-file read request would show up here. @@ -630,7 +653,9 @@ def test_iter_length_prefixed_round_trips_payloads(self) -> None: def test_iter_length_prefixed_survives_short_reads(self) -> None: payloads = [_GENESIS, b"q" * 80] blob = b"".join(struct.pack(" None: @@ -643,7 +668,7 @@ def test_iter_length_prefixed_rejects_truncation(self) -> None: def test_iter_length_prefixed_rejects_oversize_before_payload_read(self) -> None: stream = _RecordingReads(struct.pack(" None: @@ -1054,7 +1079,12 @@ def test_poisoned_append_keeps_committed_prefix_and_recovers(self) -> None: archive = _FlakyWrites(backing_archive) entries = _FlakyWrites(backing_entries) try: - writer = corpus.CorpusWriter(freeze, FIX_CORPUS, archive, entries) + writer = corpus.CorpusWriter( + freeze, + FIX_CORPUS, + _checked_binary(archive), + _checked_binary(entries), + ) writer.append(_GENESIS, expected_hash=GENESIS_HASH) writer.append(_BLOCK1, expected_hash=FIX_B1_HASH) facts = writer.prefix_facts() @@ -1068,7 +1098,11 @@ def test_poisoned_append_keeps_committed_prefix_and_recovers(self) -> None: # An old valid checkpoint may discard the poisoned tail, but the # coordinator (here, the test) owns the truncation. resumed = corpus.CorpusWriter.resume( - freeze, FIX_CORPUS, archive, entries, facts + freeze, + FIX_CORPUS, + _checked_binary(archive), + _checked_binary(entries), + facts, ) with self.assertRaises(ContractError): resumed.append(_BLOCK2, expected_hash=FIX_STOP_HASH) @@ -1098,7 +1132,12 @@ def test_failed_append_retries_on_same_writer_after_truncation(self) -> None: archive = _FlakyWrites(backing_archive) entries = _FlakyWrites(backing_entries) try: - writer = corpus.CorpusWriter(freeze, FIX_CORPUS, archive, entries) + writer = corpus.CorpusWriter( + freeze, + FIX_CORPUS, + _checked_binary(archive), + _checked_binary(entries), + ) writer.append(_GENESIS, expected_hash=GENESIS_HASH) facts = writer.prefix_facts() archive.poisoned = True @@ -1754,5 +1793,438 @@ def test_cmodern_rejects_broken_schnorr_equation(self) -> None: ) +_IDX_SUFFIX = b"indexed-tx-area!" + + +def _idx_payload(header: bytes) -> bytes: + return header + _IDX_SUFFIX + + +_IDX_LINKED = tuple(_idx_payload(header) for header in _LINKED) + + +def _idx_fork() -> bytes: + return _branch_header(_dsha(_GENESIS), "33" * 32, 6) + _IDX_SUFFIX + + +def _write_idx_sources(root: Path) -> Path: + source_dir = root / "blks" + source_dir.mkdir() + fork = _idx_fork() + (source_dir / "blk00000.dat").write_bytes( + b"".join(_frame(payload) for payload in (_IDX_LINKED[0], _IDX_LINKED[1], fork)) + ) + (source_dir / "blk00001.dat").write_bytes(_frame(_IDX_LINKED[2])) + return source_dir + + +def _packed_from_payloads(root: Path, freeze, payloads) -> bytes: + archive = io.BytesIO() + entries = io.BytesIO() + writer = corpus.CorpusWriter(freeze, FIX_CORPUS, archive, entries) + for payload, expected in zip(payloads, _EXPECTED_HASHES): + writer.append(payload, expected_hash=expected) + manifest = io.BytesIO() + writer.finish(manifest) + return archive.getvalue() + + +class IndexedParity(unittest.TestCase): + def test_indexed_logical_bytes_match_packed_archive(self) -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + freeze = _fixture_freeze(root) + source_dir = _write_idx_sources(root) + index_path = root / "canon.index" + summary = corpus.build_index_from_scan( + freeze, FIX_CORPUS, source_dir, index_path + ) + self.assertEqual(summary.count, 3) + product = corpus.verify_index(freeze, index_path, source_dir) + self.assertEqual(product.corpus_id, FIX_CORPUS) + packed = _packed_from_payloads(root, freeze, _IDX_LINKED) + framed = b"".join( + corpus.iter_indexed_framed(freeze, index_path, source_dir) + ) + self.assertEqual(framed, packed) + self.assertEqual(summary.logical_sha256, _sha256_hex(packed)) + triples = list(corpus.iter_indexed_payloads(freeze, index_path, source_dir)) + self.assertEqual([payload for _, _, payload in triples], list(_IDX_LINKED)) + self.assertEqual([height for height, _, _ in triples], [0, 1, 2]) + self.assertEqual(triples[2][1], FIX_STOP_HASH) + + def test_oversized_index_refuses_without_buffering_it(self) -> None: + with tempfile.TemporaryDirectory() as raw: + index_path = Path(raw) / "huge.index" + index_path.write_bytes(b" " * 16) + reads: list[int] = [] + real_open = open + + def bounded_open(path: object, mode: str = "r", *args: object) -> object: + handle = real_open(path, mode, *args) # type: ignore[call-overload] + if Path(str(path)) != index_path: + return handle + real_read = handle.read + + def read(size: int = -1) -> bytes: + reads.append(size) + return real_read(size) + + handle.read = read + return handle + + with mock.patch.object(corpus, "_MAX_INDEX_BYTES", 8): + with mock.patch("builtins.open", bounded_open): + with self.assertRaisesRegex(ContractError, "size bound"): + corpus._load_index(index_path) + self.assertEqual(reads, [9]) + + def test_kills_same_length_source_replacement(self) -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + freeze = _fixture_freeze(root) + source_dir = _write_idx_sources(root) + index_path = root / "canon.index" + corpus.build_index_from_scan(freeze, FIX_CORPUS, source_dir, index_path) + replacement = bytearray((source_dir / "blk00001.dat").read_bytes()) + replacement[10] ^= 0x01 + self.assertEqual(len(replacement), len(_frame(_IDX_LINKED[2]))) + (source_dir / "blk00001.dat").write_bytes(bytes(replacement)) + with self.assertRaises(ContractError): + corpus.verify_index(freeze, index_path, source_dir) + with self.assertRaises(ContractError): + list(corpus.iter_indexed_payloads(freeze, index_path, source_dir)) + + def test_kills_tx_area_content_mutation_behind_stable_header_hash(self) -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + freeze = _fixture_freeze(root) + source_dir = _write_idx_sources(root) + index_path = root / "canon.index" + corpus.build_index_from_scan(freeze, FIX_CORPUS, source_dir, index_path) + victim = bytearray((source_dir / "blk00000.dat").read_bytes()) + second_frame_at = 88 + len(_IDX_SUFFIX) + 8 + 80 + victim[second_frame_at + 5] ^= 0x01 + (source_dir / "blk00000.dat").write_bytes(bytes(victim)) + mutated_payload = _IDX_LINKED[1][:80] + bytes( + b ^ 0x01 if index == 5 else b + for index, b in enumerate(_IDX_LINKED[1][80:]) + ) + self.assertEqual(corpus.header_block_hash(mutated_payload), FIX_B1_HASH) + with self.assertRaises(ContractError): + corpus.verify_index(freeze, index_path, source_dir) + + def test_kills_source_order_mutation(self) -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + freeze = _fixture_freeze(root) + source_dir = _write_idx_sources(root) + index_path = root / "canon.index" + corpus.build_index_from_scan(freeze, FIX_CORPUS, source_dir, index_path) + fork = _idx_fork() + (source_dir / "blk00000.dat").write_bytes( + b"".join( + _frame(payload) + for payload in (_IDX_LINKED[1], _IDX_LINKED[0], fork) + ) + ) + with self.assertRaises(ContractError): + corpus.verify_index(freeze, index_path, source_dir) + + def test_kills_truncation(self) -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + freeze = _fixture_freeze(root) + source_dir = _write_idx_sources(root) + index_path = root / "canon.index" + corpus.build_index_from_scan(freeze, FIX_CORPUS, source_dir, index_path) + data = (source_dir / "blk00001.dat").read_bytes() + (source_dir / "blk00001.dat").write_bytes(data[:-1]) + with self.assertRaises(ContractError): + corpus.verify_index(freeze, index_path, source_dir) + with self.assertRaises(ContractError): + list(corpus.iter_indexed_payloads(freeze, index_path, source_dir)) + + def test_rejects_branch_only_sources_at_build(self) -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + freeze = _fixture_freeze(root) + source_dir = root / "blks" + source_dir.mkdir() + fork = _idx_fork() + (source_dir / "blk00000.dat").write_bytes( + _frame(_IDX_LINKED[0]) + _frame(fork) + ) + with self.assertRaises(ContractError): + corpus.build_index_from_scan( + freeze, FIX_CORPUS, source_dir, root / "bad.index" + ) + self.assertFalse((root / "bad.index").exists()) + + def test_rejects_unsafe_source_name_at_parse(self) -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + freeze = _fixture_freeze(root) + source_dir = _write_idx_sources(root) + index_path = root / "canon.index" + corpus.build_index_from_scan(freeze, FIX_CORPUS, source_dir, index_path) + doc = json.loads(index_path.read_text(encoding="utf-8")) + doc["sources"][0]["name"] = "../evil.dat" + tampered = root / "unsafe.index" + tampered.write_text(json.dumps(doc, sort_keys=True), encoding="utf-8") + with self.assertRaises(ContractError): + corpus.verify_index(freeze, tampered, source_dir) + + def test_rejects_embedded_offset_inside_frame(self) -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + freeze = _fixture_freeze(root) + source_dir = root / "blks" + source_dir.mkdir() + b2 = _IDX_LINKED[2] + embedded_frame = _frame(b2) + decoy_header = _branch_header(_dsha(_GENESIS), "44" * 32, 7) + decoy_payload = decoy_header + embedded_frame + b"\x00" * 16 + frame_g = _frame(_IDX_LINKED[0]) + frame_b1 = _frame(_IDX_LINKED[1]) + frame_decoy = _frame(decoy_payload) + frame_b2 = _frame(b2) + (source_dir / "blk00000.dat").write_bytes(frame_g + frame_b1 + frame_decoy) + (source_dir / "blk00001.dat").write_bytes(frame_b2) + decoy_offset = len(frame_g) + len(frame_b1) + embedded_offset = decoy_offset + 8 + 80 + with open(source_dir / "blk00000.dat", "rb") as handle: + handle.seek(embedded_offset) + header = handle.read(8) + self.assertEqual(header[:4], MAGIC) + self.assertEqual(struct.unpack_from(" None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + freeze = _fixture_freeze(root) + writer = corpus.IndexWriter(freeze, FIX_CORPUS) + writer.add_source( + corpus.IndexSource(name="blk00000.dat", size=1000, sha256="ab" * 32) + ) + at_zero = corpus.SourceOffset(source=0, offset=0) + at_later = corpus.SourceOffset(source=0, offset=208) + writer.append(_IDX_LINKED[0], at_zero, expected_hash=GENESIS_HASH) + writer.append(_IDX_LINKED[1], at_zero, expected_hash=FIX_B1_HASH) + writer.append(_IDX_LINKED[2], at_later, expected_hash=FIX_STOP_HASH) + with self.assertRaises(ContractError): + writer.finish(io.BytesIO()) + + +class OfflineScanHardening(unittest.TestCase): + def test_iter_frames_accepts_zero_padded_tail(self) -> None: + for tail in (b"\x00" * 8, b"\x00" * 63, b"\x00" * 4096): + with self.subTest(tail=len(tail)): + frames = list(corpus.iter_frames(io.BytesIO(_GOLDEN_ARCHIVE + tail))) + self.assertEqual([payload for _, payload in frames], list(_LINKED)) + + def test_iter_frames_rejects_nonzero_bytes_after_zero_padding(self) -> None: + evil = _GOLDEN_ARCHIVE + b"\x00" * 16 + b"\x01" + with self.assertRaisesRegex(ContractError, "trim the file"): + list(corpus.iter_frames(io.BytesIO(evil))) + + def test_iter_frames_offset_zero_mismatch_fails_closed_on_obfuscation( + self, + ) -> None: + with self.assertRaisesRegex(ContractError, "XOR-obfuscated"): + list(corpus.iter_frames(io.BytesIO(b"\x01\x02\x03\x04\x05\x06\x07\x08"))) + + def test_hash_file_refuses_symlink_without_following_it(self) -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + target = root / "target.dat" + target.write_bytes(b"victim-bytes") + link = root / "blk00000.dat" + os.symlink(target, link) + with self.assertRaisesRegex(ContractError, "must not be a symlink"): + corpus._hash_file(link) + self.assertEqual(target.read_bytes(), b"victim-bytes") + + def test_build_index_refuses_symlinked_source(self) -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + freeze = _fixture_freeze(root) + source_dir = _write_idx_sources(root) + stash = root / "stashed.bin" + target = source_dir / "blk00001.dat" + stash.write_bytes(target.read_bytes()) + target.unlink() + os.symlink(stash, target) + with self.assertRaisesRegex(ContractError, "must not be a symlink"): + corpus.build_index_from_scan( + freeze, FIX_CORPUS, source_dir, root / "evil.index" + ) + self.assertFalse((root / "evil.index").exists()) + + def test_verify_index_refuses_symlinked_source(self) -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + freeze = _fixture_freeze(root) + source_dir = _write_idx_sources(root) + index_path = root / "canon.index" + corpus.build_index_from_scan(freeze, FIX_CORPUS, source_dir, index_path) + stash = root / "stashed.bin" + target = source_dir / "blk00001.dat" + stash.write_bytes(target.read_bytes()) + target.unlink() + os.symlink(stash, target) + with self.assertRaisesRegex(ContractError, "must not be a symlink"): + corpus.verify_index(freeze, index_path, source_dir) + + def test_build_index_rejects_source_mutated_mid_scan(self) -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + freeze = _fixture_freeze(root) + source_dir = _write_idx_sources(root) + index_path = root / "canon.index" + real_append = corpus.IndexWriter.append + + def mutating_append(self, payload, at, expected_hash=None): + result = real_append(self, payload, at, expected_hash=expected_hash) + if self._count == 3: + victim = source_dir / "blk00001.dat" + victim.write_bytes(victim.read_bytes() + b"\x00") + return result + + with ( + mock.patch.object(corpus.IndexWriter, "append", mutating_append), + self.assertRaisesRegex(ContractError, "changed during the scan"), + ): + corpus.build_index_from_scan(freeze, FIX_CORPUS, source_dir, index_path) + self.assertFalse(index_path.exists()) + + def test_build_archive_from_index_matches_packed_bytes(self) -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + freeze = _fixture_freeze(root) + source_dir = _write_idx_sources(root) + index_path = root / "canon.index" + index_summary = corpus.build_index_from_scan( + freeze, FIX_CORPUS, source_dir, index_path + ) + packed = _packed_from_payloads(root, freeze, _IDX_LINKED) + archive_path = root / "repacked.dat" + manifest_path = root / "repacked.manifest" + built = corpus.build_archive_from_index( + freeze, index_path, source_dir, archive_path, manifest_path + ) + self.assertEqual(archive_path.read_bytes(), packed) + self.assertEqual(built.archive_sha256, index_summary.logical_sha256) + self.assertEqual(built.archive_size, index_summary.logical_size) + manifest_doc = json.loads(manifest_path.read_bytes().decode("utf-8")) + self.assertEqual( + manifest_doc["archive"]["sha256"], index_summary.logical_sha256 + ) + self.assertEqual( + manifest_doc["archive"]["size"], index_summary.logical_size + ) + with ( + archive_path.open("rb") as archive_fh, + manifest_path.open("rb") as manifest_fh, + (root / "repacked.spool").open("w+b") as spool_fh, + ): + product = corpus.verify_archive( + freeze, archive_fh, manifest_fh, entries=spool_fh + ) + self.assertEqual(product.corpus_id, FIX_CORPUS) + + def test_build_archive_cli_packs_verified_index(self) -> None: + with _isolated_products() as root: + freeze = corpus.load_freeze() + source_dir = _write_idx_sources(root) + index_path = root / "canon.index" + archive_path = root / "cli.dat" + manifest_path = root / "cli.manifest" + with redirect_stdout(io.StringIO()): + self.assertEqual( + corpus.main( + [ + "build-index", + "--corpus-id", + FIX_CORPUS, + "--source-dir", + str(source_dir), + "--index", + str(index_path), + ] + ), + 0, + ) + buffer = io.StringIO() + with redirect_stdout(buffer): + self.assertEqual( + corpus.main( + [ + "build-archive", + "--index", + str(index_path), + "--source-dir", + str(source_dir), + "--archive", + str(archive_path), + "--manifest", + str(manifest_path), + ] + ), + 0, + ) + self.assertIn(_sha256_hex(archive_path.read_bytes()), buffer.getvalue()) + self.assertEqual( + archive_path.read_bytes(), + _packed_from_payloads(root, freeze, _IDX_LINKED), + ) + + if __name__ == "__main__": unittest.main()