tiny debugging hacks

packages/next/src/build/swc/generated-native.d.ts

@@ -35,6 +35,43 @@ export declare class ExternalObject<T> {
     [K: symbol]: T
   }
 }
+/**
+ * Called by worker after WASM instantiation to register a dispatch callback.
+ *
+ * The `ops` object must have these methods:
+ *   - transform(programPtr, programLen, unresolvedMark, commentsProxy) → number
+ *   - getDiag() → number
+ *   - readBuf(ptr, len) → Buffer
+ *   - writeBuf(ptr, data: Buffer) → void
+ *   - alloc(size) → number
+ *   - free(ptr, size) → number
+ *
+ * A TSFN is created targeting this worker's event loop. When Rust needs to
+ * call a WASM export, it posts to this TSFN, which runs the appropriate
+ * method on `ops` and sends the result back via a sync channel.
+ */
+export declare function wasmWorkerRegisterCallback(
+  runtimeId: number,
+  instanceId: number,
+  ops: object
+): void
+/**
+ * Called by worker when WASM hits a host function import.
+ * Runs the Func closure synchronously on the worker thread and returns the result.
+ *
+ * The worker provides a `memoryAccessor` object with methods to read/write WASM memory:
+ *   - readBuf(ptr, len) → Buffer
+ *   - writeBuf(ptr, data: Buffer) → void
+ *   - alloc(size) → number
+ *   - free(ptr, size) → number
+ */
+export declare function wasmWorkerDispatchHostFn(
+  runtimeId: number,
+  instanceId: number,
+  fnIndex: number,
+  args: Array<number>,
+  memoryAccessor: object
+): unknown
 export declare function registerWorkerScheduler(
   creator: (arg: NapiWorkerCreation) => any,
   terminator: (arg: NapiWorkerTermination) => any

turbopack/crates/turbo-persistence/benches/mod.rs

@@ -576,6 +576,7 @@ fn prefill_multi_value_database(
 ) -> Result<Vec<Box<[u8]>>> {
     let db_config = TpDbConfig {
         family_configs: [FamilyConfig {
+            name: "test",
             kind: FamilyKind::MultiValue,
         }],
     };
@@ -648,6 +649,7 @@ fn setup_prefilled_multi_value_db(
 fn open_multi_value_db(path: &Path) -> TurboPersistence<SerialScheduler, 1> {
     let db_config = TpDbConfig {
         family_configs: [FamilyConfig {
+            name: "test",
             kind: FamilyKind::MultiValue,
         }],
     };
@@ -914,6 +916,7 @@ fn bench_write_multi_value(c: &mut Criterion) {
                     |(tempdir, keys, random_data)| {
                         let db_config = TpDbConfig {
                             family_configs: [FamilyConfig {
+                                name: "test",
                                 kind: FamilyKind::MultiValue,
                             }],
                         };
@@ -1203,7 +1206,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) {
                 },
                 |(key, hash)| {
                     let result = sst
-                        .lookup::<_, false>(hash, &key, key_block_cache, value_block_cache)
+                        .lookup::<_, _, _, false>(hash, &key, key_block_cache, value_block_cache)
                         .unwrap();
                     black_box(result)
                 },
@@ -1222,7 +1225,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) {
                 |i| keys[i as usize % keys.len()],
                 |(key, hash)| {
                     let result = sst
-                        .lookup::<_, false>(hash, &key, key_block_cache, value_block_cache)
+                        .lookup::<_, _, _, false>(hash, &key, key_block_cache, value_block_cache)
                         .unwrap();
                     black_box(result)
                 },
@@ -1248,7 +1251,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) {
                 },
                 |(key, hash)| {
                     let result = sst
-                        .lookup::<_, false>(hash, &key, key_block_cache, value_block_cache)
+                        .lookup::<_, _, _, false>(hash, &key, key_block_cache, value_block_cache)
                         .unwrap();
                     black_box(result)
                 },
@@ -1281,7 +1284,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) {
                 },
                 |(key, hash)| {
                     let result = sst
-                        .lookup::<_, false>(*hash, &key, key_block_cache, value_block_cache)
+                        .lookup::<_, _, _, false>(*hash, &key, key_block_cache, value_block_cache)
                         .unwrap();
                     black_box(result)
                 },

turbopack/crates/turbo-persistence/src/bin/sst_inspect.rs

@@ -24,6 +24,7 @@ use turbo_persistence::{
     BLOCK_HEADER_SIZE, checksum_block,
     meta_file::MetaFile,
     mmap_helper::advise_mmap_for_persistence,
+    sst_filter::SstFilter,
     static_sorted_file::{
         BLOCK_TYPE_FIXED_KEY_NO_HASH, BLOCK_TYPE_FIXED_KEY_WITH_HASH, BLOCK_TYPE_KEY_NO_HASH,
         BLOCK_TYPE_KEY_WITH_HASH, KEY_BLOCK_ENTRY_TYPE_BLOB, KEY_BLOCK_ENTRY_TYPE_DELETED,
@@ -205,33 +206,68 @@ fn format_bytes(bytes: u64) -> String {
     }
 }
 
-/// Collect SST info from all meta files in the database directory
+/// Collect SST info from all active meta files in the database directory,
+/// mirroring the DB's own open logic: read CURRENT, filter by .del files,
+/// and apply SstFilter to skip superseded entries.
 fn collect_sst_info(db_path: &Path) -> Result<BTreeMap<u32, Vec<SstInfo>>> {
-    let mut meta_files: Vec<PathBuf> = fs::read_dir(db_path)?
-        .filter_map(|entry| entry.ok())
-        .map(|entry| entry.path())
-        .filter(|path| path.extension().is_some_and(|ext| ext == "meta"))
-        .collect();
+    // Read the CURRENT sequence number — only files with seq <= current are valid.
+    let current: u32 = File::open(db_path.join("CURRENT"))
+        .context("Failed to open CURRENT file")?
+        .read_u32::<BE>()
+        .context("Failed to read CURRENT file")?;
+
+    // Read .del files to find sequences that were deleted but not yet cleaned up.
+    let mut deleted_seqs: HashSet<u32> = HashSet::new();
+    for entry in fs::read_dir(db_path)? {
+        let path = entry?.path();
+        if path.extension().and_then(|s| s.to_str()) == Some("del") {
+            let content = fs::read(&path)?;
+            let mut cursor: &[u8] = &content;
+            while !cursor.is_empty() {
+                deleted_seqs.insert(cursor.read_u32::<BE>()?);
+            }
+        }
+    }
 
-    meta_files.sort();
+    // Collect valid meta sequence numbers.
+    let mut meta_seqs: Vec<u32> = fs::read_dir(db_path)?
+        .filter_map(|e| e.ok())
+        .filter_map(|e| {
+            let path = e.path();
+            if path.extension().and_then(|s| s.to_str()) != Some("meta") {
+                return None;
+            }
+            let seq: u32 = path.file_stem()?.to_str()?.parse().ok()?;
+            if seq > current || deleted_seqs.contains(&seq) {
+                return None;
+            }
+            Some(seq)
+        })
+        .collect();
 
-    if meta_files.is_empty() {
-        bail!("No .meta files found in {}", db_path.display());
+    if meta_seqs.is_empty() {
+        bail!("No active .meta files found in {}", db_path.display());
     }
 
-    let mut family_sst_info: BTreeMap<u32, Vec<SstInfo>> = BTreeMap::new();
-
-    for meta_path in &meta_files {
-        // Extract sequence number from filename
-        let filename = meta_path.file_stem().and_then(|s| s.to_str()).unwrap_or("");
-        let seq_num: u32 = filename.parse().unwrap_or(0);
+    meta_seqs.sort_unstable();
 
-        let meta_file = MetaFile::open(db_path, seq_num)
-            .with_context(|| format!("Failed to open {}", meta_path.display()))?;
+    let mut meta_files: Vec<MetaFile> = meta_seqs
+        .iter()
+        .map(|&seq| {
+            MetaFile::open(db_path, seq).with_context(|| format!("Failed to open {seq:08}.meta"))
+        })
+        .collect::<Result<_>>()?;
 
-        let family = meta_file.family();
+    // Apply SstFilter (newest first) to drop entries superseded by a newer meta file.
+    let mut sst_filter = SstFilter::new();
+    for meta in meta_files.iter_mut().rev() {
+        sst_filter.apply_filter(meta);
+    }
 
-        for entry in meta_file.entries() {
+    let mut family_sst_info: BTreeMap<u32, Vec<SstInfo>> = BTreeMap::new();
+    for meta in &meta_files {
+        let family = meta.family();
+        for entry in meta.entries() {
             family_sst_info.entry(family).or_default().push(SstInfo {
                 sequence_number: entry.sequence_number(),
                 block_count: entry.block_count(),
@@ -317,7 +353,7 @@ fn read_block(
 ///
 /// Index block format: `[1B type][2B first_block][N * (8B hash + 2B block_index)]`.
 fn parse_key_block_indices(index_block: &[u8]) -> HashSet<u16> {
-    assert!(index_block.len() >= 4, "Index block too small");
+    assert!(index_block.len() >= 3, "Index block too small");
     let mut data = &index_block[1..]; // skip block type byte
     let first_block = data.read_u16::<BE>().unwrap();
     let mut indices = HashSet::new();