Skip to content

Commit 03ee2af

Browse files
aiexplore369claude
andcommitted
v1.0.0-rc.1: bench infra + LLM-pluggable corrector dual transport
samkhya-bench: WAVE4-F JOB-Slow real head-to-head harness wired, cold_cache module via posix_fadvise(POSIX_FADV_DONTNEED) for user-priv hosts, csv_to_parquet utility, llm_latency + tabpfn_latency campaign runners, ablation_runner --l4-variant v3 (WAVE5-E recovery), wave5f_audit example. samkhya-gpudb/scripts: LLM-pluggable corrector ships dual transport — canonical Python FastAPI (llm_infer_server.py, port 8766, what file 19 §4.1 measured) and Node TypeScript broader-appeal port (llm_infer_server.ts, port 8767). Same wire contract (POST /infer + GET /health, FEATURE_LEN=7, baseline_estimate u64, 4 backends: anthropic/openai/local/dummy). TabPFN-2.5 inference server (tabpfn_infer_server.py) for WAVE5-L2 measurement. Run scripts for both campaigns. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent a40873c commit 03ee2af

23 files changed

Lines changed: 4979 additions & 107 deletions

‎samkhya-bench/Cargo.toml‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ publish = false
1414

1515
[dependencies]
1616
samkhya-core = { workspace = true, features = ["gbt", "additive_gbt"] }
17-
samkhya-datafusion = { path = "../samkhya-datafusion", version = "1.0" }
17+
samkhya-datafusion = { path = "../samkhya-datafusion", version = "1.0.0-rc.1" }
1818
clap.workspace = true
1919
tokio.workspace = true
2020
datafusion = "46"
@@ -28,3 +28,18 @@ serde_json.workspace = true
2828
# `GbtCorrector::train` (which hardcodes `baseline_estimate`-only
2929
# features). Additive — does not affect any other binary.
3030
gbdt = "0.1"
31+
# WAVE-5M: cold-cache eviction via posix_fadvise(POSIX_FADV_DONTNEED).
32+
# libc is already transitively in tree (datafusion → arrow → ...); the
33+
# explicit dep here makes the cold_cache module's unsafe FFI call
34+
# auditable. Unix-gated at use site.
35+
libc = "0.2"
36+
# WAVE-5L: tabpfn_latency client (file 14). Pure-Rust rustls HTTP client,
37+
# matches the transport used by samkhya-core::residual::tabpfn so the
38+
# wire-level overhead is comparable to production. Only the bin needs
39+
# this; library code is unchanged.
40+
ureq = { version = "2", default-features = false, features = ["json", "tls"] }
41+
42+
[dev-dependencies]
43+
# Used by the cold_cache unit test to create a 100 MiB synthetic
44+
# parquet-shaped file in a self-cleaning temp directory.
45+
tempfile = "3"
Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
//! WAVE-5F audit helper: read row_count blobs from CSV-built `*.puffin`
2+
//! and Parquet-built `*.parquet.puffin` sidecars and print a table.
3+
//!
4+
//! Throwaway — not part of the v1.0 build matrix. Run via:
5+
//! cargo run -p samkhya-bench --release --example wave5f_audit -- samkhya-bench/data/job
6+
7+
use std::fs::File;
8+
use std::path::Path;
9+
10+
use samkhya_bench::imdb::{ROW_COUNT_KIND, TABLES};
11+
use samkhya_core::puffin::PuffinReader;
12+
13+
fn read_row_count(path: &Path) -> Option<u64> {
14+
let file = File::open(path).ok()?;
15+
let mut reader = PuffinReader::open(file).ok()?;
16+
let metas = reader.blobs().to_vec();
17+
for (i, meta) in metas.iter().enumerate() {
18+
if meta.kind == ROW_COUNT_KIND {
19+
let payload = reader.read_blob(i).ok()?;
20+
if payload.len() >= 8 {
21+
let mut buf = [0u8; 8];
22+
buf.copy_from_slice(&payload[..8]);
23+
return Some(u64::from_le_bytes(buf));
24+
}
25+
}
26+
}
27+
None
28+
}
29+
30+
fn main() {
31+
let imdb_dir = std::env::args()
32+
.nth(1)
33+
.unwrap_or_else(|| "samkhya-bench/data/job".to_string());
34+
let imdb_dir = Path::new(&imdb_dir);
35+
println!(
36+
"{:<20} {:>14} {:>14} {:>10}",
37+
"table", "csv_rows", "parquet_rows", "match"
38+
);
39+
println!("{}", "-".repeat(62));
40+
let mut total_csv: u64 = 0;
41+
let mut total_parq: u64 = 0;
42+
let mut all_match = true;
43+
for &t in TABLES {
44+
let csv_side = imdb_dir.join(format!("{t}.puffin"));
45+
let parq_side = imdb_dir.join(format!("{t}.parquet.puffin"));
46+
let c = read_row_count(&csv_side);
47+
let p = read_row_count(&parq_side);
48+
let m = matches!((c, p), (Some(x), Some(y)) if x == y);
49+
if !m {
50+
all_match = false;
51+
}
52+
total_csv += c.unwrap_or(0);
53+
total_parq += p.unwrap_or(0);
54+
println!(
55+
"{:<20} {:>14} {:>14} {:>10}",
56+
t,
57+
c.map(|n| n.to_string()).unwrap_or_else(|| "MISSING".into()),
58+
p.map(|n| n.to_string()).unwrap_or_else(|| "MISSING".into()),
59+
if m { "yes" } else { "NO" }
60+
);
61+
}
62+
println!("{}", "-".repeat(62));
63+
println!(
64+
"{:<20} {:>14} {:>14} {:>10}",
65+
"TOTAL",
66+
total_csv,
67+
total_parq,
68+
if all_match { "yes" } else { "NO" }
69+
);
70+
}

0 commit comments

Comments
 (0)