From 0d9320b028294b6a41da1f10a7e5414d0f1edda2 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Tue, 13 Oct 2026 23:20:57 -0400 Subject: [PATCH 1/3] eval: instant RQEs read one scrape, not 1 s A PromQL instant query reads one sample per series, but the synthetic spatial templates (sum by (job), quantile by (job)) looked back 1 s, so their N grew with the data rate (2e5 per job at 200/s, 1e4 at 10/s). They now look back one scrape interval (1 / samples_per_sec_per_series s), so N = series per job = 1e3 at any rate. autosketch_vs_asap scrapes a metric with samples_per_sec_per_series at that rate (record metrics keep 1 s), so a sub-second lookback holds whole scrapes, and writes sub-second windows and slides as fractional seconds. Co-Authored-By: Claude Opus 5.5 --- rqe-optimizer/examples/autosketch_vs_asap.rs | 31 +++++++++++++++----- scripts/export_autosketch_eval_table.py | 12 ++++---- scripts/test_export_autosketch_eval_table.py | 13 +++++--- 3 files changed, 39 insertions(+), 17 deletions(-) diff --git a/rqe-optimizer/examples/autosketch_vs_asap.rs b/rqe-optimizer/examples/autosketch_vs_asap.rs index 65072f7a..fe7cf115 100644 --- a/rqe-optimizer/examples/autosketch_vs_asap.rs +++ b/rqe-optimizer/examples/autosketch_vs_asap.rs @@ -98,9 +98,19 @@ const TRACES_TABLE: &str = "rqe-optimizer/data/autosketch-eval/table.json"; /// Objective weights of #777 ยง4: CPU only, then AWS Fargate (us-east-1, /// Linux/x86) $/vCPU-hour and $/GB-hour, so the objective reads in $/hour. const WEIGHTS: [(&str, f64, f64); 2] = [("cpu", 1.0, 0.0), ("fargate", 0.0405, 0.00445)]; -/// Every stream is scraped once a second: windows and slides are whole -/// seconds, and the stream's samples/s become its series count. +/// A stream's scrape interval when its table gives no per-series rate: windows +/// and slides are whole seconds, and the stream's samples/s become its series +/// count. const SCRAPE_MS: Millis = 1_000; +/// `ms` in seconds: an integer when whole, so second-grained plans read as +/// before, else fractional (an instant RQE's one-scrape window). +fn json_secs(ms: Millis) -> Value { + if ms.is_multiple_of(1000) { + json!(ms / 1000) + } else { + json!(ms as f64 / 1000.0) + } +} const BYTES_PER_GIB: f64 = (1u64 << 30) as f64; const SEED: u64 = 7; /// AutoSketch's benchmark time per evaluated configuration, at the paper's @@ -387,6 +397,12 @@ fn from_table( }; let capability = capability_of(r); let rate = r["label_set"]["arrival_rate_per_sec"].as_f64().unwrap(); + // A metric with a per-series rate (the synthetic gauge and counter) is + // scraped at it, so an instant RQE's one-scrape lookback holds one + // sample per series. Record metrics (http, flows) keep SCRAPE_MS. + let scrape_ms = workload["e2e"]["metrics"][metric]["samples_per_sec_per_series"] + .as_u64() + .map_or(SCRAPE_MS, |per_sec| 1_000 / per_sec); let groups = groups_of(r, capability); // A schema metric's RQE names its grouping (its `groups` are that // grouping's); otherwise the stream has one label `g`, the grouping. @@ -406,11 +422,10 @@ fn from_table( max_covered_share.insert(id.clone(), r["max_covered_share"].as_f64().unwrap()); } // One metric per stream, labels {g, x} (or the schema's and x), and - // with one scrape a second the full label set has one series per - // sample/s. A stream's RQEs differ only in range and grouping, so + // the full label set has one series per sample per scrape interval. A stream's RQEs differ only in range and grouping, so // take the largest per-RQE rate (conservative). Rates below `groups` // samples/s are rounded up so every group has a series. - let series = (rate * SCRAPE_MS as f64 / 1000.0).round() as u64; + let series = (rate * scrape_ms as f64 / 1000.0).round() as u64; let most = stream_series.entry(stream.clone()).or_default(); *most = (*most).max(series); let most = *most; @@ -436,7 +451,7 @@ fn from_table( MetricFacts { cardinality: [(labels.clone(), 0), (LabelSet::new(), 1)].into(), labels, - scrape_interval_ms: SCRAPE_MS, + scrape_interval_ms: scrape_ms, value_range: None, data_shape: BTreeMap::new(), // A stream is one value of its metric, so one capability. @@ -741,8 +756,8 @@ fn summarize( "rqe": r.id, "sketch": d.config.sketch, "params": d.config.sketch_config["params"], - "window_secs": d.window_ms / 1000, - "slide_secs": d.slide_ms / 1000, + "window_secs": json_secs(d.window_ms), + "slide_secs": json_secs(d.slide_ms), "merged_shards": r.lookback_ms / d.window_ms, "ingest_workers": rqe_optimizer::analytical_cost_model::ingest_workers(d, &w.facts), "latency_ms": latency, diff --git a/scripts/export_autosketch_eval_table.py b/scripts/export_autosketch_eval_table.py index 4b648fe2..23896e60 100644 --- a/scripts/export_autosketch_eval_table.py +++ b/scripts/export_autosketch_eval_table.py @@ -429,8 +429,9 @@ def synthetic_queries(windows=SYNTHETIC_WINDOWS, interval=SYNTHETIC_INTERVAL, Streams: `series/*` hold one instance per series (C groups), `job/*` one per job (J groups), and `label_0/*` one top-k sketch over all C series as keys. `*/dist` streams feed quantile sketches, `*/value` sums, `*/increment` - increases. Spatial templates repeat every 1 s and read the last second - (S = T = 1 s); temporal ones repeat every `interval` seconds over each + increases. Spatial templates repeat every 1 s and, like a PromQL instant + query, read one sample per series: one scrape interval (S = 1 / + SYNTHETIC_SAMPLES_PER_SEC s, T = 1 s); temporal ones repeat every `interval` seconds over each lookback in `windows`. RQEs that repeat another's query and range (template 10, D5) are kept once. Templates 7, `sum by (job) (rate(data_total[w]))`, and 9, `topk(32, sum by (label_0) (rate(data_total[w])))`, are dropped @@ -445,6 +446,7 @@ def synthetic_queries(windows=SYNTHETIC_WINDOWS, interval=SYNTHETIC_INTERVAL, gauge `data`, increments from the counter `data_total`. """ rate = SYNTHETIC_SERIES * SYNTHETIC_SAMPLES_PER_SEC # samples/s over all series + scrape = 1 / SYNTHETIC_SAMPLES_PER_SEC # an instant query's lookback, in s dataset = dataset or "synthetic/" + point_id({"templates": templates, "shared": 1}) ranges = [(w, WINDOW_SECS[w]) for w in windows.split(",")] seen = set() if seen is None else seen @@ -487,7 +489,7 @@ def add(query_id, stream, capability, rng, s, t, groups, keys=None, promql=None) add(f"quantile_over_time_p{q}", "series/dist", "quantile", label(rng), s, interval, C) for q in quantiles: - add(f"quantile_by_job_p{q}", "job/dist", "quantile", "1s", 1, 1, J) + add(f"quantile_by_job_p{q}", "job/dist", "quantile", "1s", scrape, 1, J) for rng, s in [(w, WINDOW_SECS[w]) for w in DASHBOARD_FIXED_WINDOWS]: if (rng, s) not in ranges: continue @@ -499,9 +501,9 @@ def add(query_id, stream, capability, rng, s, t, groups, keys=None, promql=None) return schema_queries(interval, dataset, range_with_interval, seen, metric, e2e) assert templates in ("classic", "all"), templates quantiles = quantiles or SYNTHETIC_QUANTILES - add("sum_by_job", "job/value", "sum", "1s", 1, 1, J, promql=f"sum by (job) ({gauge})") + add("sum_by_job", "job/value", "sum", "1s", scrape, 1, J, promql=f"sum by (job) ({gauge})") for q in quantiles: - add(f"quantile_by_job_p{q}", "job/dist", "quantile", "1s", 1, 1, J, + add(f"quantile_by_job_p{q}", "job/dist", "quantile", "1s", scrape, 1, J, promql=f"quantile by (job) ({q}, {gauge})") for rng, s in ranges: lbl = label(rng) diff --git a/scripts/test_export_autosketch_eval_table.py b/scripts/test_export_autosketch_eval_table.py index 91651ab8..98f4250e 100644 --- a/scripts/test_export_autosketch_eval_table.py +++ b/scripts/test_export_autosketch_eval_table.py @@ -184,8 +184,10 @@ def test_classic_templates_give_42_distinct_rqes(self): def test_items_per_instance_follow_the_data_model(self): qs = {(q["query_id"], q["range"]): q for q in synthetic_queries(templates="classic")} - # 1e4 series x 200 samples/s = 2e6 samples/s. - self.assertEqual(qs[("quantile_by_job_p0.5", "1s")]["max_N"], 2e5) + # 1e4 series x 200 samples/s = 2e6 samples/s; an instant query reads + # one scrape, one sample from each of a job's 1e3 series. + q = qs[("quantile_by_job_p0.5", "1s")] + self.assertEqual((q["range_s"], q["max_N"]), (0.005, 1e3)) self.assertEqual(qs[("quantile_over_time_p0.5", "15m")]["max_N"], 200 * 900) top = qs[("topk32_sum_by_label0_sum_over_time", "24h")] self.assertEqual((top["K"], top["groups"], top["max_N"]), (10_000, 1, 2e6 * 86400)) @@ -284,8 +286,8 @@ class SchemaTemplatesTest(unittest.TestCase): def test_classic_tables_are_main_s_all_tables_byte_for_byte(self): # The committed tables are the 10-template set's, from before "all" # grew: renamed, today's "classic" tables match them byte for byte - # without templates 7 and 9's RQEs, apart from the `e2e` block, which - # the runner does not read. + # without templates 7 and 9's RQEs and with instant RQEs reading one + # scrape, apart from the `e2e` block, which the runner does not read. with tempfile.TemporaryDirectory() as out: main(["--synthetic", "--revision", "test", "--out", out]) classic = [n for n in sorted(os.listdir(out)) if "templatesclassic" in n] @@ -298,6 +300,9 @@ def test_classic_tables_are_main_s_all_tables_byte_for_byte(self): committed = json.load(f) rqes = committed["workloads"][0]["rqes"] rqes[:] = [r for r in rqes if not r["query_id"].startswith(DROPPED)] + for r in rqes: # instant RQEs now read one scrape (5 ms) + if r["range"] == "1s": + r.update(lookback_secs=0.005, min_N=1e3, max_N=1e3) del ours["sketch_bench_revision"], committed["sketch_bench_revision"] del ours["workloads"][0]["e2e"] self.assertEqual(json.dumps(ours, sort_keys=True), From 813204b56ba5edded366f324f3efa7471517366d Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Wed, 14 Oct 2026 10:00:17 -0400 Subject: [PATCH 2/3] eval: reject samples/s rates that don't divide 1000 An instant RQE's lookback is one scrape, 1000 / rate ms, and the runner keeps lookbacks and scrape intervals in whole ms. A rate that doesn't divide 1000 (3/s gives 333 ms) truncates, and every synthetic RQE fails validation; above 1000/s the scrape is 0 ms, and 0 divides by zero. The exporter rejects such a --samples-per-sec, and the runner such a samples_per_sec_per_series. Co-Authored-By: Claude Opus 5.5 --- rqe-optimizer/examples/autosketch_vs_asap.rs | 8 +++++++- scripts/export_autosketch_eval_table.py | 5 +++++ scripts/test_export_autosketch_eval_table.py | 6 ++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/rqe-optimizer/examples/autosketch_vs_asap.rs b/rqe-optimizer/examples/autosketch_vs_asap.rs index fe7cf115..981d3ab4 100644 --- a/rqe-optimizer/examples/autosketch_vs_asap.rs +++ b/rqe-optimizer/examples/autosketch_vs_asap.rs @@ -402,7 +402,13 @@ fn from_table( // sample per series. Record metrics (http, flows) keep SCRAPE_MS. let scrape_ms = workload["e2e"]["metrics"][metric]["samples_per_sec_per_series"] .as_u64() - .map_or(SCRAPE_MS, |per_sec| 1_000 / per_sec); + .map_or(SCRAPE_MS, |per_sec| { + assert!( + per_sec > 0 && 1_000 % per_sec == 0, + "{metric}: samples_per_sec_per_series {per_sec} must divide 1000" + ); + 1_000 / per_sec + }); let groups = groups_of(r, capability); // A schema metric's RQE names its grouping (its `groups` are that // grouping's); otherwise the stream has one label `g`, the grouping. diff --git a/scripts/export_autosketch_eval_table.py b/scripts/export_autosketch_eval_table.py index 23896e60..a2353a06 100644 --- a/scripts/export_autosketch_eval_table.py +++ b/scripts/export_autosketch_eval_table.py @@ -835,6 +835,11 @@ def main(argv=None): "with it, and schema RQEs below the curves' first N are left out") parser.add_argument("--out", required=True) args = parser.parse_args(argv) + # An instant query reads one scrape, 1000 / rate ms, which the runner keeps + # in whole ms. + rate = args.samples_per_sec + if rate is not None and (rate <= 0 or 1000 % rate): + parser.error("--samples-per-sec must divide 1000") if args.samples_per_sec is not None: default = SYNTHETIC_SAMPLES_PER_SEC set_samples_per_sec(args.samples_per_sec, True) diff --git a/scripts/test_export_autosketch_eval_table.py b/scripts/test_export_autosketch_eval_table.py index 98f4250e..9b0126ca 100644 --- a/scripts/test_export_autosketch_eval_table.py +++ b/scripts/test_export_autosketch_eval_table.py @@ -458,6 +458,12 @@ def test_samples_per_sec_scales_rates_and_leaves_out_rqes_below_the_curves(self) # The default rate is restored for the rest of the process. self.assertEqual(SCHEMAS["http"]["rate"], 2_000_000) + def test_samples_per_sec_must_divide_1000(self): + # An instant RQE's one-scrape lookback, 1000 / rate ms, must be whole. + for rate in ["3", "0", "2000"]: + with self.assertRaises(SystemExit), mock.patch("sys.stderr"): + main(["--synthetic", "--samples-per-sec", rate, "--out", "unused"]) + class SyntheticPlanTest(unittest.TestCase): def test_plan_is_the_default_and_each_dimension_alone(self): From 460e290ee9e278a96f7e36bf11dbe94da8bb9c8c Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Wed, 14 Oct 2026 11:43:43 -0400 Subject: [PATCH 3/3] eval: fail loudly when an RQE's lookback isn't whole scrapes A metric missing from the e2e block falls back to a 1 s scrape, so a 5 ms instant lookback had no candidate window and was dropped as unservable, with only the dropped count logged. The runner now asserts that each RQE's lookback and interval are multiples of its stream's scrape, and rounds them to ms instead of truncating. The e2e_block docstring and the byte-for-byte test no longer claim the runner ignores the e2e block; the test checks every RQE metric's samples_per_sec_per_series before dropping the block. Co-Authored-By: Claude Opus 5.5 --- rqe-optimizer/examples/autosketch_vs_asap.rs | 13 +++++++++++-- scripts/export_autosketch_eval_table.py | 7 ++++--- scripts/test_export_autosketch_eval_table.py | 8 ++++++-- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/rqe-optimizer/examples/autosketch_vs_asap.rs b/rqe-optimizer/examples/autosketch_vs_asap.rs index 981d3ab4..446bd39e 100644 --- a/rqe-optimizer/examples/autosketch_vs_asap.rs +++ b/rqe-optimizer/examples/autosketch_vs_asap.rs @@ -513,11 +513,20 @@ fn from_table( .or_insert(shape); } } + let lookback_ms = (r["lookback_secs"].as_f64().unwrap() * 1000.0).round() as Millis; + let interval_ms = (r["interval_secs"].as_f64().unwrap() * 1000.0).round() as Millis; + // Otherwise no candidate window fits, and the RQE would be dropped as + // unservable rather than reported as a table error. + assert!( + lookback_ms.is_multiple_of(scrape_ms) && interval_ms.is_multiple_of(scrape_ms), + "{id}: lookback {lookback_ms} ms and interval {interval_ms} ms must be \ + multiples of {stream}'s {scrape_ms} ms scrape" + ); raqes.push(Raqe { id: id.clone(), capability, - lookback_ms: (r["lookback_secs"].as_f64().unwrap() * 1000.0) as Millis, - interval_ms: (r["interval_secs"].as_f64().unwrap() * 1000.0) as Millis, + lookback_ms, + interval_ms, metric: stream, spatial_filter: String::new(), grouping_labels: grouping, diff --git a/scripts/export_autosketch_eval_table.py b/scripts/export_autosketch_eval_table.py index a2353a06..18b2ab1c 100644 --- a/scripts/export_autosketch_eval_table.py +++ b/scripts/export_autosketch_eval_table.py @@ -693,9 +693,10 @@ def synthetic_entry(q): def e2e_block(qs, queries): - """What an end-to-end run of a table needs and the runner does not read: - the `queries` (one entry per distinct query and interval, with the RQEs - it reads; a binary query reads two) and the `metrics` to generate. + """What an end-to-end run of a table needs: the `queries` (one entry per + distinct query and interval, with the RQEs it reads; a binary query reads + two) and the `metrics` to generate. The runner reads only a metric's + `samples_per_sec_per_series`, as its scrape rate. Classic queries are PromQL; each classic metric is a gauge (`value` and `dist` streams) and a counter `_total` (`increment` streams), with labels in SCHEMAS' format: `label_0` has SYNTHETIC_SERIES / diff --git a/scripts/test_export_autosketch_eval_table.py b/scripts/test_export_autosketch_eval_table.py index 9b0126ca..872ec9f0 100644 --- a/scripts/test_export_autosketch_eval_table.py +++ b/scripts/test_export_autosketch_eval_table.py @@ -287,7 +287,8 @@ def test_classic_tables_are_main_s_all_tables_byte_for_byte(self): # The committed tables are the 10-template set's, from before "all" # grew: renamed, today's "classic" tables match them byte for byte # without templates 7 and 9's RQEs and with instant RQEs reading one - # scrape, apart from the `e2e` block, which the runner does not read. + # scrape, apart from the `e2e` block. Of that block the runner reads + # only each RQE metric's samples_per_sec_per_series, its scrape rate. with tempfile.TemporaryDirectory() as out: main(["--synthetic", "--revision", "test", "--out", out]) classic = [n for n in sorted(os.listdir(out)) if "templatesclassic" in n] @@ -304,7 +305,10 @@ def test_classic_tables_are_main_s_all_tables_byte_for_byte(self): if r["range"] == "1s": r.update(lookback_secs=0.005, min_N=1e3, max_N=1e3) del ours["sketch_bench_revision"], committed["sketch_bench_revision"] - del ours["workloads"][0]["e2e"] + workload = ours["workloads"][0] + metrics = workload.pop("e2e")["metrics"] + for metric in {r["metric"] for r in workload["rqes"]}: + self.assertEqual(metrics[metric]["samples_per_sec_per_series"], 200, name) self.assertEqual(json.dumps(ours, sort_keys=True), json.dumps(committed, sort_keys=True), name)