Skip to content

Commit 00195cb

Browse files
committed
test: execute level-two acceptance with the shared Rust runner
1 parent cc51f60 commit 00195cb

35 files changed

Lines changed: 3292 additions & 8 deletions
Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
name: PromQL compliance
2+
3+
on:
4+
pull_request:
5+
types: [opened, synchronize, reopened, ready_for_review]
6+
workflow_dispatch:
7+
8+
permissions:
9+
contents: read
10+
11+
jobs:
12+
differential:
13+
runs-on: ubuntu-latest
14+
timeout-minutes: 90
15+
steps:
16+
- uses: actions/checkout@v4
17+
with:
18+
path: ASAPQuery-backend
19+
- uses: actions/checkout@v4
20+
with:
21+
repository: ProjectASAP/ASAPCollector
22+
path: ASAPCollector
23+
- uses: actions/checkout@v4
24+
with:
25+
repository: ProjectASAP/asap_sketchlib
26+
path: asap_sketchlib
27+
- uses: dtolnay/rust-toolchain@stable
28+
- name: Install protobuf compiler
29+
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
30+
- name: Test Rust compliance harness
31+
working-directory: ASAPQuery-backend
32+
run: cargo test --locked -p promql-compliance
33+
- name: Run every differential corpus
34+
working-directory: ASAPQuery-backend/promql-compliance/runner
35+
run: make run-all REPORT_DIR="$GITHUB_WORKSPACE/artifacts/reports" LOGS_DIR="$GITHUB_WORKSPACE/artifacts/logs"
36+
- name: Upload reports and service logs
37+
if: always()
38+
uses: actions/upload-artifact@v4
39+
with:
40+
name: promql-compliance-evidence
41+
path: artifacts
42+
if-no-files-found: warn

‎Cargo.lock‎

Lines changed: 20 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ members = [
55
"crates/asap_types",
66
"data_plane",
77
"control_plane",
8+
"promql-compliance/runner",
89
]
910

1011
[workspace.package]

‎control_plane/Dockerfile‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,7 @@ RUN apt-get update && \
1515
rm -rf /var/lib/apt/lists/*
1616

1717
COPY . ASAPQuery-backend
18-
19-
RUN cd ASAPQuery-backend && cargo build --locked --release --bin control_plane
18+
RUN cd ASAPQuery-backend && cargo build --locked --release --bin control_plane --bin control_plane_quote_snapshot --example compile_workload_artifact
2019

2120
# Runtime image: binary + CA certs.
2221
FROM debian:bookworm-slim
@@ -25,6 +24,10 @@ RUN apt-get update && \
2524
rm -rf /var/lib/apt/lists/*
2625
COPY --from=build /src/ASAPQuery-backend/target/release/control_plane \
2726
/usr/local/bin/control_plane
27+
COPY --from=build /src/ASAPQuery-backend/target/release/control_plane_quote_snapshot \
28+
/usr/local/bin/control_plane_quote_snapshot
29+
COPY --from=build /src/ASAPQuery-backend/target/release/examples/compile_workload_artifact \
30+
/usr/local/bin/compile_workload_artifact
2831

2932
ENV RUST_LOG=info
3033
# OpAMP ws 4320, controller gRPC 4321, controller HTTP 8080.
Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
//! Complete a backend-local planning snapshot with deterministic unit-cost
2+
//! quotes for the PromQL compliance harness. Candidate identities and demand
3+
//! still come from the production planner and physical compiler.
4+
5+
use control_plane::physical::{
6+
compiler::{
7+
BackendLocalPlanningInput, PhysicalPlanCompiler, BACKEND_REVISION, PLANNER_REVISION,
8+
},
9+
workload_cost::{
10+
enumerate_exact_and_materialized_candidates, manifest, WorkloadCostEvidence, WorkloadQuote,
11+
},
12+
};
13+
use std::{collections::BTreeMap, path::Path};
14+
15+
fn quote_snapshot(
16+
mut snapshot: BackendLocalPlanningInput,
17+
) -> Result<BackendLocalPlanningInput, String> {
18+
if snapshot.workload_cost_evidence.is_some() {
19+
return Err("planning snapshot already contains workload_cost_evidence".into());
20+
}
21+
let observed_at_unix_ms = snapshot.environment.observed_at_unix_ms;
22+
let valid_for_ms = snapshot.environment.max_evidence_age_ms;
23+
let (request, environment) = snapshot
24+
.clone()
25+
.into_physical_compilation_request()
26+
.map_err(|error| error.to_string())?;
27+
let quotes = enumerate_exact_and_materialized_candidates(request)
28+
.map_err(|error| error.to_string())?
29+
.into_iter()
30+
.filter_map(|candidate| {
31+
let plan = PhysicalPlanCompiler
32+
.compile_promql(candidate.clone(), environment.clone())
33+
.ok()?;
34+
// This execution suite must exercise maintained candidates when legal.
35+
let unit_cost = if plan.precompute_plan.materializations.is_empty() {
36+
1e12
37+
} else {
38+
1.0
39+
};
40+
let manifest = manifest(&plan, &candidate.queries).ok()?;
41+
Some(WorkloadQuote {
42+
unit_costs: manifest
43+
.components
44+
.keys()
45+
.map(|key| (key.clone(), unit_cost))
46+
.collect::<BTreeMap<_, _>>(),
47+
manifest,
48+
executable: true,
49+
})
50+
})
51+
.collect();
52+
snapshot.workload_cost_evidence = Some(WorkloadCostEvidence {
53+
backend_revision: BACKEND_REVISION.into(),
54+
planner_revision: PLANNER_REVISION.into(),
55+
data_snapshot_id: "promql-compliance".into(),
56+
model_version: "promql-compliance-unit-costs".into(),
57+
observed_at_unix_ms,
58+
valid_for_ms,
59+
quotes,
60+
});
61+
Ok(snapshot)
62+
}
63+
64+
fn main() -> Result<(), String> {
65+
let mut args = std::env::args_os().skip(1);
66+
let input = args
67+
.next()
68+
.ok_or("usage: control_plane_quote_snapshot INPUT OUTPUT")?;
69+
let output = args
70+
.next()
71+
.ok_or("usage: control_plane_quote_snapshot INPUT OUTPUT")?;
72+
if args.next().is_some() {
73+
return Err("usage: control_plane_quote_snapshot INPUT OUTPUT".into());
74+
}
75+
let snapshot = std::fs::read(&input)
76+
.map_err(|error| format!("read {}: {error}", Path::new(&input).display()))?;
77+
let snapshot =
78+
serde_json::from_slice(&snapshot).map_err(|error| format!("decode snapshot: {error}"))?;
79+
let quoted = quote_snapshot(snapshot)?;
80+
std::fs::write(
81+
&output,
82+
serde_json::to_vec_pretty(&quoted).map_err(|error| error.to_string())?,
83+
)
84+
.map_err(|error| format!("write {}: {error}", Path::new(&output).display()))?;
85+
Ok(())
86+
}
87+
88+
#[cfg(test)]
89+
mod tests {
90+
use super::*;
91+
92+
#[test]
93+
fn adds_complete_candidate_quotes_to_a_snapshot() {
94+
let fixture =
95+
include_str!("../../../docs/examples/asapquery-compatibility-demo-snapshot.json");
96+
let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(fixture).unwrap();
97+
snapshot.workload_cost_evidence = None;
98+
let quoted = quote_snapshot(snapshot).unwrap();
99+
assert!(!quoted
100+
.workload_cost_evidence
101+
.as_ref()
102+
.unwrap()
103+
.quotes
104+
.is_empty());
105+
let plan = quoted.compile_promql().unwrap();
106+
assert!(
107+
!plan.precompute_plan.materializations.is_empty(),
108+
"deterministic compliance quotes should select a maintained candidate"
109+
);
110+
}
111+
}

‎data_plane/Dockerfile‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ RUN apt-get update && \
2020
rm -rf /var/lib/apt/lists/*
2121

2222
COPY . ASAPQuery-backend
23-
2423
RUN cd ASAPQuery-backend && cargo build --locked --release --bin data_plane
2524

2625
# Runtime image: binary + CA certs.

‎data_plane/src/main.rs‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -461,11 +461,6 @@ fn validate_profile(args: &Args) -> Result<()> {
461461
)
462462
.into());
463463
}
464-
if !args.forward_unsupported_queries {
465-
return Err(
466-
"--profile asapquery requires --forward-unsupported-queries for exact fallback".into(),
467-
);
468-
}
469464
let required_horizon = (args.precompute_allowed_lateness_ms.max(0) as u64)
470465
.saturating_add(args.remote_write_expected_retry_interval_ms);
471466
if args.remote_write_dedup_horizon_ms < required_horizon {
@@ -1391,6 +1386,19 @@ mod tests {
13911386
.unwrap();
13921387
assert!(validate_profile(&valid).is_ok());
13931388

1389+
let no_fallback = Args::try_parse_from([
1390+
"data_plane",
1391+
"--profile",
1392+
"asapquery",
1393+
"--physical-plan",
1394+
"plan.json",
1395+
])
1396+
.unwrap();
1397+
assert!(
1398+
validate_profile(&no_fallback).is_ok(),
1399+
"a backend-local plan must be allowed to reject unsupported queries"
1400+
);
1401+
13941402
assert!(
13951403
Args::try_parse_from(["data_plane", "--streaming-config", "streaming.yaml"]).is_err()
13961404
);
Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
# PromQL compliance design Q&A
2+
3+
> **Question:** should v1 be a strict differential suite for a deliberately supported, locally answered subset of PromQL—with unsupported queries required to fail—or should it allow Prometheus fallback?
4+
>
5+
> **Recommendation:** require local answers and reject fallback in v1. Otherwise Prometheus can answer on the backend’s behalf, yielding a green comparison that proves neither backend ingestion nor backend query semantics.
6+
7+
**Answer:** unsupported queries required to fail, yes
8+
9+
> **Question:** should the suite compare approximate sketch-backed values to Prometheus within an explicit per-query tolerance, or start with only result shapes/data sizes where equality is exact?
10+
>
11+
> **Recommendation:** support explicit per-query tolerances from day one, defaulting to exact equality. The backend is designed to return approximate sketch results, so pretending all valid local answers are exact would either constrain the suite to an unrepresentative corpus or create noisy failures.
12+
13+
**Answer:** yes
14+
15+
> **Question:** should v1 use a finite, timestamp-pinned fixture and explicitly call the backend’s `/api/v1/precompute/drain` before querying?
16+
>
17+
> **Recommendation:** yes. It makes ingestion completion deterministic and avoids flaky “did the asynchronous precompute path catch up?” failures. Live-scrape/streaming behavior should be a separate suite later, because `drain` deliberately closes the backend’s Remote Write input.
18+
19+
**Answer:** yes
20+
21+
> **Question:** before the backend can accept Remote Write, it needs a configuration called a PhysicalPlan that says what metrics/queries it should accelerate. Should this new test suite: (1) give the backend a small, fixed test configuration; or (2) start the planner/control plane too and have it create that configuration?
22+
>
23+
> **Recommendation:** option 1 for this issue. It keeps failures clear: if the test fails, it is likely ingestion or query behavior in the backend, not plan generation.
24+
25+
**Answer:** i wanna do option 2 actually. we can add option 1 later
26+
27+
> **Question:** should “planner/control plane” mean the backend repository’s `control_plane` binary and its normal plan-publication protocol, rather than importing ASAPQuery’s older `asap-planner-rs` service?
28+
>
29+
> **Recommendation:** use the backend’s own `control_plane`. It tests the deployment users of `ASAPQuery-backend` actually run and avoids coupling this repository’s suite to a separate repository’s planner implementation.
30+
31+
**Answer:** yes
32+
33+
> **Question:** how should the fixture seeder, comparator and Docker Compose runner integrate with the Rust backend?
34+
>
35+
> **Decision:** implement the runner in the Cargo workspace. Shared Rust modules provide deterministic Remote Write encoding, structured comparison, fixtures, tolerances, reports and Compose lifecycle handling. No Go toolchain is required.
36+
37+
**Answer:** yes
38+
39+
> **Question:** what should be the first required green corpus?
40+
>
41+
> **Recommendation:** begin with one small fixture and suite covering a selector, `rate`, `increase`, one aggregation, and both instant/range evaluations—plus one intentionally unsupported query that must fail locally. Add sparse/staleness, counter resets, and larger aggregation matrices after this vertical slice is stable.
42+
43+
**Answer:** Let's just add all corpora. I think we may not get any green righ tnow
44+
45+
> **Question:** when those corpora find differences, should the runner exit non-zero from day one?
46+
>
47+
> **Recommendation:** yes—never encode known failures as passes. Keep the full workflow manual/non-required initially, preserve its JSON reports as evidence, and only make a stable subset required in PR CI after the gaps are fixed.
48+
49+
**Answer:** yes
50+
51+
> **Question:** should every successful backend response be required to carry and pass a local-execution provenance check (for example, its existing ASAPQuery data-source marker), in addition to disabling fallback?
52+
>
53+
> **Recommendation:** yes. Disabling fallback catches most masking, but provenance makes the test’s claim explicit and will catch accidental routing changes that still return a successful response.
54+
55+
**Answer:** no, it's fine. just disable fallback
56+
57+
> **Question:** should the runner wait until the control plane has published and the backend reports an active Remote Write-ready plan before it sends any fixture data?
58+
>
59+
> **Recommendation:** yes. Then seed both targets, call the backend’s finite-input `drain`, and only then execute the fixed-time queries. This makes plan activation and ingestion completion explicit rather than timing-dependent.
60+
61+
**Answer:** yes
62+
63+
> **Question:** should the runner derive the control-plane workload/configuration from the same query-suite YAML it executes, rather than maintain a second hand-written plan configuration per corpus?
64+
>
65+
> **Recommendation:** yes. One source of truth prevents a test from querying expressions that the control plane was never asked to plan, and it makes adding a corpus a fixture-only change.
66+
67+
**Answer:** yes
68+
69+
> **Question:** on failure, should the runner retain the JSON report and collect service logs, while still tearing down containers by default?
70+
>
71+
> **Recommendation:** yes. CI should upload the report and logs as artifacts; local runs should offer `--keep-services` for interactive debugging. Default cleanup prevents stale volumes/ports from contaminating the next run.
72+
73+
**Answer:** yes
74+
75+
> **Question:** should `ASAPQuery-backend` own a copied/adapted version of the harness and fixtures, rather than invoke `ASAPQuery/promql-compliance` across repositories?
76+
>
77+
> **Recommendation:** own it in the backend repository. The backend needs different service wiring—its `control_plane`, PhysicalPlan lifecycle, and one HTTP listener—and an external cross-repo dependency would make local and CI runs less reproducible.
78+
79+
**Answer:** yes

0 commit comments

Comments
 (0)