Evaluate the synthetic PromQL workload of ASAPQuery#777

docs/rqe_sketch_deployment_v1.md

@@ -228,9 +228,14 @@ lookback: series per group times scrapes per lookback.
   accuracy. Past the last, the plateau if the point saturated, else none.
 - The curve's `error_metric` must be the family's metric; `SaturationCurves::load`
   refuses a curve for another metric as a stale study.
-- The window size doesn't matter: a merged answer reads the curve at the
-  lookback's item count, like a single sketch. KLL and top-k merge lossily;
-  their merge penalty is #158.
+- For sketches that merge exactly, the window size doesn't matter: a merged
+  answer reads the curve at the lookback's item count, like a single sketch.
+  KLL and top-k merge lossily (#131): an answer merged from `m = S_i / D.x`
+  windows reads the study's merge curve (`saturation_merge_curve.csv`, the
+  sketch merged from `m` shards of the same items) at the lookback's item
+  count. Between measured shard counts, the worse; past the largest, the
+  largest. Without a merge curve, a merged KLL or top-k answer has no
+  accuracy (#158).
 
 The cost table's sketch accuracies are not read, but they are one point on
 each curve: a row's `measured_at` (items, Zipf θ and population, or Pareto
@@ -473,8 +478,9 @@ each phase's CPU and memory, together with the selected deployment mapping.
 
 ## v1 scope and TODOs
 
-- **Accuracy after merging:** every merge reads the curve at the merged item
-  count. KLL and top-k don't model their merge penalty yet (#158).
+- **Accuracy after merging:** KLL and top-k read merge curves up to the
+  largest measured shard count (64) and N = 1e7; past those, the largest
+  count, and the plateau if the plain curve saturated (#158).
 - **Query-result sharing:** v1 charges every RAQE its own query and merge CPU.
   Revisit when RAQE semantics and execution timing identify safe reuse cases.
 - **Latency SLAs:** the MILP takes optional per-RAQE latency bounds; the

scripts/plot_autosketch_vs_asap.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""Plot the AutoSketch vs. ASAP evaluation on the `traces` workloads
+(ProjectASAP/ASAPQuery#777) from the JSON files of
+`rqe-optimizer/examples/autosketch_vs_asap.rs`.
+
+Writes fig1_objective.png, fig4_objective_vs_sla.png and summary.md next to
+the inputs.
+
+Usage: scripts/plot_autosketch_vs_asap.py DIR
+"""
+
+import json
+import pathlib
+import sys
+
+import matplotlib
+
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt  # noqa: E402
+
+METHODS = ["asap", "perquery", "autosketch"]
+LABELS = {"asap": "ASAP", "perquery": "PerQuery-CostAware", "autosketch": "AutoSketch-Adapted"}
+COLORS = {"asap": "#1f77b4", "perquery": "#ff7f0e", "autosketch": "#2ca02c"}
+WEIGHT_UNITS = {"cpu": "vCPU", "fargate": "$/hour"}
+
+
+def sla_text(sla):
+    return "no SLA" if sla == "inf" else f"{sla:g} ms"
+
+
+def load(directory):
+    runs = {}
+    for path in sorted(pathlib.Path(directory).glob("traces-*.json")):
+        data = json.loads(path.read_text())
+        runs[data["workload"].split("/")[-1]] = data
+    return runs
+
+
+def pick(run, method, weights, sla):
+    for r in run["results"]:
+        if r["method"] == method and r["weights"] == weights and str(r["sla_ms"]) == str(sla):
+            return r
+    return None
+
+
+def objective(r):
+    return None if r is None or "objective" not in r else r["objective"]
+
+
+def fig_objective(runs, out):
+    """Objective per dataset and method, no SLA, one panel per weight setting."""
+    weights = [w["name"] for w in next(iter(runs.values()))["weights"]]
+    fig, axes = plt.subplots(1, len(weights), figsize=(5 * len(weights), 3.5), squeeze=False)
+    for ax, wname in zip(axes[0], weights):
+        names = list(runs)
+        for i, method in enumerate(METHODS):
+            ys = [objective(pick(runs[n], method, wname, "inf")) or 0 for n in names]
+            ax.bar([x + i * 0.27 for x in range(len(names))], ys, 0.27,
+                   label=LABELS[method], color=COLORS[method])
+        ax.set_xticks([x + 0.27 for x in range(len(names))], names, fontsize=8)
+        ax.set_yscale("log")
+        ax.set_ylabel(f"objective ({WEIGHT_UNITS.get(wname, wname)})")
+        ax.set_title(f"weights: {wname}", fontsize=9)
+    axes[0][0].legend(fontsize=7)
+    fig.tight_layout()
+    fig.savefig(out / "fig1_objective.png", dpi=150)
+    plt.close(fig)
+
+
+def fig_objective_vs_sla(runs, out):
+    """Objective vs. the latency SLA, CPU-only weights, one panel per dataset."""
+    fig, axes = plt.subplots(1, len(runs), figsize=(4 * len(runs), 3.2), squeeze=False)
+    for ax, (name, run) in zip(axes[0], runs.items()):
+        slas = run["sla_grid_ms"]
+        xs = list(range(len(slas)))
+        for method in METHODS:
+            points = [(x, objective(pick(run, method, "cpu", s))) for x, s in zip(xs, slas)]
+            points = [(x, y) for x, y in points if y is not None]
+            if points:
+                ax.plot(*zip(*points), marker="o", label=LABELS[method], color=COLORS[method])
+        ax.set_xticks(xs, [sla_text(s) for s in slas], fontsize=7)
+        ax.set_yscale("log")
+        ax.set_title(name, fontsize=9)
+        ax.set_ylabel("CPU (vCPU)")
+    axes[0][0].legend(fontsize=7)
+    fig.tight_layout()
+    fig.savefig(out / "fig4_objective_vs_sla.png", dpi=150)
+    plt.close(fig)
+
+
+def fmt(x, digits=3):
+    return "—" if x is None else f"{x:.{digits}g}"
+
+
+def summary(runs, out):
+    lines = ["# AutoSketch vs. ASAP on the trace workloads\n"]
+    for name, run in runs.items():
+        lines.append(f"## {name}\n")
+        lines.append(f"{run['rqes']} RQEs; dropped as unservable: "
+                     f"{len(set(run['dropped_unservable']['asap']) | set(run['dropped_unservable']['autosketch']))}. "
+                     f"Sanity violations: {len(run['sanity_violations'])}.\n")
+        for w in run["weights"]:
+            for sla in run["sla_grid_ms"]:
+                rows = [pick(run, m, w["name"], sla) for m in METHODS]
+                if not any(rows):
+                    continue
+                base = objective(rows[0])
+                lines.append(f"Weights {w['name']}, {sla_text(sla)}:\n")
+                lines.append("| method | objective | vs ASAP | CPU (vCPU) | GiB | deployments | "
+                             "max latency (ms) | planning (s) |")
+                lines.append("|---|---|---|---|---|---|---|---|")
+                for method, r in zip(METHODS, rows):
+                    if r is None or "error" in r:
+                        lines.append(f"| {LABELS[method]} | {r and r.get('error')} |||||||")
+                        continue
+                    ratio = r["objective"] / base if base else None
+                    lines.append(
+                        f"| {LABELS[method]} | {fmt(r['objective'])} | {fmt(ratio)}× | "
+                        f"{fmt(r['cpu'])} | {fmt(r['gib'])} | {r['active_deployments']} | "
+                        f"{fmt(r['max_latency_ms'])} | {fmt(r['planning_secs'])} |")
+                lines.append("")
+    (out / "summary.md").write_text("\n".join(lines))
+
+
+def main():
+    directory = pathlib.Path(sys.argv[1])
+    runs = load(directory)
+    fig_objective(runs, directory)
+    fig_objective_vs_sla(runs, directory)
+    summary(runs, directory)
+
+
+if __name__ == "__main__":
+    main()

scripts/plot_autosketch_vs_asap_synthetic.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python3
+"""Plot the synthetic AutoSketch vs. ASAP evaluation (ProjectASAP/ASAPQuery#777,
+section 7) from the JSON files of `autosketch_vs_asap synthetic`.
+
+Writes, next to the inputs:
+  fig_objective_vs_latency.png   default workload: objective vs. achieved max
+                                 latency over the SLA grid, one panel per weights
+  fig_objective_by_dimension.png objective with no SLA, each grid dimension
+                                 varied alone (templates, shared replicas,
+                                 strictness)
+  fig_planning_time.png          planning time vs. RQEs (shared replicas)
+  summary.md                     every workload, method, weights and SLA
+
+Usage: scripts/plot_autosketch_vs_asap_synthetic.py DIR
+"""
+
+import json
+import pathlib
+import re
+import sys
+
+import matplotlib
+
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt  # noqa: E402
+
+METHODS = ["asap", "perquery", "autosketch"]
+LABELS = {"asap": "ASAP", "perquery": "PerQuery-CostAware", "autosketch": "AutoSketch-Adapted"}
+COLORS = {"asap": "#1f77b4", "perquery": "#ff7f0e", "autosketch": "#2ca02c"}
+UNITS = {"cpu": "vCPU", "fargate": "$/hour"}
+DEFAULT = ("dashboard", 1, "default")
+
+
+def point(data):
+    """(templates, shared, target) of a result's workload name."""
+    m = re.search(r"templates=(\w+)/shared=(\d+)/t(\w+)", data["workload"])
+    return m.group(1), int(m.group(2)), m.group(3)
+
+
+def load(directory):
+    return {point(d): d for d in (json.loads(p.read_text())
+                                  for p in sorted(pathlib.Path(directory).glob("synthetic-*.json")))}
+
+
+def pick(data, method, weights, sla):
+    for r in data["results"]:
+        if r["method"] == method and r["weights"] == weights and str(r["sla_ms"]) == str(sla):
+            return r if "objective" in r else None
+    return None
+
+
+def weight_names(runs):
+    return [w["name"] for w in next(iter(runs.values()))["weights"]]
+
+
+def fig_objective_vs_latency(runs, out):
+    data = runs.get(DEFAULT)
+    if data is None:
+        return
+    names = weight_names(runs)
+    fig, axes = plt.subplots(1, len(names), figsize=(5 * len(names), 3.6), squeeze=False)
+    for ax, w in zip(axes[0], names):
+        for m in METHODS:
+            pts = [(r["max_latency_ms"], r["objective"]) for s in data["sla_grid_ms"]
+                   if (r := pick(data, m, w, s))]
+            if pts:
+                ax.plot(*zip(*sorted(pts)), marker="o", label=LABELS[m], color=COLORS[m])
+        ax.set_xscale("log")
+        ax.set_yscale("log")
+        ax.set_xlabel("max estimated latency (ms)")
+        ax.set_ylabel(f"objective ({UNITS.get(w, w)})")
+        ax.set_title(f"dashboard, default strictness, weights {w}", fontsize=9)
+    axes[0][0].legend(fontsize=7)
+    fig.tight_layout()
+    fig.savefig(out / "fig_objective_vs_latency.png", dpi=150)
+    plt.close(fig)
+
+
+def fig_objective_by_dimension(runs, out):
+    dims = [("templates", 0, ["dashboard", "all"]), ("shared", 1, [1, 8, 64]),
+            ("strictness", 2, ["loose", "default", "strict"])]
+    fig, axes = plt.subplots(1, len(dims), figsize=(4.5 * len(dims), 3.4), squeeze=False)
+    for ax, (name, index, values) in zip(axes[0], dims):
+        xs = list(range(len(values)))
+        for i, m in enumerate(METHODS):
+            ys = []
+            for v in values:
+                key = list(DEFAULT)
+                key[index] = v
+                data = runs.get(tuple(key))
+                r = data and pick(data, m, "cpu", "inf")
+                ys.append(r["objective"] if r else 0)
+            ax.bar([x + i * 0.27 for x in xs], ys, 0.27, label=LABELS[m], color=COLORS[m])
+        ax.set_xticks([x + 0.27 for x in xs], [str(v) for v in values])
+        ax.set_yscale("log")
+        ax.set_title(name, fontsize=9)
+        ax.set_ylabel("CPU (vCPU), no SLA")
+    axes[0][0].legend(fontsize=7)
+    fig.tight_layout()
+    fig.savefig(out / "fig_objective_by_dimension.png", dpi=150)
+    plt.close(fig)
+
+
+def fig_planning_time(runs, out):
+    fig, ax = plt.subplots(figsize=(4.5, 3.4))
+    for m in METHODS:
+        pts = []
+        for r in (1, 8, 64):
+            data = runs.get(("dashboard", r, "default"))
+            res = data and pick(data, m, "cpu", "inf")
+            if res:
+                pts.append((data["rqes"], res["planning_secs"]))
+        if pts:
+            ax.plot(*zip(*pts), marker="o", label=LABELS[m], color=COLORS[m])
+    ax.set_xscale("log")
+    ax.set_yscale("log")
+    ax.set_xlabel("RQEs")
+    ax.set_ylabel("planning time (s)")
+    ax.legend(fontsize=7)
+    fig.tight_layout()
+    fig.savefig(out / "fig_planning_time.png", dpi=150)
+    plt.close(fig)
+
+
+def fmt(x, digits=3):
+    return "—" if x is None else f"{x:.{digits}g}"
+
+
+def summary(runs, out):
+    lines = ["# Synthetic workload: AutoSketch vs. ASAP\n"]
+    for key, data in sorted(runs.items(), key=lambda kv: str(kv[0])):
+        lines.append(f"## templates={key[0]}, shared={key[1]}, strictness={key[2]}\n")
+        lines.append(f"{data['rqes']} RQEs on {data['streams']} streams; sanity violations: "
+                     f"{len(data['sanity_violations'])}. AutoSketch probes: "
+                     f"{data['autosketch']['probes']}, benchmark time ≥ "
+                     f"{fmt(data['autosketch']['benchmark_secs_lower_bound_nbench1e8'])} s.\n")
+        lines.append("| weights | SLA (ms) | RQEs | method | objective | vs ASAP | CPU (vCPU) | "
+                     "GiB | deployments | max latency (ms) | planning (s) |")
+        lines.append("|---|---|---|---|---|---|---|---|---|---|---|")
+        for w in weight_names(runs):
+            for sla in data["sla_grid_ms"]:
+                base = pick(data, "asap", w, sla)
+                for m in METHODS:
+                    r = pick(data, m, w, sla)
+                    if r is None:
+                        continue
+                    ratio = r["objective"] / base["objective"] if base and base["objective"] else None
+                    lines.append(
+                        f"| {w} | {sla} | {r['rqes']} | {LABELS[m]} | {fmt(r['objective'])} | "
+                        f"{fmt(ratio)}× | {fmt(r['cpu'])} | {fmt(r['gib'])} | "
+                        f"{r['active_deployments']} | {fmt(r['max_latency_ms'])} | "
+                        f"{fmt(r['planning_secs'])} |")
+        lines.append("")
+    (out / "summary.md").write_text("\n".join(lines))
+
+
+def main():
+    out = pathlib.Path(sys.argv[1])
+    runs = load(out)
+    fig_objective_vs_latency(runs, out)
+    fig_objective_by_dimension(runs, out)
+    fig_planning_time(runs, out)
+    summary(runs, out)
+
+
+if __name__ == "__main__":
+    main()

scripts/run_autosketch_vs_asap.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Run the AutoSketch vs. ASAP evaluation on the `traces` workloads
+# (ProjectASAP/ASAPQuery#777) and plot it.
+#
+# Usage: scripts/run_autosketch_vs_asap.sh SATURATION_DIR
+#
+# `traces` reads rqe-optimizer/data/autosketch-eval/table.json for the
+# workload. SATURATION_DIR holds the saturation curves (out_grid_1e7_cost/,
+# out_1e9/) and the cost table (optimizer_cost/) from
+# scripts/study_saturation.py, the same inputs the planner reads.
+
+set -euo pipefail
+
+SATURATION_DIR=${1:?usage: $0 SATURATION_DIR}
+
+OUT=rqe-optimizer/results/autosketch-vs-asap
+BIN=./target/release/examples/autosketch_vs_asap
+mkdir -p "$OUT"
+
+cargo build --release -p rqe-optimizer --example autosketch_vs_asap
+
+for dataset in alibaba_v2022 google_2011 boom; do
+    "$BIN" traces --dataset "$dataset" --saturation-dir "$SATURATION_DIR" \
+        --out "$OUT/traces-$dataset.json" --runs 5
+done
+
+python3 scripts/plot_autosketch_vs_asap.py "$OUT"