feat(asap-tools): add the p99 rank-error script and results table

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Author: zzylol

asap-tools/experiments/recommended_sketch_configs/README.md

@@ -119,3 +119,10 @@ its MAPE is 13.6% (k=200), 43% (K=20) and 3.7% (K=500).
   recommendations are not run. No query in the sets uses HLL.
 - Memory is the query engine's whole RSS, not per-sketch bytes; sketch-bench
   reports 49 KB (3x4096) and 197 KB (3x16384) per sketch.
+
+## Reproducing
+
+- Running these configs end to end with the local provider needs the runner fixes in ProjectASAP/ASAPQuery#751.
+- `results_summary.csv` is the summary table of the runs above.
+- p99 rank error is computed offline against the replayed trace values:
+  `python kll_rank_error.py --trace <task_usage part-00000-of-00500.csv.gz> --outputs <experiment_outputs>`

asap-tools/experiments/recommended_sketch_configs/kll_rank_error.py

@@ -0,0 +1,73 @@
+"""Rank error of ASAP's p99 estimates against the replayed Google CPU values.
+
+During a run the exporter has exported the part-0 task_usage rows with
+start_time <= --cutoff-us (default: the 5-minute window starting at 600 s)
+and aggregation_type 0; F is their empirical CDF and the rank error of an
+estimate is |F(estimate) - 0.99|.
+"""
+
+import argparse
+import gzip
+import os
+import sys
+
+import numpy as np
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, os.path.join(HERE, ".."))
+sys.path.insert(
+    0, os.path.join(HERE, "../../../asap-common/dependencies/py/promql_utilities")
+)
+from post_experiment.lib.results_loader import load_results  # noqa: E402
+
+
+def replayed_values(trace: str, cutoff_us: int) -> np.ndarray:
+    values = []
+    with gzip.open(trace, "rt") as f:
+        for line in f:
+            c = line.rstrip("\n").split(",")
+            if int(c[0]) <= cutoff_us and c[18] in ("", "0") and c[5] != "":
+                values.append(float(c[5]))
+    return np.sort(values)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--trace", required=True, help="task_usage part-00000-of-00500.csv.gz"
+    )
+    parser.add_argument(
+        "--outputs", required=True, help="experiment_outputs directory of the runs"
+    )
+    parser.add_argument("--cutoff-us", type=int, default=615_000_000)
+    parser.add_argument(
+        "--runs",
+        default="recommended,default,k500",
+        help="suffixes of google_2011_cpu_p99_instant_<run>",
+    )
+    args = parser.parse_args()
+
+    v = replayed_values(args.trace, args.cutoff_us)
+    print(f"{len(v)} values, exact p99 {np.quantile(v, 0.99):.6f}")
+    for name in args.runs.split(","):
+        out = os.path.join(
+            args.outputs,
+            f"google_2011_cpu_p99_instant_{name}",
+            "sketchdb",
+            "prometheus_client_output",
+        )
+        results = load_results(out)
+        est = [
+            list(q.result.values())[0]
+            for q in results["sketchdb"][0].query_results
+            if q.result
+        ]
+        err = np.abs(np.searchsorted(v, est, side="right") / len(v) - 0.99)
+        print(
+            f"{name}: n={len(est)} rank_err median={np.median(err):.4f} "
+            f"mean={np.mean(err):.4f} max={np.max(err):.4f}"
+        )
+
+
+if __name__ == "__main__":
+    main()

asap-tools/experiments/recommended_sketch_configs/results_summary.csv

@@ -0,0 +1,11 @@
+experiment,family,config,predicted_error,target,measured_error,measured_error_all_keys,answered,asap_latency_ms,prom_latency_ms,asap_peak_rss_mb,meets_target
+alibaba_v2022_ms_cpu_by_msname_instant_recommended,cms,rows=3 cols=16384,0.020573849606818375,0.05,0.0012830390988093434,2.2210426168040818,20/20,628.7422180175781,3235.399007797241,316.6208,True
+alibaba_v2022_ms_cpu_by_msname_instant_default,cms,default,0.020573849606818375,0.05,0.1995992177052795,257.7338964738049,20/20,619.9700832366943,3130.3480863571167,274.51392,False
+alibaba_v2022_ms_cpu_by_msname_5m_recommended,cms,rows=3 cols=16384,0.021473512975033845,0.05,0.001300943674761616,2.1992003698696116,20/20,677.9897212982178,4349.95698928833,282.226688,True
+alibaba_v2022_ms_cpu_by_msname_5m_default,cms,default,0.021473512975033845,0.05,0.19694224821114614,250.54328076568058,20/20,640.0543451309204,4493.685245513916,257.98656,False
+google_2011_cpu_by_job_id_instant_recommended,cms,rows=3 cols=4096,0.04232304463607425,0.05,4.5252910014443425e-05,0.6798157955982075,20/20,124.03130531311035,1638.5921239852905,98.885632,True
+google_2011_cpu_by_job_id_instant_default,cms,default,0.04232304463607425,0.05,0.003137059413584803,57.16644243494396,20/20,115.76724052429199,1847.3784923553467,78.856192,True
+google_2011_cpu_by_job_id_5m_recommended,cms,rows=3 cols=4096,0.04301674918691996,0.05,4.902173506506417e-05,0.7003128433987799,20/20,99.68626499176025,3803.6561012268066,90.025984,True
+google_2011_cpu_by_job_id_5m_default,cms,default,0.04301674918691996,0.05,0.0031331704745136144,57.60383734423756,20/20,91.85421466827393,4261.834502220154,73.785344,True
+google_2011_cpu_p99_instant_recommended,kll,k=200,0.002039960011051183,0.01,0.10553291066016723,0.10553291066016723,20/20,3.455996513366699,1860.2386713027954,44.056576,False
+google_2011_cpu_p99_instant_default,kll,default,0.002039960011051183,0.01,0.458229353342154,0.458229353342154,20/20,3.654956817626953,1857.015609741211,43.003904,False