Skip to content

Commit a4e0017

Browse files
feat(tools): measure Prometheus ingest cost from a skip_querying run (#550)
* feat(tools): measure Prometheus ingest cost from a skip_querying run Add --ingest_only_experiment_name to compare_costs.py so the baseline mode's Query CPU can subtract a directly-measured ingest baseline (median Prometheus CPU% from a skip_querying, no-queries-executed run) instead of the 5th-percentile-of-this-run heuristic. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> * fix(tools): warn loudly when compare_costs.py falls back to the ingest heuristic Print a stderr warning when --ingest_only_experiment_name isn't given and compare_costs.py falls back to the 5th-percentile-of-this-run heuristic for Prometheus ingest cost, so it's obvious the Query CPU numbers are a guess rather than a measured baseline. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> --------- Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
1 parent 89fd022 commit a4e0017

1 file changed

Lines changed: 78 additions & 4 deletions

File tree

‎asap-tools/experiments/post_experiment/compare_costs.py‎

Lines changed: 78 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,44 @@ def pretty_print(key, value):
3030
print(key, round(value, 2))
3131

3232

33-
def calculate_query_cpu(monitor_info, experiment_mode):
33+
def measure_prometheus_ingest_cost(ingest_only_experiment_name: str) -> float:
34+
"""
35+
Median CPU% of the prometheus process during a skip_querying (ingest-only,
36+
no queries executed) run, i.e. a directly-measured ingestion baseline
37+
rather than an inferred percentile.
38+
"""
39+
monitor_info_file = os.path.join(
40+
constants.LOCAL_EXPERIMENT_DIR,
41+
ingest_only_experiment_name,
42+
constants.BASELINE_EXPERIMENT_NAME,
43+
"remote_monitor_output",
44+
"monitor_output.json",
45+
)
46+
with open(monitor_info_file, "r") as f:
47+
monitor_info = json.load(f)
48+
49+
for pid in monitor_info:
50+
if monitor_info[pid]["keyword"] == PROMETHEUS_PROCESS_KEYWORD:
51+
return float(np.median(monitor_info[pid]["cpu_percent"]))
52+
53+
raise ValueError(
54+
f"No prometheus PID found in ingest-only experiment "
55+
f"{ingest_only_experiment_name}"
56+
)
57+
58+
59+
def calculate_query_cpu(
60+
monitor_info, experiment_mode, ingest_baseline_cpu_percent=None
61+
):
3462
"""
3563
Calculate Query CPU timeseries for the given experiment mode.
3664
3765
Args:
3866
monitor_info: Dictionary with PIDs as keys and monitoring data as values
3967
experiment_mode: Name of the current experiment mode
68+
ingest_baseline_cpu_percent: For baseline mode, a directly-measured
69+
ingestion CPU% (see measure_prometheus_ingest_cost) to subtract
70+
instead of the 5th-percentile heuristic.
4071
4172
Returns:
4273
List of Query CPU values (timeseries)
@@ -69,8 +100,20 @@ def calculate_query_cpu(monitor_info, experiment_mode):
69100
if prometheus_cpu is None:
70101
raise ValueError(f"No prometheus PID found in mode {experiment_mode}")
71102

72-
# Calculate 5th percentile (ingestion cost)
73-
prometheus_ingestion_cost = np.percentile(prometheus_cpu, 5)
103+
if ingest_baseline_cpu_percent is not None:
104+
prometheus_ingestion_cost = ingest_baseline_cpu_percent
105+
else:
106+
# Fallback heuristic: 5th percentile of this run's own CPU series.
107+
prometheus_ingestion_cost = np.percentile(prometheus_cpu, 5)
108+
print(
109+
"\n" + "!" * 70 + "\n"
110+
"WARNING: no --ingest_only_experiment_name given. Estimating "
111+
"Prometheus ingest cost as the 5th percentile of this run's own "
112+
"CPU series (a heuristic guess), instead of a directly-measured "
113+
"baseline from a skip_querying run. Query CPU numbers below may "
114+
"be inaccurate.\n" + "!" * 70 + "\n",
115+
file=sys.stderr,
116+
)
74117

75118
# Subtract ingestion cost from each time point
76119
query_cpu = [cpu - prometheus_ingestion_cost for cpu in prometheus_cpu]
@@ -284,6 +327,18 @@ def main(args):
284327
if not args.machine_readable:
285328
print(f"Experiment modes to analyze: {experiment_modes}")
286329

330+
ingest_baseline_cpu_percent = None
331+
if args.ingest_only_experiment_name:
332+
ingest_baseline_cpu_percent = measure_prometheus_ingest_cost(
333+
args.ingest_only_experiment_name
334+
)
335+
if not args.machine_readable:
336+
print(
337+
f"Using measured Prometheus ingest baseline from "
338+
f"{args.ingest_only_experiment_name}: "
339+
f"{round(ingest_baseline_cpu_percent, 2)}%"
340+
)
341+
287342
experiment_mode_to_overall_resource_usage = {}
288343
experiment_mode_to_query_cpu = {}
289344
experiment_mode_to_precompute_cpu = {}
@@ -346,7 +401,15 @@ def main(args):
346401

347402
# Calculate Query CPU for this experiment mode
348403
try:
349-
query_cpu = calculate_query_cpu(monitor_info, experiment_mode)
404+
query_cpu = calculate_query_cpu(
405+
monitor_info,
406+
experiment_mode,
407+
ingest_baseline_cpu_percent=(
408+
ingest_baseline_cpu_percent
409+
if experiment_mode == constants.BASELINE_EXPERIMENT_NAME
410+
else None
411+
),
412+
)
350413
experiment_mode_to_query_cpu[experiment_mode] = query_cpu
351414
except (AssertionError, ValueError) as e:
352415
if not args.machine_readable:
@@ -534,6 +597,17 @@ def main(args):
534597
action="store_true",
535598
help="Save to experiment directory",
536599
)
600+
parser.add_argument(
601+
"--ingest_only_experiment_name",
602+
type=str,
603+
required=False,
604+
help=(
605+
"Name of a skip_querying (ingest-only) experiment run. When set, "
606+
"the baseline mode's Query CPU is computed by subtracting the "
607+
"median Prometheus CPU%% measured in that run, instead of the "
608+
"5th-percentile-of-this-run heuristic."
609+
),
610+
)
537611
parser.add_argument(
538612
"--machine-readable",
539613
action="store_true",

0 commit comments

Comments
 (0)