Skip to content

Commit fe74d3c

Browse files
feat(planner): add asap-planner --planner legacy|milp (#808)
* refactor(planner): move MILP load-and-solve into the library plan_milp loads the facts and costs, builds the objective and solves, so asap-planner can share it with asap-optimizer-cli. PlannerOutput gains write_to_dir. Refs #790 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * feat(planner): add asap-planner --planner legacy|milp --planner milp plans with the rqe-optimizer MILP and writes its configs; legacy stays the default. milp takes --workload-facts and --atomic-costs (required) and --w-cpu/--w-mem, and supports only PromQL with --input_config. It rejects --query-log, --prometheus-url, --enable-punting, --range-duration-ms, --step-ms, and per-group step_ms/range_duration_ms, whose retention MILP configs don't model yet (#800). Refs #790 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * docs(planner): mention value_range in asap-planner --workload-facts help Refs #790 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * refactor(planner): write controller outputs with PlannerOutput::write_to_dir The PromQL, SQL and Elastic controllers each repeated the two-file write; the output file names now live in one place. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(planner): reject inputs the MILP planner would silently ignore - windowing, sketch_parameters and aggregate_cleanup are rejected by build_milp_workload, since the MILP picks windows and parameters itself. - Per-group step_ms/range_duration_ms move to reject_unwritable_queries (formerly reject_avg_queries), so print-only asap-optimizer-cli runs still plan them. - asap-planner --planner milp rejects a zero scrape interval and the ClickHouse flags. Refs #790 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 847bf1e commit fe74d3c

9 files changed

Lines changed: 327 additions & 100 deletions

File tree

‎asap-planner-rs/src/bin/optimizer_cli.rs‎

Lines changed: 18 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,12 @@
33
44
use std::path::PathBuf;
55

6-
use anyhow::Context;
76
use asap_planner::optimizer::{
8-
build_milp_workload, load_flat_atomic_cost_table, load_workload_facts, plan_to_planner_output,
9-
reject_avg_queries, solve_milp, MilpError,
7+
parse_weight, plan_milp, plan_to_planner_output, reject_unwritable_queries, MilpInputs,
8+
MilpPlan,
109
};
1110
use asap_planner::ControllerConfig;
1211
use clap::Parser;
13-
use rqe_optimizer::milp::Objective;
14-
use rqe_optimizer::saturation::SaturationCurves;
1512

1613
#[derive(Parser, Debug)]
1714
#[command(
@@ -84,51 +81,26 @@ fn main() -> anyhow::Result<()> {
8481
run_milp(&args, &config)
8582
}
8683

87-
/// Objective weights must be finite and non-negative: a negative weight
88-
/// rewards cost, and NaN poisons every coefficient.
89-
fn parse_weight(s: &str) -> Result<f64, String> {
90-
match s.parse::<f64>() {
91-
Ok(w) if w.is_finite() && w >= 0.0 => Ok(w),
92-
Ok(w) => Err(format!("must be finite and >= 0, got {w}")),
93-
Err(e) => Err(e.to_string()),
94-
}
95-
}
96-
9784
fn run_milp(args: &Args, config: &ControllerConfig) -> anyhow::Result<()> {
98-
config.warn_default_slas();
99-
let Some(hints) = config.metrics.as_deref() else {
100-
return Err(MilpError::MissingMetricHints.into());
101-
};
102-
let facts = load_workload_facts(&args.workload_facts, hints, args.data_ingestion_interval_ms)?;
103-
let curves = SaturationCurves::load(&args.saturation_dir).with_context(|| {
104-
format!(
105-
"loading saturation curves from --saturation-dir {}",
106-
args.saturation_dir.display()
107-
)
108-
})?;
109-
let costs = load_flat_atomic_cost_table(&args.atomic_costs)?;
110-
let Objective::AUCCost { w_cpu, w_mem } = Objective::default();
111-
let (w_cpu, w_mem) = (args.w_cpu.unwrap_or(w_cpu), args.w_mem.unwrap_or(w_mem));
112-
// All-zero weights make every plan cost 0, so the solver's pick is arbitrary.
113-
anyhow::ensure!(
114-
w_cpu > 0.0 || w_mem > 0.0,
115-
"--w-cpu and --w-mem are both 0; at least one must be positive"
116-
);
117-
let objective = Objective::AUCCost { w_cpu, w_mem };
118-
tracing::debug!(?objective, cost_rows = costs.len(), "milp: inputs loaded");
119-
12085
// Fail before solving when the plan would be written but can't be.
12186
if args.output_dir.is_some() {
122-
reject_avg_queries(config)?;
87+
reject_unwritable_queries(config)?;
12388
}
124-
let workload = build_milp_workload(config, &facts, args.data_ingestion_interval_ms)?;
125-
let solution = solve_milp(
126-
&workload,
127-
&facts,
128-
&costs,
89+
let MilpPlan {
90+
workload,
91+
solution,
12992
objective,
130-
args.allow_undeployable_families,
131-
&|raqe, deployment| curves.accuracy(raqe, deployment, &facts),
93+
} = plan_milp(
94+
config,
95+
&MilpInputs {
96+
workload_facts: &args.workload_facts,
97+
atomic_costs: &args.atomic_costs,
98+
saturation_dir: &args.saturation_dir,
99+
scrape_interval_ms: args.data_ingestion_interval_ms,
100+
w_cpu: args.w_cpu,
101+
w_mem: args.w_mem,
102+
allow_undeployable_families: args.allow_undeployable_families,
103+
},
132104
)?;
133105

134106
println!("=== Deployments: {} ===", solution.deployments.len());
@@ -180,29 +152,8 @@ fn run_milp(args: &Args, config: &ControllerConfig) -> anyhow::Result<()> {
180152
}
181153

182154
if let Some(dir) = &args.output_dir {
183-
let output = plan_to_planner_output(config, &workload, &solution)?;
184-
// Serialize both before writing either, so a failure can't leave a
185-
// new streaming config next to a stale inference config.
186-
let streaming = output.to_streaming_yaml_string()?;
187-
let inference = output.to_inference_yaml_string()?;
188-
std::fs::create_dir_all(dir)?;
189-
std::fs::write(dir.join("streaming_config.yaml"), streaming)?;
190-
std::fs::write(dir.join("inference_config.yaml"), inference)?;
155+
plan_to_planner_output(config, &workload, &solution)?.write_to_dir(dir)?;
191156
println!("\nwrote configs to {}", dir.display());
192157
}
193158
Ok(())
194159
}
195-
196-
#[cfg(test)]
197-
mod tests {
198-
use super::parse_weight;
199-
200-
#[test]
201-
fn weights_must_be_finite_and_non_negative() {
202-
assert_eq!(parse_weight("0.5"), Ok(0.5));
203-
assert_eq!(parse_weight("0"), Ok(0.0));
204-
for bad in ["-1", "NaN", "inf", "x"] {
205-
assert!(parse_weight(bad).is_err(), "{bad}");
206-
}
207-
}
208-
}

‎asap-planner-rs/src/elastic_dsl/controller.rs‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,7 @@ impl ElasticController {
3535

3636
pub fn generate_to_dir(&self, dir: &Path) -> Result<PlannerOutput, ControllerError> {
3737
let output = self.generate()?;
38-
std::fs::create_dir_all(dir)?;
39-
let streaming_str = serde_yaml::to_string(output.streaming_yaml())?;
40-
let inference_str = serde_yaml::to_string(output.inference_yaml())?;
41-
std::fs::write(dir.join("streaming_config.yaml"), streaming_str)?;
42-
std::fs::write(dir.join("inference_config.yaml"), inference_str)?;
38+
output.write_to_dir(dir)?;
4339
Ok(output)
4440
}
4541
}

‎asap-planner-rs/src/main.rs‎

Lines changed: 118 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
1+
use asap_planner::optimizer::{
2+
parse_weight, plan_milp, plan_to_planner_output, reject_unwritable_queries, MilpInputs,
3+
};
14
use asap_planner::{
2-
Controller, ElasticController, ElasticRuntimeOptions, RuntimeOptions, SQLController,
3-
SQLRuntimeOptions, StreamingEngine,
5+
Controller, ControllerConfig, ElasticController, ElasticRuntimeOptions, RuntimeOptions,
6+
SQLController, SQLRuntimeOptions, StreamingEngine,
47
};
58
use asap_types::enums::QueryLanguage;
69
use clap::Parser;
@@ -53,6 +56,36 @@ struct Args {
5356
#[arg(long = "clickhouse-database", required = false)]
5457
clickhouse_database: Option<String>,
5558

59+
/// `milp` plans with sketch-bench's rqe-optimizer: PromQL with
60+
/// --input_config only, labels from its `metrics:` hints.
61+
#[arg(long, value_enum, default_value = "legacy")]
62+
planner: PlannerArg,
63+
64+
/// MILP only. YAML workload facts: per metric, positive `value_range` and
65+
/// `cardinality` per label set, including all labels (the series count),
66+
/// plus the `shape` of each grouping sketches may serve.
67+
#[arg(long = "workload-facts", required_if_eq("planner", "milp"))]
68+
workload_facts: Option<PathBuf>,
69+
70+
/// MILP only. The flat cost table sketch-bench's
71+
/// `study_saturation.py --phase optimizer-cost` writes
72+
/// (`rqe_atomic_costs.json`).
73+
#[arg(long = "atomic-costs", required_if_eq("planner", "milp"))]
74+
atomic_costs: Option<PathBuf>,
75+
76+
/// MILP only. sketch-bench's saturation-study directory: sketch accuracy
77+
/// is read off its error-vs-N curves at each grouping's `shape`.
78+
#[arg(long = "saturation-dir", required_if_eq("planner", "milp"))]
79+
saturation_dir: Option<PathBuf>,
80+
81+
/// MILP only. Objective weight on CPU-sec/sec. Default: rqe-optimizer's.
82+
#[arg(long = "w-cpu", value_parser = parse_weight)]
83+
w_cpu: Option<f64>,
84+
85+
/// MILP only. Objective weight on memory GiB. Default: rqe-optimizer's.
86+
#[arg(long = "w-mem", value_parser = parse_weight)]
87+
w_mem: Option<f64>,
88+
5689
#[arg(short, long, action = clap::ArgAction::Count)]
5790
verbose: u8,
5891
}
@@ -62,6 +95,12 @@ enum EngineArg {
6295
Precompute,
6396
}
6497

98+
#[derive(clap::ValueEnum, Debug, Clone, Copy, PartialEq, Eq)]
99+
enum PlannerArg {
100+
Legacy,
101+
Milp,
102+
}
103+
65104
fn main() -> anyhow::Result<()> {
66105
let args = Args::parse();
67106

@@ -77,6 +116,21 @@ fn main() -> anyhow::Result<()> {
77116
EngineArg::Precompute => StreamingEngine::Precompute,
78117
};
79118

119+
if args.planner == PlannerArg::Milp {
120+
run_milp(&args)?;
121+
println!("Generated configs in {}", args.output_dir.display());
122+
return Ok(());
123+
}
124+
anyhow::ensure!(
125+
args.workload_facts.is_none()
126+
&& args.atomic_costs.is_none()
127+
&& args.saturation_dir.is_none()
128+
&& args.w_cpu.is_none()
129+
&& args.w_mem.is_none(),
130+
"--workload-facts, --atomic-costs, --saturation-dir, --w-cpu and --w-mem require \
131+
--planner milp"
132+
);
133+
80134
match args.query_language {
81135
QueryLanguage::promql => {
82136
let scrape_interval_ms = args.data_ingestion_interval_ms.ok_or_else(|| {
@@ -158,3 +212,65 @@ fn main() -> anyhow::Result<()> {
158212
println!("Generated configs in {}", args.output_dir.display());
159213
Ok(())
160214
}
215+
216+
fn run_milp(args: &Args) -> anyhow::Result<()> {
217+
anyhow::ensure!(
218+
matches!(args.query_language, QueryLanguage::promql),
219+
"--planner milp supports only --query-language promql"
220+
);
221+
anyhow::ensure!(
222+
args.query_log.is_none(),
223+
"--planner milp needs --input_config: query logs have no `metrics:` hints"
224+
);
225+
anyhow::ensure!(
226+
args.prometheus_url.is_none(),
227+
"--planner milp takes labels from the config's `metrics:` hints, not --prometheus-url"
228+
);
229+
anyhow::ensure!(
230+
!args.enable_punting && args.range_duration_ms == 0 && args.step_ms == 0,
231+
"--enable-punting, --range-duration-ms and --step-ms don't apply to --planner milp"
232+
);
233+
anyhow::ensure!(
234+
args.clickhouse_url.is_none() && args.clickhouse_database.is_none(),
235+
"--clickhouse-url and --clickhouse-database don't apply to --planner milp"
236+
);
237+
let config_path = args
238+
.input_config
239+
.as_deref()
240+
.ok_or_else(|| anyhow::anyhow!("--planner milp requires --input_config"))?;
241+
let scrape_interval_ms = args
242+
.data_ingestion_interval_ms
243+
.ok_or_else(|| anyhow::anyhow!("--planner milp requires --data-ingestion-interval-ms"))?;
244+
// Per-series sample rates divide by it.
245+
anyhow::ensure!(
246+
scrape_interval_ms > 0,
247+
"--data-ingestion-interval-ms must be positive"
248+
);
249+
let config: ControllerConfig = serde_yaml::from_str(&std::fs::read_to_string(config_path)?)?;
250+
// Fail before solving: the plan is always written.
251+
reject_unwritable_queries(&config)?;
252+
let plan = plan_milp(
253+
&config,
254+
&MilpInputs {
255+
workload_facts: args
256+
.workload_facts
257+
.as_deref()
258+
.expect("clap requires --workload-facts with --planner milp"),
259+
atomic_costs: args
260+
.atomic_costs
261+
.as_deref()
262+
.expect("clap requires --atomic-costs with --planner milp"),
263+
saturation_dir: args
264+
.saturation_dir
265+
.as_deref()
266+
.expect("clap requires --saturation-dir with --planner milp"),
267+
scrape_interval_ms,
268+
w_cpu: args.w_cpu,
269+
w_mem: args.w_mem,
270+
allow_undeployable_families: false,
271+
},
272+
)?;
273+
plan_to_planner_output(&config, &plan.workload, &plan.solution)?
274+
.write_to_dir(&args.output_dir)?;
275+
Ok(())
276+
}

0 commit comments

Comments
 (0)