@@ -41,14 +41,7 @@ pub enum MilpOutputError {
4141 } ,
4242 #[ error( "sketch-bench variant dd: alpha must be finite and in (0, 1), got {alpha}" ) ]
4343 InvalidDdsAlpha { alpha : f64 } ,
44- #[ error( "topk query {0:?} has no literal k" ) ]
45- TopkWithoutK ( String ) ,
46- #[ error( "{variant}: can't size the heap for a {lookback_ms} ms lookback: {reason}" ) ]
47- HeapSize {
48- variant : String ,
49- lookback_ms : u64 ,
50- reason : & ' static str ,
51- } ,
44+
5245 #[ error( "query {0:?} is served by two deployments; a query string can name only one" ) ]
5346 QueryOnTwoDeployments ( String ) ,
5447 #[ error( transparent) ]
@@ -190,7 +183,7 @@ fn aggregation_config(
190183 vec ! [
191184 ( "depth" , Value :: from( integer_param( "rows" ) ?) ) ,
192185 ( "width" , Value :: from( integer_param( "cols" ) ?) ) ,
193- ( "heapsize" , Value :: from( heap_size( deployment, items ) ?) ) ,
186+ ( "heapsize" , Value :: from( heap_size( deployment) ?) ) ,
194187 ] ,
195188 ) ,
196189 ( _, capability) => {
@@ -248,42 +241,17 @@ fn missing_param(variant: &str, param: &'static str) -> MilpOutputError {
248241 }
249242}
250243
251- /// `m · k` for the most windows `m` a served query merges, so the merged
252- /// heaps still hold the top k, and never below the heap the plan priced
253- /// (sketch-bench's `heap` param, which can be a measured size above
254- /// `m · TOPK_K`). A literal k above `TOPK_K` is one the plan doesn't price.
255- fn heap_size ( deployment : & Deployment , items : & [ & OptimizerItem ] ) -> Result < u64 , MilpOutputError > {
256- let variant = deployment. config . sketch . as_str ( ) ;
257- let planned = rqe_optimizer:: heap_capacity ( & deployment. config )
258- . ok_or_else ( || missing_param ( variant, "heap" ) ) ?;
259- let k = max_topk_k ( items) ?. max ( rqe_optimizer:: TOPK_K ) ;
260- let mut heap = planned;
261- for item in items {
262- let lookback_ms = item. requirements . data_range_ms ;
263- let error = |reason| MilpOutputError :: HeapSize {
264- variant : variant. to_string ( ) ,
265- lookback_ms,
266- reason,
267- } ;
268- let m = deployment
269- . query_instance_count ( lookback_ms)
270- . ok_or_else ( || error ( "the lookback isn't whole windows" ) ) ?;
271- heap = heap. max ( m. checked_mul ( k) . ok_or_else ( || error ( "m · k overflows" ) ) ?) ;
272- }
273- Ok ( heap)
274- }
275-
276- /// One heap serves every topk query on the deployment, so it is sized for
277- /// the largest k.
278- fn max_topk_k ( items : & [ & OptimizerItem ] ) -> Result < u64 , MilpOutputError > {
279- items
280- . iter ( )
281- . flat_map ( |item| & item. query_strings )
282- . map ( |query| topk_k ( query) . ok_or_else ( || MilpOutputError :: TopkWithoutK ( query. clone ( ) ) ) )
283- . try_fold ( 0 , |max, k| Ok ( max. max ( k?) ) )
244+ /// The heap the plan priced: sketch-bench's `heap` param, `m · k` for the
245+ /// `m` windows a served query merges and the deployment's answered `k`
246+ /// (the largest its queries ask), or a measured size above it. The engine
247+ /// takes each query's `k` from the query itself.
248+ fn heap_size ( deployment : & Deployment ) -> Result < u64 , MilpOutputError > {
249+ rqe_optimizer:: heap_capacity ( & deployment. config )
250+ . ok_or_else ( || missing_param ( & deployment. config . sketch , "heap" ) )
284251}
285252
286- fn topk_k ( query : & str ) -> Option < u64 > {
253+ /// A `topk` query's literal `k`; `None` for anything else.
254+ pub ( crate ) fn topk_k ( query : & str ) -> Option < u64 > {
287255 let Ok ( Expr :: Aggregate ( aggregate) ) = promql_parser:: parser:: parse ( query) else {
288256 return None ;
289257 } ;
@@ -413,7 +381,7 @@ mod tests {
413381 assert_eq ! ( a. parameters[ "width" ] , json!( 2048 ) ) ;
414382 assert_eq ! (
415383 a. parameters[ "heapsize" ] ,
416- json!( heap_needed ( & workload, & solution, topk_value) )
384+ json!( planned_heap ( & workload, & solution, topk_value) )
417385 ) ;
418386 assert_eq ! (
419387 aggregation_for( & output, topk_count) . aggregation_sub_type,
@@ -428,30 +396,70 @@ mod tests {
428396 }
429397 }
430398
431- /// The heap `query`'s deployment needs for it: `m · TOPK_K` over the
432- /// windows its lookback merges (`Deployment::heap_needed`).
399+ /// The heap `query`'s deployment needs for it: `m · k` for the windows
400+ /// its lookback merges and its own `k` (`Deployment::heap_needed`).
433401 fn heap_needed ( workload : & MilpWorkload , solution : & MilpSolution , query : & str ) -> u64 {
434402 let i = workload
435403 . raqes
436404 . iter ( )
437405 . position ( |r| r. id . contains ( query) )
438406 . unwrap_or_else ( || panic ! ( "no raqe for {query}" ) ) ;
407+ let raqe = & workload. raqes [ i] ;
439408 let deployment = & solution. deployments [ solution. raqes [ i] . deployment ] . deployment ;
440409 deployment
441- . heap_needed ( workload . raqes [ i ] . lookback_ms )
410+ . heap_needed ( raqe . lookback_ms , raqe . topk_k ( ) )
442411 . expect ( "the lookback is whole windows" )
443412 }
444413
445- /// The engine's heap is the one the plan priced (m · k for the windows it
446- /// merges), not just the query's k, so merged windows keep the top k.
414+ /// The heap the plan priced for `query`'s deployment: its `heap` param.
415+ fn planned_heap ( workload : & MilpWorkload , solution : & MilpSolution , query : & str ) -> u64 {
416+ let i = workload
417+ . raqes
418+ . iter ( )
419+ . position ( |r| r. id . contains ( query) )
420+ . unwrap_or_else ( || panic ! ( "no raqe for {query}" ) ) ;
421+ let deployment = & solution. deployments [ solution. raqes [ i] . deployment ] . deployment ;
422+ rqe_optimizer:: heap_capacity ( & deployment. config ) . expect ( "a heap top-k deployment" )
423+ }
424+
425+ /// Each top-k query plans with its own literal k, and the engine's heap is
426+ /// the one the plan priced: `m · k` for the windows it merges, or the
427+ /// smallest measured heap (32) that already holds it.
428+ #[ test]
429+ fn topk_plans_with_its_own_k_and_heap_m_times_k ( ) {
430+ for ( k, measured_floor) in [ ( 10 , true ) , ( 100 , false ) ] {
431+ let query = format ! ( "topk({k}, sum_over_time(http_requests_total[1m]))" ) ;
432+ let ( config, workload, solution) = plan ( & group ( & query, 0.99 ) ) ;
433+ assert_eq ! ( workload. raqes[ 0 ] . topk_k, Some ( k) ) ;
434+ let deployment = & solution. deployments [ 0 ] . deployment ;
435+ assert_eq ! ( rqe_optimizer:: answered_k( & deployment. config) , k) ;
436+ let needed = heap_needed ( & workload, & solution, & query) ;
437+ let heap = planned_heap ( & workload, & solution, & query) ;
438+ if measured_floor {
439+ assert_eq ! ( heap, rqe_optimizer:: TOPK_K , "k = {k}" ) ;
440+ assert ! ( heap >= needed) ;
441+ } else {
442+ assert_eq ! ( heap, needed, "k = {k}: m · k" ) ;
443+ }
444+ let output = plan_to_planner_output ( & config, & workload, & solution) . unwrap ( ) ;
445+ assert_eq ! (
446+ aggregation_for( & output, & query) . parameters[ "heapsize" ] ,
447+ json!( heap) ,
448+ "k = {k}"
449+ ) ;
450+ }
451+ }
452+
453+ /// Queries sharing one heap get the heap the plan priced, which holds the
454+ /// largest k among them.
447455 #[ test]
448- fn heap_is_m_times_k_for_every_query_sharing_it ( ) {
456+ fn a_shared_heap_is_the_planned_heap ( ) {
449457 let small = "topk(5, sum_over_time(http_requests_total[1m]))" ;
450458 let large = "topk(10, sum_over_time(http_requests_total[1m]))" ;
451459 let ( config, workload, solution) = plan ( & ( group ( small, 0.99 ) + & group ( large, 0.99 ) ) ) ;
452460 assert_eq ! ( solution. deployments. len( ) , 1 ) ;
453- let heap = heap_needed ( & workload, & solution, small ) ;
454- assert_eq ! ( heap, heap_needed( & workload, & solution, large) ) ;
461+ let heap = planned_heap ( & workload, & solution, large ) ;
462+ assert ! ( heap >= heap_needed( & workload, & solution, large) ) ;
455463 let output = plan_to_planner_output ( & config, & workload, & solution) . unwrap ( ) ;
456464 for query in [ small, large] {
457465 assert_eq ! (
@@ -461,64 +469,18 @@ mod tests {
461469 }
462470 }
463471
464- /// `query`'s planned top-k deployment, re-windowed to 15 s so its 1 m
465- /// lookback merges m = 4 windows, with the cost row's `heap`.
466- fn merging_deployment ( query : & str , heap : serde_json:: Value ) -> ( MilpWorkload , Deployment ) {
467- let ( _, workload, solution) = plan ( & group ( query, 0.99 ) ) ;
468- let mut deployment = solution. deployments [ 0 ] . deployment . clone ( ) ;
469- ( deployment. window_ms , deployment. slide_ms ) = ( 15_000 , 15_000 ) ;
470- deployment. config . sketch_config [ "params" ] [ "heap" ] = heap;
471- ( workload, deployment)
472- }
473-
474- /// m comes from the lookback, not the row's heap: a measured heap of 2048
475- /// (above 4 · TOPK_K) stays the floor, and a large k needs 4 · k.
476- #[ test]
477- fn heap_size_takes_m_from_the_lookback ( ) {
478- for ( k, heap) in [ ( 5 , 2048 ) , ( 4096 , 4 * 4096 ) ] {
479- let query = format ! ( "topk({k}, sum_over_time(http_requests_total[1m]))" ) ;
480- let ( workload, deployment) = merging_deployment ( & query, json ! ( 2048 ) ) ;
481- let items: Vec < & OptimizerItem > = workload. items . iter ( ) . collect ( ) ;
482- assert_eq ! ( heap_size( & deployment, & items) . unwrap( ) , heap, "k = {k}" ) ;
483- }
484- }
485-
486472 #[ test]
487- fn heap_size_reports_overflow_and_a_missing_heap ( ) {
488- let query = "topk(4611686018427387904, sum_over_time(http_requests_total[1m]))" ;
489- let ( workload, deployment) = merging_deployment ( query, json ! ( 2048 ) ) ;
490- let items: Vec < & OptimizerItem > = workload. items . iter ( ) . collect ( ) ;
491- assert ! ( matches!(
492- heap_size( & deployment, & items) ,
493- Err ( MilpOutputError :: HeapSize {
494- reason: "m · k overflows" ,
495- ..
496- } )
497- ) ) ;
498- let small = "topk(5, sum_over_time(http_requests_total[1m]))" ;
499- let ( workload, mut deployment) = merging_deployment ( small, json ! ( 2048 ) ) ;
473+ fn a_heap_family_without_a_heap_is_a_missing_param ( ) {
474+ let query = "topk(5, sum_over_time(http_requests_total[1m]))" ;
475+ let ( _, _, solution) = plan ( & group ( query, 0.99 ) ) ;
476+ let mut deployment = solution. deployments [ 0 ] . deployment . clone ( ) ;
500477 deployment. config . sketch = "cms-fastpath-vector2d" . into ( ) ;
501- let items: Vec < & OptimizerItem > = workload. items . iter ( ) . collect ( ) ;
502478 assert ! ( matches!(
503- heap_size( & deployment, & items ) ,
479+ heap_size( & deployment) ,
504480 Err ( MilpOutputError :: MissingParam { param: "heap" , .. } )
505481 ) ) ;
506482 }
507483
508- /// A k above the plan's `TOPK_K` gets `m · k`, so its merged heaps still
509- /// hold the top k.
510- #[ test]
511- fn heap_scales_a_k_above_topk_k_by_the_merged_windows ( ) {
512- let query = "topk(4096, sum_over_time(http_requests_total[1m]))" ;
513- let ( config, workload, solution) = plan ( & group ( query, 0.99 ) ) ;
514- let m = heap_needed ( & workload, & solution, query) / rqe_optimizer:: TOPK_K ;
515- let output = plan_to_planner_output ( & config, & workload, & solution) . unwrap ( ) ;
516- assert_eq ! (
517- aggregation_for( & output, query) . parameters[ "heapsize" ] ,
518- json!( m * 4096 )
519- ) ;
520- }
521-
522484 #[ test]
523485 fn ddsketch_plan_preserves_the_cost_row_alpha ( ) {
524486 let query = "quantile_over_time(0.99, http_requests_total[5m])" ;
0 commit comments