77//! from the data workload and the deployment's operand-domain contracts),
88//! or when the deployment's empirical evidence shows this
99//! candidate's output does (`AccuracyEvidenceProvider::output_guarantee`,
10- //! read only here). An output with no guarantee from either source is
11- //! accepted only when the deployment's accuracy model accepts it without one
12- //! (`AccuracyModel::accepts_output_without_guarantee `). For a query with a target, Count-Min is admitted only
10+ //! read only here). Before either, the deployment's accuracy model may accept
11+ //! the output directly, whatever its guarantee says
12+ //! (`AccuracyModel::accepts_output_regardless_of_guarantee `). For a query with a target, Count-Min is admitted only
1313//! over weights proven non-negative; a query without one constrains nothing.
1414//! A miss rejects the candidate as invalid, with a reason; only this stage
1515//! discards valid candidates.
@@ -589,8 +589,8 @@ pub enum RejectionCause {
589589 AccuracyTargetMissed ,
590590 /// A query's output has no accuracy guarantee (Stage 1 has no
591591 /// analytical rule for it and the deployment's evidence supplies none),
592- /// and the deployment's accuracy model does not accept it without one
593- /// ([`AccuracyModel::accepts_output_without_guarantee `]).
592+ /// and the deployment's accuracy model did not accept it directly
593+ /// ([`AccuracyModel::accepts_output_regardless_of_guarantee `]).
594594 NoAccuracyGuarantee ,
595595 /// It scans raw data at query time in a deployment that does not
596596 /// retain raw data.
@@ -645,14 +645,14 @@ pub struct AcceptedAccuracy {
645645 /// The target's metric; `None` when the user did not set one.
646646 pub target_metric : Option < AccuracyMetric > ,
647647 /// The metric the accepted guarantee is stated in; `None` for an exact
648- /// output, which meets every metric, and for one accepted without a
649- /// guarantee.
648+ /// output, which meets every metric, and for one the deployment's model
649+ /// accepted regardless of its guarantee (no guarantee was checked) .
650650 pub accepted_metric : Option < AccuracyMetric > ,
651- /// Whether the output has no guarantee and the deployment's accuracy
652- /// model accepted it anyway
653- /// ([`AccuracyModel::accepts_output_without_guarantee `]); `false`
654- /// otherwise .
655- pub accepted_without_guarantee : bool ,
651+ /// Whether the deployment's accuracy model accepted the output directly,
652+ /// whatever its guarantee says
653+ /// ([`AccuracyModel::accepts_output_regardless_of_guarantee `]); `false`
654+ /// when a guarantee met the target .
655+ pub accepted_regardless_of_guarantee : bool ,
656656 /// The sketch algorithms the query's output reads, in plan order
657657 /// (e.g. `["Kll"]`); empty for an exact plan.
658658 pub sketches : Vec < String > ,
@@ -661,7 +661,7 @@ pub struct AcceptedAccuracy {
661661impl std:: fmt:: Display for AcceptedAccuracy {
662662 /// E.g. `metric not set; accepted rank (Kll)`, or
663663 /// `metric relative; accepted relative (DDSketch)`, or
664- /// `metric rank; accepted without a guarantee (Kll)`.
664+ /// `metric rank; accepted by the deployment's accuracy model regardless of guarantee (Kll)`.
665665 fn fmt ( & self , f : & mut std:: fmt:: Formatter < ' _ > ) -> std:: fmt:: Result {
666666 let name = |metric : AccuracyMetric | {
667667 serde_json:: to_value ( metric)
@@ -674,7 +674,10 @@ impl std::fmt::Display for AcceptedAccuracy {
674674 Some ( metric) => write ! ( f, "metric {}; " , name( metric) ) ?,
675675 }
676676 match self . accepted_metric {
677- _ if self . accepted_without_guarantee => write ! ( f, "accepted without a guarantee" ) ?,
677+ _ if self . accepted_regardless_of_guarantee => write ! (
678+ f,
679+ "accepted by the deployment's accuracy model regardless of guarantee"
680+ ) ?,
678681 None => write ! ( f, "accepted exact" ) ?,
679682 Some ( metric) => write ! ( f, "accepted {}" , name( metric) ) ?,
680683 }
@@ -1416,12 +1419,13 @@ pub fn resize_for_failure_probability(
14161419/// its ε, δ and scope ([`AccuracyModel::satisfies`]); one that sets a metric
14171420/// accepts only guarantees in it (or convertible to it).
14181421///
1419- /// An output with no guarantee from either source (Stage 1 had no rule,
1420- /// and the evidence has none) is accepted only when the deployment's
1421- /// accuracy model accepts it without one
1422- /// ([`AccuracyModel::accepts_output_without_guarantee`]); otherwise the
1423- /// candidate is rejected as [`RejectionCause::NoAccuracyGuarantee`]. Any
1424- /// other miss is [`RejectionCause::AccuracyTargetMissed`].
1422+ /// The deployment's accuracy model is asked first
1423+ /// ([`AccuracyModel::accepts_output_regardless_of_guarantee`]): an output it
1424+ /// accepts is accepted whatever its guarantee says, and none of the checks
1425+ /// above runs for it. Otherwise, an output with no guarantee from either
1426+ /// source (Stage 1 had no rule, and the evidence has none) is rejected as
1427+ /// [`RejectionCause::NoAccuracyGuarantee`], and any other miss as
1428+ /// [`RejectionCause::AccuracyTargetMissed`].
14251429fn accuracy_check (
14261430 candidate : & PhysicalCandidate ,
14271431 targets : & [ AccuracyTarget ] ,
@@ -1432,6 +1436,9 @@ fn accuracy_check(
14321436 for ( query, ( root, target) ) in candidate. roots . iter ( ) . zip ( targets) . enumerate ( ) {
14331437 let query = query + 1 ;
14341438 let mut sketches: Vec < String > = Vec :: new ( ) ;
1439+ // The first Count-Min whose update weights are not proven
1440+ // non-negative, which its bound assumes, as a rejection reason.
1441+ let mut unproven_weights: Option < String > = None ;
14351442 for node in root_operators ( root)
14361443 . iter ( )
14371444 . flat_map ( |node| OperatorNode :: reachable ( node) )
@@ -1461,16 +1468,32 @@ fn accuracy_check(
14611468 }
14621469 _ => String :: new ( ) ,
14631470 } ;
1464- return Err ( (
1465- RejectionCause :: AccuracyTargetMissed ,
1471+ unproven_weights. get_or_insert_with ( || {
14661472 format ! (
14671473 "q{query}: {} needs non-negative update weights, and these are not \
14681474 proven non-negative{unconfirmed}",
14691475 family_name( family)
1470- ) ,
1471- ) ) ;
1476+ )
1477+ } ) ;
14721478 }
14731479 }
1480+ // The deployment's model may accept the output directly, whatever
1481+ // its guarantee says; no guarantee or evidence check runs then.
1482+ if models
1483+ . accuracy
1484+ . accepts_output_regardless_of_guarantee ( root, target)
1485+ {
1486+ accepted. push ( AcceptedAccuracy {
1487+ target_metric : target. accuracy_metric ,
1488+ accepted_metric : None ,
1489+ accepted_regardless_of_guarantee : true ,
1490+ sketches,
1491+ } ) ;
1492+ continue ;
1493+ }
1494+ if let Some ( reason) = unproven_weights {
1495+ return Err ( ( RejectionCause :: AccuracyTargetMissed , reason) ) ;
1496+ }
14741497 // A guarantee per value is stated over the whole result's `G`
14751498 // values for a whole-result target.
14761499 let series = series_read ( root, data) ;
@@ -1495,31 +1518,17 @@ fn accuracy_check(
14951518 accepted. push ( AcceptedAccuracy {
14961519 target_metric : target. accuracy_metric ,
14971520 accepted_metric : ( !met. is_exact ( ) ) . then_some ( met. spec . accuracy_metric ) ,
1498- accepted_without_guarantee : false ,
1521+ accepted_regardless_of_guarantee : false ,
14991522 sketches,
15001523 } ) ;
15011524 continue ;
15021525 }
1503- // No guarantee from either source: the deployment's model decides.
15041526 if guarantee. is_none ( ) && evidence. is_none ( ) {
1505- if models
1506- . accuracy
1507- . accepts_output_without_guarantee ( root, target)
1508- {
1509- accepted. push ( AcceptedAccuracy {
1510- target_metric : target. accuracy_metric ,
1511- accepted_metric : None ,
1512- accepted_without_guarantee : true ,
1513- sketches,
1514- } ) ;
1515- continue ;
1516- }
15171527 return Err ( (
15181528 RejectionCause :: NoAccuracyGuarantee ,
15191529 format ! (
1520- "q{query}: the output has no guarantee, the deployment has no evidence \
1521- for it, and its accuracy model does not accept it for target {target:?} \
1522- without one"
1530+ "q{query}: the output has no guarantee, and the deployment has no evidence \
1531+ for it, so it cannot meet target {target:?}"
15231532 ) ,
15241533 ) ) ;
15251534 }
@@ -1534,24 +1543,23 @@ fn accuracy_check(
15341543 {
15351544 format ! (
15361545 "q{query}: the output's guarantee holds for each value, and the number \
1537- of values one evaluation returns is not bound (no declared series count \
1538- for a source a PromQL vector reads; no bound on a SQL result's groups \
1539- yet), \
1540- so it does not meet whole-result target {target:?}, and the deployment \
1541- has no evidence that it meets it"
1546+ of values one evaluation returns is not bound (no declared series \
1547+ count for a source a PromQL vector reads; no bound on a SQL result's \
1548+ groups yet), so it does not meet whole-result target {target:?}, and \
1549+ the deployment has no evidence that it meets it"
15421550 )
15431551 }
15441552 Some ( g) => format ! (
1545- "q{query}: the output guarantees a {:?} bound {:?}, failure probability {:?}, \
1546- which misses target {target:?}, and the deployment has no evidence that it \
1547- meets it",
1553+ "q{query}: the output guarantees a {:?} bound {:?}, failure probability \
1554+ {:?}, which misses target {target:?}, and the deployment has no evidence \
1555+ that it meets it",
15481556 g. spec. accuracy_metric,
15491557 g. spec. error. evaluate( ) ,
15501558 g. spec. failure_probability. evaluate( ) ,
15511559 ) ,
15521560 None => format ! (
1553- "q{query}: the output has no guarantee, and the deployment has no evidence \
1554- that it meets target {target:?}"
1561+ "q{query}: the output has no guarantee, and the deployment has no \
1562+ evidence that it meets target {target:?}"
15551563 ) ,
15561564 } ,
15571565 ) ) ;
@@ -4249,6 +4257,91 @@ mod tests {
42494257 }
42504258 }
42514259
4260+ /// Deployment evidence of a large absolute error (1.0) for every output.
4261+ struct EveryOutputMeasuredLoose ;
4262+
4263+ impl AccuracyEvidenceProvider for EveryOutputMeasuredLoose {
4264+ fn output_guarantee ( & self , _output : & QueryRoot ) -> Option < AccuracyGuarantee > {
4265+ Some ( AccuracyGuarantee :: new (
4266+ AccuracyMetric :: ABSOLUTE ,
4267+ BoundExpr :: Constant { value : 1.0 } ,
4268+ ProbabilityExpr :: Constant { value : 1e-6 } ,
4269+ vec ! [ ] ,
4270+ ) )
4271+ }
4272+ }
4273+
4274+ /// A model that accepts every output directly, whatever its guarantee.
4275+ struct AcceptsEveryOutput ;
4276+
4277+ impl AccuracyModel for AcceptsEveryOutput {
4278+ fn satisfies ( & self , guarantee : & AccuracyGuarantee , target : & AccuracyTarget ) -> bool {
4279+ BuiltinAccuracyModel . satisfies ( guarantee, target)
4280+ }
4281+
4282+ fn accepts_output_regardless_of_guarantee (
4283+ & self ,
4284+ _output : & QueryRoot ,
4285+ _target : & AccuracyTarget ,
4286+ ) -> bool {
4287+ true
4288+ }
4289+ }
4290+
4291+ /// The model's hook has priority over every guarantee. The summary
4292+ /// candidates of a difference of quantile estimates have no theoretical
4293+ /// guarantee: with evidence that misses the target the built-in model
4294+ /// rejects them as `AccuracyTargetMissed`, and with no evidence as
4295+ /// `NoAccuracyGuarantee`; a model that accepts every output accepts them
4296+ /// in both cases, regardless of guarantee.
4297+ #[ test]
4298+ fn the_accuracy_model_accepts_an_output_whatever_its_guarantee ( ) {
4299+ let target = AccuracyTarget :: max_error ( 0.01 ) . failure_probability ( 0.001 ) ;
4300+ let candidates = all_candidates (
4301+ "quantile_over_time(0.99, m[1m]) - quantile_over_time(0.5, m[1m])" ,
4302+ & target,
4303+ ) ;
4304+ let summaries: Vec < _ > = candidates
4305+ . iter ( )
4306+ . filter ( |c| has_summary_estimate ( c) )
4307+ . map ( |c| c. id . as_str ( ) )
4308+ . collect ( ) ;
4309+ assert ! ( !summaries. is_empty( ) ) ;
4310+ let select = |accuracy : & dyn AccuracyModel , evidence : & dyn AccuracyEvidenceProvider | {
4311+ select_plan (
4312+ & candidates,
4313+ & [ entry_with (
4314+ target. clone ( ) . accuracy_metric ( AccuracyMetric :: ABSOLUTE ) ,
4315+ repeating ( 10_000 ) ,
4316+ ) ] ,
4317+ & data ( ) ,
4318+ PlanningModels :: builtin ( )
4319+ . with_accuracy ( accuracy)
4320+ . with_evidence ( evidence) ,
4321+ )
4322+ . unwrap ( )
4323+ } ;
4324+ for ( evidence, builtin_cause) in [
4325+ (
4326+ & EveryOutputMeasuredLoose as & dyn AccuracyEvidenceProvider ,
4327+ RejectionCause :: AccuracyTargetMissed ,
4328+ ) ,
4329+ ( & NoAccuracyEvidence , RejectionCause :: NoAccuracyGuarantee ) ,
4330+ ] {
4331+ let builtin = select ( & BuiltinAccuracyModel , evidence) ;
4332+ let accepting = select ( & AcceptsEveryOutput , evidence) ;
4333+ for id in & summaries {
4334+ let rejection = builtin. rejected . iter ( ) . find ( |r| r. id == * id) . unwrap ( ) ;
4335+ assert_eq ! ( rejection. cause, builtin_cause, "{}" , rejection. reason) ;
4336+ assert ! ( accepting. costs. contains_key( * id) , "{id} is valid" ) ;
4337+ assert ! (
4338+ accepting. accepted[ * id] [ 0 ] . accepted_regardless_of_guarantee,
4339+ "{id}"
4340+ ) ;
4341+ }
4342+ }
4343+ }
4344+
42524345 /// `candidates` with every state node (a summary build or an exact
42534346 /// aggregate) kept at ingestion time, so selection charges its kept
42544347 /// bytes every second (a `MatAtQuery` state is held only for one
0 commit comments