Repository navigation
eval: one RQE per schema query, the strictest template's #216
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -297,6 +297,10 @@ def rqe_entry(q, curves, curves_big, sat, sat_big, merge): | |
| # Set by --samples-per-sec: a schema RQE whose smallest covered group falls | ||
| # below SCHEMA_MIN_COVERED_N is left out (logged) instead of failing. | ||
| SKIP_BELOW_CURVES = False | ||
| # Set by --keep-duplicate-queries: every template's copy of a query is an | ||
| # RQE. Otherwise only the strictest copy is, since the engine answers one | ||
| # query string from one aggregation. | ||
| KEEP_DUPLICATE_QUERIES = False | ||
|
|
||
|
|
||
| def set_samples_per_sec(rate, skip_below_curves): | ||
|
|
@@ -534,6 +538,28 @@ def add(query_id, stream, capability, rng, s, t, groups, keys=None, promql=None) | |
| return out | ||
|
|
||
|
|
||
| def schema_query(name, value, capability, what, grouping): | ||
| """A schema template's query on `grouping`: the fields that make it | ||
| one query string.""" | ||
| query = {"table": name, "value": value, "capability": capability, "grouping": grouping} | ||
| if capability == "quantile": # p99_latency -> 0.99 | ||
| query["quantile"] = int(what[1:what.index("_")]) / 100 | ||
| return query | ||
|
|
||
|
|
||
| def strictest_templates(): | ||
| """{query: (number of templates asking it, the strictest one's number)} | ||
| over SCHEMA_TEMPLATES. Stricter covers more groups: every group (None), | ||
| then the smallest `covers_share`.""" | ||
| copies = {} | ||
| for n, what, name, value, capability, groupings, covers in SCHEMA_TEMPLATES: | ||
| for grouping in groupings: | ||
| key = json.dumps(schema_query(name, value, capability, what, grouping), | ||
| sort_keys=True) | ||
| copies.setdefault(key, []).append((covers is not None, covers or 0, n)) | ||
| return {key: (len(c), min(c)[2]) for key, c in copies.items()} | ||
|
|
||
|
|
||
| def schema_queries(interval, dataset, range_with_interval, seen, metric=None, e2e=None): | ||
| """SCHEMA_TEMPLATES as synthetic_queries' query-ranges, over | ||
| SCHEMA_WINDOWS, repeating every `interval` seconds. A metric's values are | ||
|
|
@@ -545,13 +571,21 @@ def schema_queries(interval, dataset, range_with_interval, seen, metric=None, e2 | |
| ASAPQuery SQL (there is no PromQL): {sql, table, value, | ||
| capability, grouping, quantile (quantiles only), window_secs, | ||
| interval_secs, rqes}. `table` is the metric with `/` as `_` | ||
| (`data_1_http`). | ||
|
|
||
| A query several templates ask (14, 15 and 17 share groupings) is only | ||
| its strictest template's RQE, unless KEEP_DUPLICATE_QUERIES.""" | ||
| out = [] | ||
| strictest = strictest_templates() | ||
| for n, what, name, value, capability, groupings, covers in SCHEMA_TEMPLATES: | ||
| schema = SCHEMAS[name] | ||
| shape = schema["values"][value] | ||
| full = f"{metric}/{name}" if metric else name | ||
| for grouping in groupings: | ||
| copies, keep = strictest[json.dumps( | ||
| schema_query(name, value, capability, what, grouping), sort_keys=True)] | ||
| if not KEEP_DUPLICATE_QUERIES and keep != n: | ||
| continue | ||
| groups = grouping_cardinality(name, grouping) | ||
| query_id = f"t{n}_{what}_by_{','.join(grouping)}" | ||
| if metric: | ||
|
|
@@ -571,16 +605,21 @@ def schema_queries(interval, dataset, range_with_interval, seen, metric=None, e2 | |
| # can't serve the RQE, so a lower --samples-per-sec leaves it | ||
| # (and its e2e query) out. | ||
| if SKIP_BELOW_CURVES and covered_min_n < SCHEMA_MIN_COVERED_N: | ||
| # A weaker template's copy might fit, but serving the | ||
| # query at a weaker target would change the workload. | ||
| if not KEEP_DUPLICATE_QUERIES and copies > 1: | ||
| raise ValueError( | ||
| f"{dataset}/{query_id}/{kind}/{rng}, the strictest of {copies} " | ||
| f"templates asking its query, has a covered group of " | ||
| f"{covered_min_n:.0f} items, below the curves' first N") | ||
| print(f"left out {dataset}/{query_id}/{kind}/{rng}: smallest covered " | ||
| f"group holds {covered_min_n:.0f} items, below the curves' first N", | ||
| file=sys.stderr) | ||
| continue | ||
| assert covered_min_n >= SCHEMA_MIN_COVERED_N, (query_id, rng, covered_min_n) | ||
| if e2e is not None: | ||
| query = schema_query(full.replace("/", "_"), value, capability, what, | ||
| grouping) | ||
| query["sql"] = asapquery_sql(query, s) | ||
| e2e.append({**query, "window_secs": s, "interval_secs": interval, | ||
| "rqes": [f"{dataset}/{query_id}/{kind}/{rng}"]}) | ||
|
|
@@ -834,21 +873,29 @@ def main(argv=None): | |
| help=f"--synthetic: samples/s per series (default " | ||
| f"{SYNTHETIC_SAMPLES_PER_SEC}); the schema metrics' rates scale " | ||
| "with it, and schema RQEs below the curves' first N are left out") | ||
|
Contributor
Author
There was a problem hiding this comment. Stale: without
Contributor
Author
There was a problem hiding this comment. Done in 5445c5c: both helps now say the strictest copy below the curves' first N fails the export without --keep-duplicate-queries. |
||
| parser.add_argument("--keep-duplicate-queries", action="store_true", | ||
| help="--synthetic: an RQE for every template asking a query, not " | ||
| "just the strictest (the section 6.3 tables)") | ||
| parser.add_argument("--out", required=True) | ||
| args = parser.parse_args(argv) | ||
| # An instant query reads one scrape, 1000 / rate ms, which the runner keeps | ||
| # in whole ms. | ||
| rate = args.samples_per_sec | ||
| if rate is not None and (rate <= 0 or 1000 % rate): | ||
| parser.error("--samples-per-sec must divide 1000") | ||
| global KEEP_DUPLICATE_QUERIES | ||
| KEEP_DUPLICATE_QUERIES = args.keep_duplicate_queries | ||
| try: | ||
| if args.samples_per_sec is not None: | ||
| default = SYNTHETIC_SAMPLES_PER_SEC | ||
| set_samples_per_sec(args.samples_per_sec, True) | ||
| try: | ||
| return run(args, parser) | ||
| finally: | ||
| set_samples_per_sec(default, False) | ||
| return run(args, parser) | ||
| finally: | ||
| KEEP_DUPLICATE_QUERIES = False | ||
|
|
||
|
|
||
| def run(args, parser): | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This turns a leave-out into a hard failure for the whole export. With
--synthetic --samples-per-sec 2(or 1) it raises ont17_distinct_users_by_service,endpoint/keys/5m(the strictest of 3 templates, covered group of 449 items), and no tables get written, not even classic. Before this PR only that RQE was dropped. Should this keep leaving the query out (all its copies) with a stderr line, instead of raising?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Keeping the error; that was the decision in #214. A query several templates ask must keep its strictest requirement, so the export stops rather than drop or weaken it. At --samples-per-sec 2 the fix is to change the templates (or pass --keep-duplicate-queries), not to plan a workload without that query.