diff --git a/.github/workflows/analysis_workflow.yml b/.github/workflows/analysis_workflow.yml index 085c1686e49..5c19df5242c 100644 --- a/.github/workflows/analysis_workflow.yml +++ b/.github/workflows/analysis_workflow.yml @@ -218,6 +218,18 @@ jobs: run: | python -m pip install -ve .[Testing] + - name: Build C++ benchmarks (needed for ASV discovery) + # cpp_microbenchmarks.py invokes the binary with + # --benchmark_list_tests=true at import time to register ASV classes. + # asv_checks.py needs those classes so the regenerated benchmarks.json + # matches the committed one. + shell: bash -l {0} + run: | + cmake --preset linux-release -DTEST=ON cpp + cmake --build cpp/out/linux-release-build --target benchmarks + env: + CMAKE_BUILD_PARALLEL_LEVEL: ${{vars.CMAKE_BUILD_PARALLEL_LEVEL}} + - name: Run ASV Tests Check script run: | python python/utils/asv_checks.py diff --git a/.github/workflows/benchmark_commits.yml b/.github/workflows/benchmark_commits.yml index 858805123b9..5c91a5039fc 100644 --- a/.github/workflows/benchmark_commits.yml +++ b/.github/workflows/benchmark_commits.yml @@ -71,6 +71,17 @@ jobs: env: CMAKE_BUILD_PARALLEL_LEVEL: ${{vars.CMAKE_BUILD_PARALLEL_LEVEL}} + - name: Build C++ benchmarks + # cpp_microbenchmarks.py shells out to this binary for both discovery + # (at ASV import) and per-variant timing. No results JSON is produced + # here; the binary is invoked by ASV during `asv run` below. + shell: bash -l {0} + run: | + cmake --preset linux-release -DTEST=ON cpp + cmake --build cpp/out/linux-release-build --target benchmarks + env: + CMAKE_BUILD_PARALLEL_LEVEL: ${{vars.CMAKE_BUILD_PARALLEL_LEVEL}} + - name: Create test bucket shell: bash -el {0} run: | diff --git a/build_tooling/check_cpp_bench_stability.py b/build_tooling/check_cpp_bench_stability.py new file mode 100644 index 00000000000..bd12acfb776 --- /dev/null +++ b/build_tooling/check_cpp_bench_stability.py @@ -0,0 +1,265 @@ +""" +Copyright 2026 Man Group Operations Limited + +Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt. + +As of the Change Date specified in that file, in accordance with the Business Source License, use of this +software will be governed by the Apache License, version 2.0. + +Run each Google microbenchmark N times and report which ones have enough +run-to-run variance to risk a false-positive `asv compare -f 1.15` regression +on PRs. + +Why this exists +--------------- +On PRs the CI does ``asv compare -s -f 1.15 HEAD``: any benchmark +whose HEAD time is more than 1.15x the master time is reported as a +regression and fails the build. For ``track_time_ms`` benchmarks (which is +what cpp_microbenchmarks.py registers) ASV records a single value per run, +so the comparison is a pure ratio — there is no statistical-significance +filter to absorb noise. A benchmark whose natural variance is greater than +~15% will therefore flap. + +What it does +------------ +For each benchmark name discovered from the binary, invoke the binary once +per simulated run (the same way cpp_microbenchmarks.py does in CI) and +collect the real_time. Then report: + + - max/min ratio — worst-case pair of runs; if > threshold, two ASV runs + (e.g. master vs PR HEAD) could trip the regression check. + - CV % — coefficient of variation (stddev / mean), useful for + spotting steady jitter even when max/min looks OK. + +Usage +----- + python build_tooling/check_cpp_bench_stability.py --runs 10 + python build_tooling/check_cpp_bench_stability.py --filter '^BM_arrow_' + python build_tooling/check_cpp_bench_stability.py --runs 20 --output stability.json +""" + +import argparse +import json +import re +import statistics +import subprocess +import sys +import time +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_BINARY = REPO_ROOT / "cpp/out/linux-release-build/arcticdb/benchmarks" + + +def list_benchmark_names(binary: Path, filter_regex: str | None) -> list[str]: + cmd = [str(binary), "--benchmark_list_tests=true"] + if filter_regex: + cmd.append(f"--benchmark_filter={filter_regex}") + result = subprocess.run(cmd, check=True, capture_output=True, text=True) + return [line.strip() for line in result.stdout.splitlines() if line.strip()] + + +def escape_for_benchmark_filter(name: str) -> str: + # Match cpp_microbenchmarks.py: do not escape '-' (std::regex rejects '\-'). + return re.sub(r"([\\^$.|?*+()\[\]{}])", r"\\\1", name) + + +def to_milliseconds(real_time: float, unit: str) -> float: + if unit == "ns": + return real_time / 1e6 + if unit == "us": + return real_time / 1e3 + if unit == "ms": + return real_time + if unit == "s": + return real_time * 1e3 + raise RuntimeError(f"Unexpected time_unit {unit!r}") + + +def run_one(binary: Path, full_name: str) -> float: + """Same shell-out shape as cpp_microbenchmarks.run_one — one process per run. + + Mirrors the bridge: prefer the ``median`` aggregate row when the C++ side sets + ``->Repetitions(N)->ReportAggregatesOnly(true)``; otherwise fall back to the + single iteration value. Scoring matches what CI will see. + """ + result = subprocess.run( + [ + str(binary), + f"--benchmark_filter=^{escape_for_benchmark_filter(full_name)}$", + "--benchmark_format=json", + ], + check=True, + capture_output=True, + text=True, + ) + data = json.loads(result.stdout) + benchmarks = data.get("benchmarks", []) + median_row = next( + (b for b in benchmarks if b.get("run_type") == "aggregate" and b.get("aggregate_name") == "median"), + None, + ) + if median_row is not None: + return to_milliseconds(median_row["real_time"], median_row.get("time_unit", "ms")) + iterations = [b for b in benchmarks if b.get("run_type", "iteration") == "iteration"] + if not iterations: + raise RuntimeError(f"No iteration or aggregate result returned for {full_name!r}") + bm = iterations[0] + return to_milliseconds(bm["real_time"], bm.get("time_unit", "ms")) + + +def summarize(times: list[float]) -> dict: + tmin = min(times) + tmax = max(times) + mean = statistics.mean(times) + stddev = statistics.pstdev(times) if len(times) > 1 else 0.0 + return { + "runs_ms": times, + "min_ms": tmin, + "max_ms": tmax, + "mean_ms": mean, + "median_ms": statistics.median(times), + "stddev_ms": stddev, + "max_over_min": tmax / tmin if tmin > 0 else float("inf"), + "cv_pct": (stddev / mean * 100.0) if mean > 0 else 0.0, + } + + +def print_table(rows: list[tuple[str, dict]]) -> None: + if not rows: + print(" (none)") + return + print(f" {'max/min':>8} {'CV %':>6} {'min (ms)':>12} {'max (ms)':>12} {'median (ms)':>12} name") + for name, r in rows: + print( + f" {r['max_over_min']:>8.3f}" + f" {r['cv_pct']:>6.2f}" + f" {r['min_ms']:>12.4f}" + f" {r['max_ms']:>12.4f}" + f" {r['median_ms']:>12.4f}" + f" {name}" + ) + +def validate_args(args): + if not args.binary.exists(): + raise ValueError( + f"Benchmark binary not found at {args.binary}.\n" + f"Build it with:\n" + f" cmake --preset linux-release -DTEST=ON cpp\n" + f" cmake --build cpp/out/linux-release-build --target benchmarks" + ) + + if args.runs < 2: + raise ValueError("--runs must be >= 2 to compute variance") + + names = list_benchmark_names(args.binary, args.filter) + if not names: + raise ValueError("No benchmarks discovered (check --filter).") + + +def parse_args(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--binary", type=Path, default=DEFAULT_BINARY, + help=f"Path to the benchmarks binary (default: {DEFAULT_BINARY})", + ) + parser.add_argument( + "--runs", type=int, default=10, + help="Number of times to invoke each benchmark (default: 10). Each invocation is a separate process, " + "matching how cpp_microbenchmarks.py drives the binary in CI.", + ) + parser.add_argument( + "--threshold", type=float, default=1.15, + help="Ratio above which a benchmark is flagged. Default 1.15 matches the CI `asv compare -f 1.15`.", + ) + parser.add_argument( + "--filter", type=str, default=None, + help="Google Benchmark filter regex. Use to scope the run, e.g. '^BM_arrow_'.", + ) + parser.add_argument( + "--output", type=Path, default=None, + help="Optional JSON file to dump full per-benchmark results.", + ) + parser.add_argument( + "--fail-on-flagged", action="store_true", + help="Exit non-zero if any benchmark exceeds the threshold (useful in CI).", + ) + + return parser.parse_args() + + +def main() -> int: + args = parse_args() + validate_args(args) + names = list_benchmark_names(args.binary, args.filter) + total_invocations = len(names) * args.runs + print( + f"Running {len(names)} benchmark(s) x {args.runs} run(s) = {total_invocations} invocation(s). " + f"Threshold = {args.threshold}x.", + flush=True, + ) + + results: dict[str, dict] = {} + failed: list[str] = [] + started = time.monotonic() + + for i, name in enumerate(names, 1): + times: list[float] = [] + for r in range(args.runs): + try: + times.append(run_one(args.binary, name)) + except Exception as e: + print(f" [{i}/{len(names)}] {name} run {r + 1} FAILED: {e}", flush=True) + if times: + stats = summarize(times) + results[name] = stats + flag = " [FLAG]" if stats["max_over_min"] > args.threshold else "" + print( + f" [{i}/{len(names)}] {name} " + f"min={stats['min_ms']:.4f}ms max={stats['max_ms']:.4f}ms " + f"max/min={stats['max_over_min']:.3f} cv={stats['cv_pct']:.2f}%{flag}", + flush=True, + ) + else: + failed.append(name) + + elapsed = time.monotonic() - started + print(f"\nDone in {elapsed:.1f}s.") + + flagged = sorted( + ((n, r) for n, r in results.items() if r["max_over_min"] > args.threshold), + key=lambda kv: kv[1]["max_over_min"], + reverse=True, + ) + safe = [(n, r) for n, r in results.items() if r["max_over_min"] <= args.threshold] + + print() + print("=" * 100) + print(f"At risk (max/min > {args.threshold}x) — {len(flagged)} of {len(results)} benchmark(s):") + print_table(flagged) + print() + print(f"Stable — {len(safe)} benchmark(s).") + if failed: + print() + print(f"Failed to run — {len(failed)} benchmark(s):") + for n in failed: + print(f" {n}") + + if args.output: + args.output.write_text(json.dumps( + {"threshold": args.threshold, "runs": args.runs, "results": results, "failed": failed}, + indent=2, + )) + print(f"\nFull results written to {args.output}") + + if args.fail_on_flagged and flagged: + return 1 + if failed: + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/cpp/arcticdb/arrow/test/benchmark_arrow_reads.cpp b/cpp/arcticdb/arrow/test/benchmark_arrow_reads.cpp index c32c5e98b4d..33447fdf3fb 100644 --- a/cpp/arcticdb/arrow/test/benchmark_arrow_reads.cpp +++ b/cpp/arcticdb/arrow/test/benchmark_arrow_reads.cpp @@ -90,66 +90,105 @@ static void BM_arrow_string_handler(benchmark::State& state) { } BENCHMARK(BM_arrow_string_handler) + ->Name("BM_arrow_string_handler_10k") // Dynamic strings // ArrowOutputStringFormat::CATEGORICAL // Not sparse, small string buffers ->Args({10'000, 1, 0, 0, 0}) - ->Args({100'000, 1, 0, 0, 0}) // Not sparse, large string buffers ->Args({10'000, 10'000, 0, 0, 0}) - ->Args({100'000, 100'000, 0, 0, 0}) // Half sparse ->Args({10'000, 1, 5'000, 0, 0}) - ->Args({100'000, 1, 50'000, 0, 0}) // Fully sparse ->Args({10'000, 1, 10'000, 0, 0}) - ->Args({100'000, 1, 100'000, 0, 0}) // ArrowOutputStringFormat::LARGE_STRING // Not sparse, small string buffers ->Args({10'000, 1, 0, 1, 0}) - ->Args({100'000, 1, 0, 1, 0}) // Not sparse, large string buffers ->Args({10'000, 10'000, 0, 1, 0}) - ->Args({100'000, 100'000, 0, 1, 0}) // Half sparse ->Args({10'000, 1, 5'000, 1, 0}) - ->Args({100'000, 1, 50'000, 1, 0}) // Fully sparse ->Args({10'000, 1, 10'000, 1, 0}) - ->Args({100'000, 1, 100'000, 1, 0}) // ArrowOutputStringFormat::SMALL_STRING // Not sparse, small string buffers ->Args({10'000, 1, 0, 2, 0}) - ->Args({100'000, 1, 0, 2, 0}) // Not sparse, large string buffers ->Args({10'000, 10'000, 0, 2, 0}) - ->Args({100'000, 100'000, 0, 2, 0}) // Half sparse ->Args({10'000, 1, 5'000, 2, 0}) - ->Args({100'000, 1, 50'000, 2, 0}) // Fully sparse ->Args({10'000, 1, 10'000, 2, 0}) - ->Args({100'000, 1, 100'000, 2, 0}) // Fixed-width strings // ArrowOutputStringFormat::CATEGORICAL // Not sparse, small string buffers ->Args({10'000, 1, 0, 0, 1}) - ->Args({100'000, 1, 0, 0, 1}) // Not sparse, large string buffers ->Args({10'000, 10'000, 0, 0, 1}) - ->Args({100'000, 100'000, 0, 0, 1}) // ArrowOutputStringFormat::LARGE_STRING // Not sparse, small string buffers ->Args({10'000, 1, 0, 1, 1}) - ->Args({100'000, 1, 0, 1, 1}) // Not sparse, large string buffers ->Args({10'000, 10'000, 0, 1, 1}) - ->Args({100'000, 100'000, 0, 1, 1}) // ArrowOutputStringFormat::STRING // Not sparse, small string buffers ->Args({10'000, 1, 0, 2, 1}) - ->Args({100'000, 1, 0, 2, 1}) // Not sparse, large string buffers ->Args({10'000, 10'000, 0, 2, 1}) - ->Args({100'000, 100'000, 0, 2, 1}); \ No newline at end of file + ->MinWarmUpTime(0.5) + ->MinTime(1.0) + ->Repetitions(14) + ->ReportAggregatesOnly(true); + +BENCHMARK(BM_arrow_string_handler) + ->Name("BM_arrow_string_handler_100k") + // Dynamic strings + // ArrowOutputStringFormat::CATEGORICAL + // Not sparse, small string buffers + ->Args({100'000, 1, 0, 0, 0}) + // Not sparse, large string buffers + ->Args({100'000, 100'000, 0, 0, 0}) + // Half sparse + ->Args({100'000, 1, 50'000, 0, 0}) + // Fully sparse + ->Args({100'000, 1, 100'000, 0, 0}) + // ArrowOutputStringFormat::LARGE_STRING + // Not sparse, small string buffers + ->Args({100'000, 1, 0, 1, 0}) + // Not sparse, large string buffers + ->Args({100'000, 100'000, 0, 1, 0}) + // Half sparse + ->Args({100'000, 1, 50'000, 1, 0}) + // Fully sparse + ->Args({100'000, 1, 100'000, 1, 0}) + // ArrowOutputStringFormat::SMALL_STRING + // Not sparse, small string buffers + ->Args({100'000, 1, 0, 2, 0}) + // Not sparse, large string buffers + ->Args({100'000, 100'000, 0, 2, 0}) + // Half sparse + ->Args({100'000, 1, 50'000, 2, 0}) + // Fully sparse + ->Args({100'000, 1, 100'000, 2, 0}) + + // Fixed-width strings + // ArrowOutputStringFormat::CATEGORICAL + // Not sparse, small string buffers + ->Args({100'000, 1, 0, 0, 1}) + // Not sparse, large string buffers + ->Args({100'000, 100'000, 0, 0, 1}) + // ArrowOutputStringFormat::LARGE_STRING + // Not sparse, small string buffers + ->Args({100'000, 1, 0, 1, 1}) + // Not sparse, large string buffers + ->Args({100'000, 100'000, 0, 1, 1}) + // ArrowOutputStringFormat::STRING + // Not sparse, small string buffers + ->Args({100'000, 1, 0, 2, 1}) + // Not sparse, large string buffers + ->Args({100'000, 100'000, 0, 2, 1}) + ->MinWarmUpTime(0.5) + ->MinTime(1.0) + ->Repetitions(9) + ->ReportAggregatesOnly(true); \ No newline at end of file diff --git a/cpp/arcticdb/arrow/test/benchmark_arrow_writes.cpp b/cpp/arcticdb/arrow/test/benchmark_arrow_writes.cpp index e3452e9af95..8985b0c2402 100644 --- a/cpp/arcticdb/arrow/test/benchmark_arrow_writes.cpp +++ b/cpp/arcticdb/arrow/test/benchmark_arrow_writes.cpp @@ -76,44 +76,100 @@ static void BM_arrow_convert_multiple_record_batches_to_segment(benchmark::State } BENCHMARK(BM_arrow_convert_single_record_batch_to_segment) - // Numeric data - // Short and wide + ->Name("BM_arrow_convert_single_record_batch_to_segment_short_and_wide_numeric") ->Args({10, 100'000, -1, true}) ->Args({10, 100'000, 0, true}) ->Args({10, 100'000, 50'000, true}) - // Long and thin - ->Args({10'000'000, 10, -1, true}) - ->Args({10'000'000, 10, 0, true}) - ->Args({10'000'000, 10, 5, true}) - // String data - // Short and wide + ->ArgNames({"num_rows", "num_columns", "index_column", "numeric_data"}) + ->MinWarmUpTime(0.5) + ->Iterations(3) + ->Repetitions(4) + ->ReportAggregatesOnly(true); + +BENCHMARK(BM_arrow_convert_single_record_batch_to_segment) + ->Name("BM_arrow_convert_single_record_batch_to_segment_short_and_wide_string") ->Args({10, 100'000, -1, false}) ->Args({10, 100'000, 0, false}) ->Args({10, 100'000, 50'000, false}) -// Long and thin - Windows CI can't handle this + ->ArgNames({"num_rows", "num_columns", "index_column", "numeric_data"}) + ->MinWarmUpTime(0.5) + ->Iterations(3) + ->Repetitions(8) + ->ReportAggregatesOnly(true); + +BENCHMARK(BM_arrow_convert_single_record_batch_to_segment) + ->Name("BM_arrow_convert_single_record_batch_to_segment_long_and_thin_numeric") + ->Args({10'000'000, 10, -1, true}) + ->Args({10'000'000, 10, 0, true}) + ->Args({10'000'000, 10, 5, true}) + ->ArgNames({"num_rows", "num_columns", "index_column", "numeric_data"}) + ->MinWarmUpTime(0.5) + ->Repetitions(4) + ->ReportAggregatesOnly(true); + +// Windows CI can't handle this #ifndef WIN32 +BENCHMARK(BM_arrow_convert_single_record_batch_to_segment) + ->Name("BM_arrow_convert_single_record_batch_to_segment_long_and_thin_string") ->Args({10'000'000, 10, -1, false}) ->Args({10'000'000, 10, 0, false}) ->Args({10'000'000, 10, 5, false}) + ->ArgNames({"num_rows", "num_columns", "index_column", "numeric_data"}) + ->MinWarmUpTime(0.5) + ->Repetitions(4) + ->ReportAggregatesOnly(true); #endif - ; BENCHMARK(BM_arrow_convert_multiple_record_batches_to_segment) - // Numeric data - // Short and wide + ->Name("BM_arrow_convert_multiple_record_batches_to_segment_short_and_wide_numeric") ->Args({10, 100'000, 10, true}) - // Long and thin + ->ArgNames({"num_rows", "num_columns", "num_record_batches", "numeric_data"}) + ->MinWarmUpTime(0.5) + ->Iterations(2) + ->Repetitions(9) + ->ReportAggregatesOnly(true); + +BENCHMARK(BM_arrow_convert_multiple_record_batches_to_segment) + ->Name("BM_arrow_convert_multiple_record_batches_to_segment_long_and_thin_numeric") ->Args({10'000'000, 10, 100, true}) ->Args({10'000'000, 10, 10'000, true}) - // Highly fragmented + ->ArgNames({"num_rows", "num_columns", "num_record_batches", "numeric_data"}) + ->MinWarmUpTime(0.5) + ->MinTime(5.0) + ->Repetitions(22) + ->ReportAggregatesOnly(true); + +BENCHMARK(BM_arrow_convert_multiple_record_batches_to_segment) + ->Name("BM_arrow_convert_multiple_record_batches_to_segment_numeric_fragmented") ->Args({1'000, 10, 1'000, true}) - // String data - // Short and wide + ->ArgNames({"num_rows", "num_columns", "num_record_batches", "numeric_data"}) + ->Repetitions(3) + ->ReportAggregatesOnly(true); + +BENCHMARK(BM_arrow_convert_multiple_record_batches_to_segment) + ->Name("BM_arrow_convert_multiple_record_batches_to_segment_short_and_wide_string") ->Args({10, 100'000, 10, false}) -// Long and thin - Windows CI can't handle this + ->ArgNames({"num_rows", "num_columns", "num_record_batches", "numeric_data"}) + ->MinWarmUpTime(0.5) + ->Repetitions(6) + ->ReportAggregatesOnly(true); + +// Windows CI can't handle this #ifndef WIN32 +BENCHMARK(BM_arrow_convert_multiple_record_batches_to_segment) + ->Name("BM_arrow_convert_multiple_record_batches_to_segment_long_and_thin_string") ->Args({10'000'000, 10, 100, false}) ->Args({10'000'000, 10, 10'000, false}) + ->ArgNames({"num_rows", "num_columns", "num_record_batches", "numeric_data"}) + ->MinWarmUpTime(0.5) + ->MinTime(4.0) + ->Repetitions(6) + ->ReportAggregatesOnly(true); #endif - // Highly fragmented - ->Args({1'000, 10, 1'000, false}); + +BENCHMARK(BM_arrow_convert_multiple_record_batches_to_segment) + ->Name("BM_arrow_convert_multiple_record_batches_to_segment_string_fragmented") + ->Args({1'000, 10, 1'000, false}) + ->ArgNames({"num_rows", "num_columns", "num_record_batches", "numeric_data"}) + ->Repetitions(3) + ->ReportAggregatesOnly(true); diff --git a/cpp/arcticdb/column_store/test/benchmark_chunked_buffer.cpp b/cpp/arcticdb/column_store/test/benchmark_chunked_buffer.cpp index dd121d02ca0..db50f7a6844 100644 --- a/cpp/arcticdb/column_store/test/benchmark_chunked_buffer.cpp +++ b/cpp/arcticdb/column_store/test/benchmark_chunked_buffer.cpp @@ -45,17 +45,41 @@ static void BM_chunked_buffer_random_access(benchmark::State& state) { } BENCHMARK(BM_chunked_buffer_allocate_with_ensure) + ->Name("BM_chunked_buffer_allocate_with_ensure_100k") ->Args({100'000, 203, true, static_cast(entity::AllocationType::DYNAMIC)}) ->Args({100'000, 203, false, static_cast(entity::AllocationType::DYNAMIC)}) ->Args({100'000, 203, false, static_cast(entity::AllocationType::DETACHABLE)}) + ->Repetitions(3) + ->ReportAggregatesOnly(true); + +BENCHMARK(BM_chunked_buffer_allocate_with_ensure) + ->Name("BM_chunked_buffer_allocate_with_ensure_10k") ->Args({10'000, 2003, true, static_cast(entity::AllocationType::DYNAMIC)}) ->Args({10'000, 2003, false, static_cast(entity::AllocationType::DYNAMIC)}) - ->Args({10'000, 2003, false, static_cast(entity::AllocationType::DETACHABLE)}); + ->Args({10'000, 2003, false, static_cast(entity::AllocationType::DETACHABLE)}) + ->Repetitions(3) + ->ReportAggregatesOnly(true); +// Sub-microsecond per call — measurement-precision limited. Fixed Iterations() forces +// many inner samples per rep so per-rep mean has tighter CI than auto-tuned MinTime. BENCHMARK(BM_chunked_buffer_random_access) + ->Name("BM_chunked_buffer_random_access_100k") ->Args({100'000, 203, true, static_cast(entity::AllocationType::DYNAMIC)}) ->Args({100'000, 203, false, static_cast(entity::AllocationType::DYNAMIC)}) ->Args({100'000, 203, false, static_cast(entity::AllocationType::DETACHABLE)}) + ->MinWarmUpTime(0.5) + ->Iterations(10'000'000) + ->Repetitions(6) + ->ReportAggregatesOnly(true); + +// 30ns per call — pure measurement-precision noise. Fixed Iterations() forces +// many inner samples per rep so per-rep mean has tighter CI than auto-tuned MinTime. +BENCHMARK(BM_chunked_buffer_random_access) + ->Name("BM_chunked_buffer_random_access_10k") ->Args({10'000, 2003, true, static_cast(entity::AllocationType::DYNAMIC)}) ->Args({10'000, 2003, false, static_cast(entity::AllocationType::DYNAMIC)}) - ->Args({10'000, 2003, false, static_cast(entity::AllocationType::DETACHABLE)}); + ->Args({10'000, 2003, false, static_cast(entity::AllocationType::DETACHABLE)}) + ->MinWarmUpTime(0.5) + ->Iterations(10'000'000) + ->Repetitions(9) + ->ReportAggregatesOnly(true); diff --git a/cpp/arcticdb/column_store/test/benchmark_column.cpp b/cpp/arcticdb/column_store/test/benchmark_column.cpp index 47b9a344e95..0eb1df57fc6 100644 --- a/cpp/arcticdb/column_store/test/benchmark_column.cpp +++ b/cpp/arcticdb/column_store/test/benchmark_column.cpp @@ -61,5 +61,9 @@ static void BM_search_sorted_single_value(benchmark::State& state) { } } -BENCHMARK(BM_search_sorted_random)->Args({100'000}); -BENCHMARK(BM_search_sorted_single_value)->Args({100'000, true})->Args({100'000, false}); +BENCHMARK(BM_search_sorted_random)->Args({100'000})->Repetitions(6)->ReportAggregatesOnly(true); +BENCHMARK(BM_search_sorted_single_value) + ->Args({100'000, true}) + ->Args({100'000, false}) + ->Repetitions(11) + ->ReportAggregatesOnly(true); diff --git a/cpp/arcticdb/column_store/test/benchmark_memory_segment.cpp b/cpp/arcticdb/column_store/test/benchmark_memory_segment.cpp index d53fdf06c9b..f6e1d509d8f 100644 --- a/cpp/arcticdb/column_store/test/benchmark_memory_segment.cpp +++ b/cpp/arcticdb/column_store/test/benchmark_memory_segment.cpp @@ -144,8 +144,14 @@ static void BM_iterate_with_iterator(benchmark::State& state) { // The {100k, 100} puts more weight on the sort_external part of the sort // where the {1M, 1} puts more weight on the create_jive_table part. -BENCHMARK(BM_sort_shuffled)->Args({100'000, 100})->Args({1'000'000, 1}); -BENCHMARK(BM_sort_ordered)->Args({100'000, 100}); -BENCHMARK(BM_sort_sparse)->Args({100'000, 100}); -BENCHMARK(BM_iterate_with_scalar_at)->Args({100'000, 100}); -BENCHMARK(BM_iterate_with_iterator)->Args({100'000, 100}); +BENCHMARK(BM_sort_shuffled) + ->Args({100'000, 100}) + ->Args({1'000'000, 1}) + ->Repetitions(3) + ->ReportAggregatesOnly(true); +// baseline max/min was 1.19 / 1.28; modest reps is enough. +BENCHMARK(BM_sort_ordered)->Args({100'000, 100})->Repetitions(3)->ReportAggregatesOnly(true); +BENCHMARK(BM_sort_sparse)->Args({100'000, 100})->Repetitions(3)->ReportAggregatesOnly(true); +// Naturally stable in baseline — no decoration needed. +BENCHMARK(BM_iterate_with_scalar_at)->Args({100'000, 100})->Repetitions(6)->ReportAggregatesOnly(true); +BENCHMARK(BM_iterate_with_iterator)->Args({100'000, 100})->Repetitions(6)->ReportAggregatesOnly(true); diff --git a/cpp/arcticdb/processing/test/benchmark_binary.cpp b/cpp/arcticdb/processing/test/benchmark_binary.cpp index 242fae8478b..734b9b97597 100644 --- a/cpp/arcticdb/processing/test/benchmark_binary.cpp +++ b/cpp/arcticdb/processing/test/benchmark_binary.cpp @@ -28,4 +28,7 @@ BENCHMARK(BM_regex_match) ->Args({100'000, 1'000, true}) ->Args({100'000, 1'000, false}) ->Args({100'000, 10'000, true}) - ->Args({100'000, 10'000, false}); \ No newline at end of file + ->Args({100'000, 10'000, false}) + ->MinWarmUpTime(0.5) + ->Repetitions(9) + ->ReportAggregatesOnly(true); \ No newline at end of file diff --git a/cpp/arcticdb/processing/test/benchmark_clause.cpp b/cpp/arcticdb/processing/test/benchmark_clause.cpp index 28e4216e1d1..f37ee1cb9f3 100644 --- a/cpp/arcticdb/processing/test/benchmark_clause.cpp +++ b/cpp/arcticdb/processing/test/benchmark_clause.cpp @@ -170,9 +170,10 @@ void BM_hash_grouping_string(benchmark::State& state) { } } -BENCHMARK(BM_merge_interleaved)->Args({10'000, 100}); -BENCHMARK(BM_merge_ordered)->Args({10'000, 100}); +BENCHMARK(BM_merge_interleaved)->Args({10'000, 100})->Repetitions(3)->ReportAggregatesOnly(true); +BENCHMARK(BM_merge_ordered)->Args({10'000, 100})->Repetitions(3)->ReportAggregatesOnly(true); +// Naturally stable in baseline (max/min < 1.15) — no decoration needed. BENCHMARK(BM_hash_grouping_int)->Args({100'000, 10, 2}); BENCHMARK(BM_hash_grouping_int)->Args({100'000, 10, 2})->Args({100'000, 10'000, 2}); BENCHMARK(BM_hash_grouping_int)->Args({100'000, 10, 2})->Args({100'000, 100'000, 2}); @@ -182,4 +183,6 @@ BENCHMARK(BM_hash_grouping_string) ->Args({100'000, 10, 2, 10}) ->Args({100'000, 100'000, 2, 10}) ->Args({100'000, 10, 2, 100}) - ->Args({100'000, 100'000, 2, 100}); + ->Args({100'000, 100'000, 2, 100}) + ->Repetitions(6) + ->ReportAggregatesOnly(true); diff --git a/cpp/arcticdb/processing/test/benchmark_projection.cpp b/cpp/arcticdb/processing/test/benchmark_projection.cpp index 30ce22e367e..afb9fce1ed0 100644 --- a/cpp/arcticdb/processing/test/benchmark_projection.cpp +++ b/cpp/arcticdb/processing/test/benchmark_projection.cpp @@ -90,5 +90,21 @@ static void BM_two_column_projection(benchmark::State& state) { } } -BENCHMARK(BM_single_column_projection)->Args({100})->Args({99})->Args({90})->Args({50})->Args({10})->Args({1}); -BENCHMARK(BM_two_column_projection)->Args({100})->Args({99})->Args({90})->Args({50})->Args({10})->Args({1}); +BENCHMARK(BM_single_column_projection) + ->Args({100}) + ->Args({99}) + ->Args({90}) + ->Args({50}) + ->Args({10}) + ->Args({1}) + ->Repetitions(3) + ->ReportAggregatesOnly(true); +BENCHMARK(BM_two_column_projection) + ->Args({100}) + ->Args({99}) + ->Args({90}) + ->Args({50}) + ->Args({10}) + ->Args({1}) + ->Repetitions(3) + ->ReportAggregatesOnly(true); diff --git a/cpp/arcticdb/processing/test/benchmark_ternary.cpp b/cpp/arcticdb/processing/test/benchmark_ternary.cpp index 173702a1a20..48561db766a 100644 --- a/cpp/arcticdb/processing/test/benchmark_ternary.cpp +++ b/cpp/arcticdb/processing/test/benchmark_ternary.cpp @@ -310,60 +310,110 @@ static void BM_ternary_bool_bool(benchmark::State& state) { } } -BENCHMARK(BM_ternary_bitset_bitset)->Args({100'000}); +BENCHMARK(BM_ternary_bitset_bitset)->Args({100'000})->Repetitions(3)->ReportAggregatesOnly(true); BENCHMARK(BM_ternary_bitset_bool) ->Args({100'000, true, true}) ->Args({100'000, true, false}) ->Args({100'000, false, true}) - ->Args({100'000, false, false}); -BENCHMARK(BM_ternary_numeric_dense_col_dense_col)->Args({100'000}); -BENCHMARK(BM_ternary_numeric_sparse_col_sparse_col)->Args({100'000}); -BENCHMARK(BM_ternary_numeric_dense_col_sparse_col)->Args({100'000, true})->Args({100'000, false}); + ->Args({100'000, false, false}) + ->Repetitions(3) + ->ReportAggregatesOnly(true); +BENCHMARK(BM_ternary_numeric_dense_col_dense_col)->Args({100'000})->Repetitions(3)->ReportAggregatesOnly(true); +BENCHMARK(BM_ternary_numeric_sparse_col_sparse_col)->Args({100'000})->Repetitions(3)->ReportAggregatesOnly(true); +// baseline max/min was 1.24; modest reps is enough. +BENCHMARK(BM_ternary_numeric_dense_col_sparse_col) + ->Args({100'000, true}) + ->Args({100'000, false}) + ->Repetitions(3) + ->ReportAggregatesOnly(true); BENCHMARK(BM_ternary_string_dense_col_dense_col) ->Args({100'000, 100'000, true}) ->Args({100'000, 100'000, false}) ->Args({100'000, 2, true}) - ->Args({100'000, 2, false}); + ->Args({100'000, 2, false}) + ->Repetitions(3) + ->ReportAggregatesOnly(true); BENCHMARK(BM_ternary_string_sparse_col_sparse_col) ->Args({100'000, 100'000, true}) ->Args({100'000, 100'000, false}) ->Args({100'000, 2, true}) - ->Args({100'000, 2, false}); + ->Args({100'000, 2, false}) + ->Repetitions(3) + ->ReportAggregatesOnly(true); BENCHMARK(BM_ternary_string_dense_col_sparse_col) ->Args({100'000, 100'000, true}) ->Args({100'000, 100'000, false}) ->Args({100'000, 2, true}) - ->Args({100'000, 2, false}); -BENCHMARK(BM_ternary_numeric_dense_col_val)->Args({100'000, true})->Args({100'000, false}); -BENCHMARK(BM_ternary_numeric_sparse_col_val)->Args({100'000, true})->Args({100'000, false}); + ->Args({100'000, 2, false}) + ->Repetitions(3) + ->ReportAggregatesOnly(true); +BENCHMARK(BM_ternary_numeric_dense_col_val) + ->Args({100'000, true}) + ->Args({100'000, false}) + ->MinWarmUpTime(0.5) + ->Repetitions(3) + ->ReportAggregatesOnly(true); +BENCHMARK(BM_ternary_numeric_sparse_col_val) + ->Args({100'000, true}) + ->Args({100'000, false}) + ->Repetitions(3) + ->ReportAggregatesOnly(true); BENCHMARK(BM_ternary_string_dense_col_val) ->Args({100'000, true, 100'000}) ->Args({100'000, false, 100'000}) ->Args({100'000, true, 2}) - ->Args({100'000, false, 2}); + ->Args({100'000, false, 2}) + ->MinWarmUpTime(0.5) + ->MinTime(2.0) + ->Repetitions(14) + ->ReportAggregatesOnly(true); BENCHMARK(BM_ternary_string_sparse_col_val) ->Args({100'000, true, 100'000}) ->Args({100'000, false, 100'000}) ->Args({100'000, true, 2}) - ->Args({100'000, false, 2}); -BENCHMARK(BM_ternary_numeric_dense_col_empty)->Args({100'000, true})->Args({100'000, false}); + ->Args({100'000, false, 2}) + ->Repetitions(3) + ->ReportAggregatesOnly(true); +// baseline max/min was 1.24; modest reps is enough. +BENCHMARK(BM_ternary_numeric_dense_col_empty) + ->Args({100'000, true}) + ->Args({100'000, false}) + ->Repetitions(3) + ->ReportAggregatesOnly(true); +// Naturally stable in baseline — no decoration needed. BENCHMARK(BM_ternary_numeric_sparse_col_empty)->Args({100'000, true})->Args({100'000, false}); BENCHMARK(BM_ternary_string_dense_col_empty) ->Args({100'000, true, 100'000}) ->Args({100'000, false, 100'000}) ->Args({100'000, true, 2}) - ->Args({100'000, false, 2}); + ->Args({100'000, false, 2}) + ->Repetitions(3) + ->ReportAggregatesOnly(true); BENCHMARK(BM_ternary_string_sparse_col_empty) ->Args({100'000, true, 100'000}) ->Args({100'000, false, 100'000}) ->Args({100'000, true, 2}) - ->Args({100'000, false, 2}); -BENCHMARK(BM_ternary_numeric_val_val)->Args({100'000}); -BENCHMARK(BM_ternary_string_val_val)->Args({100'000}); -BENCHMARK(BM_ternary_numeric_val_empty)->Args({100'000, true})->Args({100'000, false}); -BENCHMARK(BM_ternary_string_val_empty)->Args({100'000, true})->Args({100'000, false}); + ->Args({100'000, false, 2}) + ->Repetitions(3) + ->ReportAggregatesOnly(true); +BENCHMARK(BM_ternary_numeric_val_val)->Args({100'000})->Repetitions(3)->ReportAggregatesOnly(true); +BENCHMARK(BM_ternary_string_val_val)->Args({100'000})->Repetitions(5)->ReportAggregatesOnly(true); +BENCHMARK(BM_ternary_numeric_val_empty) + ->Args({100'000, true}) + ->Args({100'000, false}) + ->Repetitions(10) + ->ReportAggregatesOnly(true); +BENCHMARK(BM_ternary_string_val_empty) + ->Args({100'000, true}) + ->Args({100'000, false}) + ->Repetitions(3) + ->ReportAggregatesOnly(true); +// 1us per call — precision-limited; Iterations() amortises measurement noise. BENCHMARK(BM_ternary_bool_bool) ->Args({100'000, true, true}) ->Args({100'000, true, false}) ->Args({100'000, false, true}) - ->Args({100'000, false, false}); + ->Args({100'000, false, false}) + ->Iterations(1'000'000) + ->Repetitions(3) + ->ReportAggregatesOnly(true); diff --git a/cpp/arcticdb/util/test/benchmark_bitset.cpp b/cpp/arcticdb/util/test/benchmark_bitset.cpp index d6fa1d59f50..e4e9d8dc14a 100644 --- a/cpp/arcticdb/util/test/benchmark_bitset.cpp +++ b/cpp/arcticdb/util/test/benchmark_bitset.cpp @@ -45,7 +45,9 @@ BENCHMARK(BM_packed_bits_to_buffer) ->Args({1'000'000}) ->Args({10'000'000}) ->Args({100'000'000}) - ->Args({1'000'000'000}); + ->Args({1'000'000'000}) + ->Repetitions(9) + ->ReportAggregatesOnly(true); static void BM_bools_to_packed_bits(benchmark::State& state) { auto num_bools = static_cast(state.range(0)); @@ -62,4 +64,9 @@ static void BM_bools_to_packed_bits(benchmark::State& state) { } } -BENCHMARK(BM_bools_to_packed_bits)->Args({100'000})->Args({1'000'000})->Args({10'000'000}); +BENCHMARK(BM_bools_to_packed_bits) + ->Args({100'000}) + ->Args({1'000'000}) + ->Args({10'000'000}) + ->Repetitions(3) + ->ReportAggregatesOnly(true); diff --git a/python/.asv/results/benchmarks.json b/python/.asv/results/benchmarks.json index 626fa120b7b..1953f360934 100644 --- a/python/.asv/results/benchmarks.json +++ b/python/.asv/results/benchmarks.json @@ -1926,6 +1926,808 @@ "version": "41803cb351c8642d01cb2d0408295179cc4b2d24992efcda84a315e8d23b5237", "warmup_time": 0.5 }, + "cpp_microbenchmarks.BM_arrow_convert_multiple_record_batches_to_segment.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_arrow_convert_multiple_record_batches_to_segment.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'10/100000/10/1'", + "'10000000/10/100/1'", + "'10000000/10/10000/1'", + "'1000/10/1000/1'", + "'10/100000/10/0'", + "'10000000/10/100/0'", + "'10000000/10/10000/0'", + "'1000/10/1000/0'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_arrow_convert_single_record_batch_to_segment.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_arrow_convert_single_record_batch_to_segment.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'10/100000/-1/1'", + "'10/100000/0/1'", + "'10/100000/50000/1'", + "'10000000/10/-1/1'", + "'10000000/10/0/1'", + "'10000000/10/5/1'", + "'10/100000/-1/0'", + "'10/100000/0/0'", + "'10/100000/50000/0'", + "'10000000/10/-1/0'", + "'10000000/10/0/0'", + "'10000000/10/5/0'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_arrow_string_handler.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_arrow_string_handler.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'10000/1/0/0/0'", + "'100000/1/0/0/0'", + "'10000/10000/0/0/0'", + "'100000/100000/0/0/0'", + "'10000/1/5000/0/0'", + "'100000/1/50000/0/0'", + "'10000/1/10000/0/0'", + "'100000/1/100000/0/0'", + "'10000/1/0/1/0'", + "'100000/1/0/1/0'", + "'10000/10000/0/1/0'", + "'100000/100000/0/1/0'", + "'10000/1/5000/1/0'", + "'100000/1/50000/1/0'", + "'10000/1/10000/1/0'", + "'100000/1/100000/1/0'", + "'10000/1/0/2/0'", + "'100000/1/0/2/0'", + "'10000/10000/0/2/0'", + "'100000/100000/0/2/0'", + "'10000/1/5000/2/0'", + "'100000/1/50000/2/0'", + "'10000/1/10000/2/0'", + "'100000/1/100000/2/0'", + "'10000/1/0/0/1'", + "'100000/1/0/0/1'", + "'10000/10000/0/0/1'", + "'100000/100000/0/0/1'", + "'10000/1/0/1/1'", + "'100000/1/0/1/1'", + "'10000/10000/0/1/1'", + "'100000/100000/0/1/1'", + "'10000/1/0/2/1'", + "'100000/1/0/2/1'", + "'10000/10000/0/2/1'", + "'100000/100000/0/2/1'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_bools_to_packed_bits.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_bools_to_packed_bits.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000'", + "'1000000'", + "'10000000'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_chunked_buffer_allocate_with_ensure.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_chunked_buffer_allocate_with_ensure.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/203/1/0'", + "'100000/203/0/0'", + "'100000/203/0/2'", + "'10000/2003/1/0'", + "'10000/2003/0/0'", + "'10000/2003/0/2'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_chunked_buffer_random_access.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_chunked_buffer_random_access.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/203/1/0'", + "'100000/203/0/0'", + "'100000/203/0/2'", + "'10000/2003/1/0'", + "'10000/2003/0/0'", + "'10000/2003/0/2'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_hash_grouping_int_int16_t_.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_hash_grouping_int_int16_t_.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/10/2'", + "'100000/10000/2'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_hash_grouping_int_int32_t_.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_hash_grouping_int_int32_t_.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/10/2'", + "'100000/100000/2'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_hash_grouping_int_int64_t_.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_hash_grouping_int_int64_t_.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/10/2'", + "'100000/100000/2'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_hash_grouping_int_int8_t_.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_hash_grouping_int_int8_t_.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/10/2'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_hash_grouping_string.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_hash_grouping_string.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/10/2/10'", + "'100000/100000/2/10'", + "'100000/10/2/100'", + "'100000/100000/2/100'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_iterate_with_iterator.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_iterate_with_iterator.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/100'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_iterate_with_scalar_at.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_iterate_with_scalar_at.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/100'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_merge_interleaved.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_merge_interleaved.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'10000/100'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_merge_ordered.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_merge_ordered.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'10000/100'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_packed_bits_to_buffer.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_packed_bits_to_buffer.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000'", + "'1000000'", + "'10000000'", + "'100000000'", + "'1000000000'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_regex_match.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_regex_match.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/1000/1'", + "'100000/1000/0'", + "'100000/10000/1'", + "'100000/10000/0'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_search_sorted_random.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_search_sorted_random.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_search_sorted_single_value.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_search_sorted_single_value.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/1'", + "'100000/0'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_sort_ordered.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_sort_ordered.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/100'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_sort_shuffled.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_sort_shuffled.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/100'", + "'1000000/1'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_sort_sparse.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_sort_sparse.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/100'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_ternary_bitset_bitset.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_ternary_bitset_bitset.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_ternary_bitset_bool.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_ternary_bitset_bool.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/1/1'", + "'100000/1/0'", + "'100000/0/1'", + "'100000/0/0'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_ternary_bool_bool.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_ternary_bool_bool.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/1/1'", + "'100000/1/0'", + "'100000/0/1'", + "'100000/0/0'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_ternary_numeric_dense_col_dense_col.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_ternary_numeric_dense_col_dense_col.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_ternary_numeric_dense_col_empty.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_ternary_numeric_dense_col_empty.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/1'", + "'100000/0'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_ternary_numeric_dense_col_sparse_col.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_ternary_numeric_dense_col_sparse_col.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/1'", + "'100000/0'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_ternary_numeric_dense_col_val.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_ternary_numeric_dense_col_val.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/1'", + "'100000/0'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_ternary_numeric_sparse_col_empty.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_ternary_numeric_sparse_col_empty.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/1'", + "'100000/0'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_ternary_numeric_sparse_col_sparse_col.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_ternary_numeric_sparse_col_sparse_col.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_ternary_numeric_sparse_col_val.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_ternary_numeric_sparse_col_val.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/1'", + "'100000/0'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_ternary_numeric_val_empty.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_ternary_numeric_val_empty.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/1'", + "'100000/0'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_ternary_numeric_val_val.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_ternary_numeric_val_val.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_ternary_string_dense_col_dense_col.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_ternary_string_dense_col_dense_col.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/100000/1'", + "'100000/100000/0'", + "'100000/2/1'", + "'100000/2/0'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_ternary_string_dense_col_empty.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_ternary_string_dense_col_empty.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/1/100000'", + "'100000/0/100000'", + "'100000/1/2'", + "'100000/0/2'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_ternary_string_dense_col_sparse_col.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_ternary_string_dense_col_sparse_col.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/100000/1'", + "'100000/100000/0'", + "'100000/2/1'", + "'100000/2/0'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_ternary_string_dense_col_val.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_ternary_string_dense_col_val.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/1/100000'", + "'100000/0/100000'", + "'100000/1/2'", + "'100000/0/2'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_ternary_string_sparse_col_empty.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_ternary_string_sparse_col_empty.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/1/100000'", + "'100000/0/100000'", + "'100000/1/2'", + "'100000/0/2'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_ternary_string_sparse_col_sparse_col.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_ternary_string_sparse_col_sparse_col.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/100000/1'", + "'100000/100000/0'", + "'100000/2/1'", + "'100000/2/0'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_ternary_string_sparse_col_val.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_ternary_string_sparse_col_val.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/1/100000'", + "'100000/0/100000'", + "'100000/1/2'", + "'100000/0/2'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_ternary_string_val_empty.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_ternary_string_val_empty.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000/1'", + "'100000/0'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, + "cpp_microbenchmarks.BM_ternary_string_val_val.track_time_ms": { + "code": "class :\n def track_time_ms(self, params):\n full_name = f\"{func_name}/{params}\" if params is not None else func_name\n return run_one(binary, full_name)", + "name": "cpp_microbenchmarks.BM_ternary_string_val_val.track_time_ms", + "param_names": [ + "params" + ], + "params": [ + [ + "'100000'" + ] + ], + "timeout": 3600, + "type": "track", + "unit": "ms", + "version": "9e23b54b4290a12064f709786b94703933dec8d2c75335e9f7c47fb55fdcdb1e" + }, "finalize_staged_data.FinalizeStagedData.peakmem_finalize_staged_data": { "code": "class FinalizeStagedData:\n def peakmem_finalize_staged_data(self, *args):\n staged_symbols = self.lib.get_staged_symbols()\n assert self.symbol + \"-mem\" in staged_symbols\n self.lib.finalize_staged_data(self.symbol + \"-mem\", mode=StagedDataFinalizeMethod.WRITE)\n\n def setup(self, lib_for_storage, num_chunks, storage):\n self.lib = lib_for_storage[storage]\n if self.lib is None:\n raise SkipNotImplemented\n \n assert len(self.lib.list_symbols()) == 0 # check we are in a clean state\n initial_timestamp = TimestampNumber(0, self.df_generator.TIME_UNIT)\n \n list_of_chunks = [10_000] * num_chunks\n \n for suffix in (\"-time\", \"-mem\"):\n symbol = _symbol_name(num_chunks) + suffix\n stage_chunks(self.lib, symbol, self.df_generator, initial_timestamp, list_of_chunks)\n self.logger.info(f\"Created Symbol: {symbol}\")\n self.symbol = _symbol_name(num_chunks)\n\n def setup_cache(self):\n lib_for_storage = create_libraries_across_storages(self.storages)\n return lib_for_storage", "name": "finalize_staged_data.FinalizeStagedData.peakmem_finalize_staged_data", diff --git a/python/benchmarks/cpp_microbenchmarks.py b/python/benchmarks/cpp_microbenchmarks.py new file mode 100644 index 00000000000..bde4bfa5eb2 --- /dev/null +++ b/python/benchmarks/cpp_microbenchmarks.py @@ -0,0 +1,191 @@ +""" +Copyright 2026 Man Group Operations Limited + +Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt. + +As of the Change Date specified in that file, in accordance with the Business Source License, use of this +software will be governed by the Apache License, version 2.0. + +C++ micro-benchmark tracking. + +Exposes each Google Benchmark function as a separate ASV class, mirroring the +Google Benchmark structure. Parameter variants (e.g. +``BM_arrow_string_handler/10000/1/0/0/0``) become ASV parameters of the +corresponding class. + +Execution model +--------------- +The module is self-sufficient: it owns both discovery and execution. At +import time it invokes the benchmark binary with +``--benchmark_list_tests=true`` to enumerate benchmark names and registers +one ASV class per top-level Google Benchmark function. Each ``track_time_ms`` +call shells out to the binary with +``--benchmark_filter=^$ --benchmark_format=json`` and returns +``real_time`` converted to milliseconds. + +Binary path is taken from ``ARCTICDB_CPP_BENCHMARKS_BINARY`` or defaults to +``cpp/out/linux-release-build/arcticdb/benchmarks`` relative to the repo +root. If the binary is missing the module raises ``FileNotFoundError`` at +import with instructions for building it — a silent fallback would cause +asv_checks.py to regenerate ``benchmarks.json`` without the C++ entries +and produce a confusing hash mismatch on PRs. +""" + +import json +import os +import re +import subprocess +from collections import defaultdict +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] + +CPP_BENCHMARKS_BINARY = Path( + os.environ.get( + "ARCTICDB_CPP_BENCHMARKS_BINARY", + REPO_ROOT / "cpp/out/linux-release-build/arcticdb/benchmarks", + ) +) + +TIMEOUT_S = 3600 + + +def to_identifier(name: str) -> str: + """Replace non-identifier characters with '_', prepending '_' if the name starts with a digit.""" + ident = re.sub(r"[^a-zA-Z0-9_]", "_", name) + if ident and ident[0].isdigit(): + ident = "_" + ident + return ident + + +def to_milliseconds(real_time: float, unit: str, full_name: str) -> float: + """Convert a Google Benchmark real_time value to milliseconds.""" + if unit == "ns": + return real_time / 1e6 + elif unit == "us": + return real_time / 1e3 + elif unit == "ms": + return real_time + elif unit == "s": + return real_time * 1e3 + else: + raise RuntimeError(f"Unexpected time_unit {unit!r} in benchmark output for {full_name!r}") + + +def list_benchmark_names(binary: Path) -> list[str]: + """Enumerate all benchmark names (including param variants) from the binary.""" + result = subprocess.run( + [str(binary), "--benchmark_list_tests=true"], + check=True, + capture_output=True, + text=True, + ) + return [line.strip() for line in result.stdout.splitlines() if line.strip()] + + +def escape_for_benchmark_filter(name: str) -> str: + # Google Benchmark uses std::regex with ECMAScript syntax, where '-' is not + # a metacharacter outside character classes. Python's re.escape still + # escapes '-' to '\-', which std::regex rejects as "Unexpected escape + # character" — the filter fails to compile, no benchmarks run, and the + # binary exits 0 with empty stdout. + return re.sub(r"([\\^$.|?*+()\[\]{}])", r"\\\1", name) + + +def run_one(binary: Path, full_name: str) -> float: + """Run a single benchmark by exact-match filter, return its real_time in milliseconds. + + When the C++ side sets ``->Repetitions(N)->ReportAggregatesOnly(true)``, the JSON + contains only aggregate rows (mean/median/stddev/cv) and we use the median — + robust against a single outlier repetition (typically the first, which pays + cold-cache costs). When no repetitions are configured, there is exactly one + iteration row and we use that value directly. + """ + result = subprocess.run( + [ + str(binary), + f"--benchmark_filter=^{escape_for_benchmark_filter(full_name)}$", + "--benchmark_format=json", + ], + check=True, + capture_output=True, + text=True, + ) + data = json.loads(result.stdout) + benchmarks = data.get("benchmarks", []) + median_row = next( + (b for b in benchmarks if b.get("run_type") == "aggregate" and b.get("aggregate_name") == "median"), + None, + ) + if median_row is not None: + return to_milliseconds(median_row["real_time"], median_row.get("time_unit", "ms"), full_name) + iterations = [b for b in benchmarks if b.get("run_type", "iteration") == "iteration"] + if not iterations: + raise RuntimeError(f"No iteration or aggregate result returned for benchmark {full_name!r}") + bm = iterations[0] + return to_milliseconds(bm["real_time"], bm.get("time_unit", "ms"), full_name) + + +def group_benchmarks(names: list[str]) -> dict[str, list[str | None]]: + """Split each name on the first '/' into function name and variant (None if no '/').""" + groups: dict[str, list[str | None]] = defaultdict(list) + for name in names: + if "/" in name: + func, variant = name.split("/", 1) + else: + func, variant = name, None + groups[func].append(variant) + return dict(groups) + + +def make_benchmark_class(binary: Path, func_name: str, variants: list[str | None]) -> type: + """Return a dynamically created ASV benchmark class for one C++ benchmark function.""" + + def track_time_ms(self, params): + full_name = f"{func_name}/{params}" if params is not None else func_name + return run_one(binary, full_name) + + return type( + to_identifier(func_name), + (), + { + "timeout": TIMEOUT_S, + # variants is [None] for benchmarks with no parameters (from group_benchmarks). + # ASV requires params to be a non-empty list of lists, hence the outer wrap. + "params": [variants], + "param_names": ["params"], + "unit": "ms", + "track_time_ms": track_time_ms, + }, + ) + + +if not CPP_BENCHMARKS_BINARY.exists(): + raise FileNotFoundError( + f"C++ benchmark binary not found at {CPP_BENCHMARKS_BINARY}.\n" + f"\n" + f"The ASV C++ microbenchmark tracking module needs this binary to\n" + f"enumerate benchmark names at import time. Build it with:\n" + f"\n" + f" cmake --preset linux-release -DTEST=ON cpp\n" + f" cmake --build cpp/out/linux-release-build --target benchmarks\n" + f"\n" + f"Or set ARCTICDB_CPP_BENCHMARKS_BINARY to an existing binary path." + ) + +_NAMES = list_benchmark_names(CPP_BENCHMARKS_BINARY) + +# Register one ASV class per C++ benchmark function in this module's namespace. +# to_identifier() is lossy (e.g. 'BM_foo-bar' and 'BM_foo_bar' both become +# 'BM_foo_bar'), so refuse to register colliding names rather than silently +# dropping one from ASV tracking. +_registered: dict[str, str] = {} +for func_name, variants in group_benchmarks(_NAMES).items(): + ident = to_identifier(func_name) + if ident in _registered: + raise RuntimeError( + f"C++ benchmark name collision: {func_name!r} and {_registered[ident]!r} " + f"both map to Python identifier {ident!r}. Rename one of the C++ benchmarks." + ) + _registered[ident] = func_name + globals()[ident] = make_benchmark_class(CPP_BENCHMARKS_BINARY, func_name, variants)