From 45150fbbee1da6dc63306ac6c10b545921b8b057 Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Gafoor <94980075+Gafoor2005@users.noreply.github.com> Date: Tue, 19 May 2026 12:41:35 +0000 Subject: [PATCH 01/15] ci: Add span performance benchmarks (#1167) Add benchmark suite comparing gsl::span vs std::span performance across is_sorted and min_element operations. Includes GitHub Actions workflow for automated testing on GCC 14, Clang 18, and MSVC with results uploaded as artifacts for performance regression analysis. --- .github/workflows/benchmarks.yml | 133 +++++++++++++++++++++++ benchmark/CMakeLists.txt | 30 ++++++ benchmark/README.md | 83 +++++++++++++++ benchmark/span_bench.cpp | 174 +++++++++++++++++++++++++++++++ 4 files changed, 420 insertions(+) create mode 100644 .github/workflows/benchmarks.yml create mode 100644 benchmark/CMakeLists.txt create mode 100644 benchmark/README.md create mode 100644 benchmark/span_bench.cpp diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 00000000..0e9d8907 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,133 @@ +name: Performance Benchmarks + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + benchmarks-gcc: + strategy: + matrix: + gcc_version: [ 14 ] + build_type: [ Release ] + cxx_version: [ 20 ] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Install GCC + run: | + sudo apt-get update + sudo apt-get install -y g++-${{ matrix.gcc_version }} + + - name: Configure CMake for benchmarks + working-directory: benchmark + run: | + mkdir -p build + cd build + cmake .. \ + -DCMAKE_CXX_COMPILER=g++-${{ matrix.gcc_version }} \ + -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ + -DCMAKE_CXX_STANDARD=${{ matrix.cxx_version }} + + - name: Build benchmarks + working-directory: benchmark/build + run: cmake --build . --config ${{ matrix.build_type }} + + - name: Run benchmarks + working-directory: benchmark/build + run: ./span_bench --benchmark_format=json > benchmark_results.json 2>&1 | tee benchmark_output.txt + continue-on-error: true + + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + with: + name: benchmark-results-gcc-${{ matrix.gcc_version }}-cxx${{ matrix.cxx_version }} + path: | + benchmark/build/benchmark_results.json + benchmark/build/benchmark_output.txt + + benchmarks-clang: + strategy: + matrix: + clang_version: [ 18 ] + build_type: [ Release ] + cxx_version: [ 20 ] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Install Clang + run: | + sudo apt-get update + sudo apt-get install -y clang-${{ matrix.clang_version }} + + - name: Configure CMake for benchmarks + working-directory: benchmark + run: | + mkdir -p build + cd build + cmake .. \ + -DCMAKE_CXX_COMPILER=clang++-${{ matrix.clang_version }} \ + -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ + -DCMAKE_CXX_STANDARD=${{ matrix.cxx_version }} + + - name: Build benchmarks + working-directory: benchmark/build + run: cmake --build . --config ${{ matrix.build_type }} + + - name: Run benchmarks + working-directory: benchmark/build + run: ./span_bench --benchmark_format=json > benchmark_results.json 2>&1 | tee benchmark_output.txt + continue-on-error: true + + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + with: + name: benchmark-results-clang-${{ matrix.clang_version }}-cxx${{ matrix.cxx_version }} + path: | + benchmark/build/benchmark_results.json + benchmark/build/benchmark_output.txt + + benchmarks-msvc: + strategy: + matrix: + image: [ windows-2025 ] + build_type: [ Release ] + cxx_version: [ 20 ] + runs-on: ${{ matrix.image }} + steps: + - uses: actions/checkout@v6 + + - name: Configure CMake for benchmarks + working-directory: benchmark + run: | + mkdir build + cd build + cmake .. -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DCMAKE_CXX_STANDARD=${{ matrix.cxx_version }} + + - name: Build benchmarks + working-directory: benchmark/build + run: cmake --build . --config ${{ matrix.build_type }} + + - name: Run benchmarks + working-directory: benchmark/build + run: | + $config = "${{ matrix.build_type }}" + & ".\$config\span_bench.exe" --benchmark_format=json > benchmark_results.json 2>&1 | Tee-Object -FilePath benchmark_output.txt + shell: pwsh + continue-on-error: true + + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + with: + name: benchmark-results-msvc-${{ matrix.image }}-cxx${{ matrix.cxx_version }} + path: | + benchmark/build/benchmark_results.json + benchmark/build/benchmark_output.txt diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt new file mode 100644 index 00000000..b6739512 --- /dev/null +++ b/benchmark/CMakeLists.txt @@ -0,0 +1,30 @@ +cmake_minimum_required(VERSION 3.20) + +project(myproject C CXX) +set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type") # Default build type + +set(BENCHMARK_ENABLE_TESTING off) # to suppress benchmark internal tests +set(CMAKE_CXX_STANDARD 20 CACHE STRING "C++ standard for benchmarks") +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +include(FetchContent) +FetchContent_Declare(googletest + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG release-1.12.1) # or "main" for latest +FetchContent_Declare(googlebenchmark + GIT_REPOSITORY https://github.com/google/benchmark.git + GIT_TAG v1.9.0) # or "main" for latest +FetchContent_Declare(GSL + GIT_REPOSITORY https://github.com/microsoft/GSL.git + GIT_TAG v4.1.0 + GIT_SHALLOW ON) + + +FetchContent_MakeAvailable(googletest googlebenchmark GSL) + + +add_executable(span_bench span_bench.cpp) +target_link_libraries(span_bench PRIVATE benchmark::benchmark) +target_link_libraries(span_bench PRIVATE Microsoft.GSL::GSL) + diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 00000000..8e85dd90 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,83 @@ +# GSL Performance Benchmarks + +This directory contains performance benchmarks comparing `gsl::span` with `std::span` across various workloads and compiler implementations. + +## Overview + +The benchmarks test critical span operations to ensure `gsl::span` maintains **performance parity** with `std::span`, particularly when [Safe Buffers](https://clang.llvm.org/docs/SafeBuffers.html) are enabled. + +### Tested Operations + +- **`is_sorted`**: Custom iterator-based check, `std::is_sorted`, and `std::ranges::is_sorted` +- **`min_element`**: Algorithm-based and range-for iteration approaches + +Each test compares both `std::span` and `gsl::span` implementations. + +## Building Locally + +### Prerequisites + +- CMake 3.20+ +- C++20 compiler or newer (GCC, Clang, or MSVC) +- Google Benchmark (fetched automatically via CMake) +- Google Test (fetched automatically via CMake) + +### Build Steps + +```bash +# From the repository root +cd benchmark +mkdir -p build +cd build + +# Configure with Release build type (important for accurate benchmarks) +cmake .. -DCMAKE_BUILD_TYPE=Release + +# Build +cmake --build . --config Release +``` + +## Running Benchmarks + +```bash +# Run all benchmarks with default output +./span_bench + +# Run with JSON output (useful for analysis) +./span_bench --benchmark_format=json > results.json + +# Run specific benchmark (regex matching) +./span_bench --benchmark_filter="IsSorted" + +# Show detailed statistics +./span_bench --benchmark_report_aggregates_only=true +``` + +For more options, see [Google Benchmark documentation](https://github.com/google/benchmark#usage). + +## CI Integration + +Benchmarks automatically run on every pull request across: +- **Compilers**: GCC 14 (latest), Clang 18 (latest), MSVC (latest) +- **Standards**: C++20 +- **Platforms**: Linux (GCC/Clang), Windows (MSVC on windows-2025) + +Results are uploaded as GitHub Actions artifacts for easy download and analysis. + +Results are uploaded as GitHub Actions artifacts and can be analyzed for performance regressions. + +## Performance Analysis + +When comparing results: +1. Always compare the same compiler, C++ standard, and platform +2. Release builds should be used for accurate measurements +3. Significant differences (>5%) between `std::span` and `gsl::span` warrant investigation +4. Account for noise in CI environments when analyzing small variations + +## Contributing + +When adding new benchmarks: +1. Follow the existing pattern in `span_bench.cpp` +2. Compare both `std::span` and `gsl::span` implementations +3. Use `benchmark::DoNotOptimize` to prevent compiler optimizations from skewing results +4. Document what the benchmark tests and why it matters diff --git a/benchmark/span_bench.cpp b/benchmark/span_bench.cpp new file mode 100644 index 00000000..a5f9e4a0 --- /dev/null +++ b/benchmark/span_bench.cpp @@ -0,0 +1,174 @@ +// Setup: +// 1) cmake . -DCMAKE_BUILD_TYPE=Release +// 2) cmake --build . --config Release + +#include "gsl/span" +#include +#include + +static std::vector make_vector() +{ + std::vector v; + constexpr size_t vec_size = 1000; + v.reserve(vec_size); + for (int i = 0; i < vec_size; ++i) + v.push_back(i); + + return v; +} + +template +bool custom_is_sorted(ForwardIt first, ForwardIt last) +{ + if (first != last) + { + ForwardIt next = first; + while (++next != last) + { + if (*next < *first) + return false; + first = next; + } + } + return true; +} + +static void BM_IsSortedCustomStdSpan(benchmark::State& state) +{ + auto vec = make_vector(); + + for (auto _ : state) + { + std::span sp = vec; + benchmark::DoNotOptimize(custom_is_sorted(sp.begin(), sp.end())); + } +} + +static void BM_IsSortedCustomGslSpan(benchmark::State& state) +{ + auto vec = make_vector(); + + for (auto _ : state) + { + gsl::span sp = vec; + benchmark::DoNotOptimize(custom_is_sorted(sp.begin(), sp.end())); + } +} + +static void BM_IsSortedStdSpan(benchmark::State& state) +{ + auto vec = make_vector(); + + for (auto _ : state) + { + std::span sp = vec; + benchmark::DoNotOptimize(std::is_sorted(sp.begin(), sp.end())); + } +} + +static void BM_IsSortedGslSpan(benchmark::State& state) +{ + auto vec = make_vector(); + + for (auto _ : state) + { + gsl::span sp = vec; + benchmark::DoNotOptimize(std::is_sorted(sp.begin(), sp.end())); + } +} + +static void BM_IsSortedRangesStdSpan(benchmark::State& state) +{ + auto vec = make_vector(); + + for (auto _ : state) + { + std::span sp = vec; + benchmark::DoNotOptimize(std::ranges::is_sorted(sp)); + } +} + + +static void BM_IsSortedRangesGslSpan(benchmark::State& state) +{ + auto vec = make_vector(); + + for (auto _ : state) + { + gsl::span sp = vec; + benchmark::DoNotOptimize(std::ranges::is_sorted(sp)); + } +} + +BENCHMARK(BM_IsSortedStdSpan); +BENCHMARK(BM_IsSortedGslSpan); +BENCHMARK(BM_IsSortedRangesStdSpan); +BENCHMARK(BM_IsSortedRangesGslSpan); +BENCHMARK(BM_IsSortedCustomStdSpan); +BENCHMARK(BM_IsSortedCustomGslSpan); + +template +static int CustomMinElement(TSpan span) +{ + auto min = std::numeric_limits::max(); + for (int e : span) + { + if (e < min) + min = e; + } + return min; +} + +static void BM_MinElementAlgorithmStdSpan(benchmark::State& state) +{ + auto vec = make_vector(); + + for (auto _ : state) + { + std::span sp = vec; + benchmark::DoNotOptimize(std::min_element(sp.begin(), sp.end())); + } +} + +static void BM_MinElementAlgorithmGslSpan(benchmark::State& state) +{ + auto vec = make_vector(); + + for (auto _ : state) + { + gsl::span sp = vec; + benchmark::DoNotOptimize(std::min_element(sp.begin(), sp.end())); + } +} + +static void BM_MinElementRangeForStdSpan(benchmark::State& state) +{ + auto vec = make_vector(); + + for (auto _ : state) + { + std::span sp = vec; + benchmark::DoNotOptimize(CustomMinElement(sp)); + } +} + + +static void BM_MinElementRangeForGslSpan(benchmark::State& state) +{ + auto vec = make_vector(); + + for (auto _ : state) + { + gsl::span sp = vec; + benchmark::DoNotOptimize(CustomMinElement(sp)); + } +} + + +BENCHMARK(BM_MinElementAlgorithmStdSpan); +BENCHMARK(BM_MinElementAlgorithmGslSpan); +BENCHMARK(BM_MinElementRangeForStdSpan); +BENCHMARK(BM_MinElementRangeForGslSpan); + +BENCHMARK_MAIN(); + From 34f9be74df60668c21758d044d49b51381da5954 Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Gafoor <94980075+Gafoor2005@users.noreply.github.com> Date: Tue, 19 May 2026 12:52:26 +0000 Subject: [PATCH 02/15] ci: Fix benchmark output artifact capture Ensure both text and JSON benchmark outputs are properly populated in workflow artifacts by running benchmarks separately instead of using pipe redirection, which was leaving the text file empty. --- .github/workflows/benchmarks.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 0e9d8907..77bcaa03 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -42,7 +42,8 @@ jobs: - name: Run benchmarks working-directory: benchmark/build - run: ./span_bench --benchmark_format=json > benchmark_results.json 2>&1 | tee benchmark_output.txt + run: |\n ./span_bench > benchmark_output.txt 2>&1 + ./span_bench --benchmark_format=json > benchmark_results.json 2>/dev/null continue-on-error: true - name: Upload benchmark results @@ -84,7 +85,9 @@ jobs: - name: Run benchmarks working-directory: benchmark/build - run: ./span_bench --benchmark_format=json > benchmark_results.json 2>&1 | tee benchmark_output.txt + run: | + ./span_bench > benchmark_output.txt 2>&1 + ./span_bench --benchmark_format=json > benchmark_results.json 2>/dev/null continue-on-error: true - name: Upload benchmark results @@ -120,7 +123,8 @@ jobs: working-directory: benchmark/build run: | $config = "${{ matrix.build_type }}" - & ".\$config\span_bench.exe" --benchmark_format=json > benchmark_results.json 2>&1 | Tee-Object -FilePath benchmark_output.txt + & ".\$config\span_bench.exe" | Tee-Object -FilePath benchmark_output.txt | Out-Null + & ".\$config\span_bench.exe" --benchmark_format=json | Out-File -FilePath benchmark_results.json shell: pwsh continue-on-error: true From c78ac6d3e883ad4cad306a0d71ae817609c0ee9d Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Gafoor <94980075+Gafoor2005@users.noreply.github.com> Date: Tue, 19 May 2026 13:00:19 +0000 Subject: [PATCH 03/15] ci: Fix YAML syntax in benchmark workflow --- .github/workflows/benchmarks.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 77bcaa03..31c995e8 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -42,7 +42,8 @@ jobs: - name: Run benchmarks working-directory: benchmark/build - run: |\n ./span_bench > benchmark_output.txt 2>&1 + run: | + ./span_bench > benchmark_output.txt 2>&1 ./span_bench --benchmark_format=json > benchmark_results.json 2>/dev/null continue-on-error: true From c5b216e9a92dab0e7111676cba42b15b798d2364 Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Gafoor <94980075+Gafoor2005@users.noreply.github.com> Date: Tue, 19 May 2026 14:05:24 +0000 Subject: [PATCH 04/15] ci: Add PR comments for benchmark results and update upload-artifact to v5 --- .github/workflows/benchmarks.yml | 48 ++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 31c995e8..76968af3 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -48,13 +48,27 @@ jobs: continue-on-error: true - name: Upload benchmark results - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: benchmark-results-gcc-${{ matrix.gcc_version }}-cxx${{ matrix.cxx_version }} path: | benchmark/build/benchmark_results.json benchmark/build/benchmark_output.txt + - name: Comment PR with benchmark results + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const benchmark_output = fs.readFileSync('benchmark/build/benchmark_output.txt', 'utf8'); + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `## Benchmark Results - GCC ${{ matrix.gcc_version }} (C++${{ matrix.cxx_version }})\n\n\`\`\`\n${benchmark_output}\n\`\`\`` + }); + benchmarks-clang: strategy: matrix: @@ -92,13 +106,27 @@ jobs: continue-on-error: true - name: Upload benchmark results - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: benchmark-results-clang-${{ matrix.clang_version }}-cxx${{ matrix.cxx_version }} path: | benchmark/build/benchmark_results.json benchmark/build/benchmark_output.txt + - name: Comment PR with benchmark results + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const benchmark_output = fs.readFileSync('benchmark/build/benchmark_output.txt', 'utf8'); + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `## Benchmark Results - Clang ${{ matrix.clang_version }} (C++${{ matrix.cxx_version }})\n\n\`\`\`\n${benchmark_output}\n\`\`\`` + }); + benchmarks-msvc: strategy: matrix: @@ -130,9 +158,23 @@ jobs: continue-on-error: true - name: Upload benchmark results - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: benchmark-results-msvc-${{ matrix.image }}-cxx${{ matrix.cxx_version }} path: | benchmark/build/benchmark_results.json benchmark/build/benchmark_output.txt + + - name: Comment PR with benchmark results + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const benchmark_output = fs.readFileSync('benchmark/build/benchmark_output.txt', 'utf8'); + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `## Benchmark Results - MSVC on ${{ matrix.image }} (C++${{ matrix.cxx_version }})\n\n\`\`\`\n${benchmark_output}\n\`\`\`` + }); From 511a2e2df54fb893c864bda4b437a65371dfef3c Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Gafoor <94980075+Gafoor2005@users.noreply.github.com> Date: Fri, 29 May 2026 10:45:47 +0000 Subject: [PATCH 05/15] added benchmarks ci with gsl/std ratio comparision --- .github/workflows/benchmarks.yml | 465 +++++++++++++++++++++---------- benchmark/CMakeLists.txt | 93 +++++-- benchmark/check_regression.py | 290 +++++++++++++++++++ 3 files changed, 689 insertions(+), 159 deletions(-) create mode 100644 benchmark/check_regression.py diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 76968af3..97e2263f 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -1,180 +1,363 @@ -name: Performance Benchmarks +# .github/workflows/span_benchmark.yml +# +# Benchmarks gsl::span vs std::span on every PR across all supported compilers. +# Strategy: in-run ratio (gsl_ns / std_ns) — noise-resistant because both +# spans run in the same process, same machine, same moment. +# +# All dependencies (google-benchmark v1.9.0, GSL v4.1.0) are pulled via +# FetchContent inside the repo's CMakeLists.txt — no separate install steps. +# The build/_deps directory is cached, keyed on CMakeLists.txt hash, so +# repeated runs don't re-clone anything. +# +# Three separate job groups (one per OS) because `runs-on` can't share a +# matrix with OS-specific shell/path differences cleanly: +# +# benchmark-linux → ubuntu-latest → GCC-13, GCC-14, Clang-17, Clang-18 +# benchmark-windows → windows-latest → MSVC 2022, clang-cl +# benchmark-macos → macos-14 → Apple Clang (Xcode latest) +# +# A final `comment` job waits for all three groups, collects every JSON +# artifact, runs check_regression.py, and posts (or updates) one PR comment. -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true +name: Span Benchmark on: - push: - branches: [ main ] pull_request: - branches: [ main ] + branches: [main] + +# Cancel stale runs when a new commit is pushed to the same PR. +concurrency: + group: span-bench-${{ github.event.pull_request.number }} + cancel-in-progress: true jobs: - benchmarks-gcc: + +# ───────────────────────────────────────────────────────────────────────────── +# LINUX — GCC 13/14 · Clang 17/18 (× C++20 and C++23) +# ───────────────────────────────────────────────────────────────────────────── + benchmark-linux: + name: Linux / ${{ matrix.label }} + runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - gcc_version: [ 14 ] - build_type: [ Release ] - cxx_version: [ 20 ] - runs-on: ubuntu-latest + include: + # ── GCC ──────────────────────────────────────────────────────── + - label: GCC-13-cpp20 + cxx: g++-13 + cppstd: 20 + extra_flags: "" + + - label: GCC-14-cpp20 + cxx: g++-14 + cppstd: 20 + extra_flags: "" + + - label: GCC-14-cpp23 + cxx: g++-14 + cppstd: 23 + extra_flags: "" + + # ── Clang ────────────────────────────────────────────────────── + - label: Clang-17-cpp20 + cxx: clang++-17 + cppstd: 20 + extra_flags: "" + + - label: Clang-18-cpp20 + cxx: clang++-18 + cppstd: 20 + extra_flags: "" + + - label: Clang-18-cpp23 + cxx: clang++-18 + cppstd: 23 + extra_flags: "" + + # ── Clang + -fbounds-safety (core motivation of issue #1167) ─── + # Safe Buffers enforcement is what triggered this whole tracking effort. + - label: Clang-18-cpp20-bounds-safety + cxx: clang++-18 + cppstd: 20 + extra_flags: "-fbounds-safety" + steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 - - name: Install GCC + # ── Compiler install ─────────────────────────────────────────────── + # GCC-13 and Clang pre-17 are already on ubuntu-latest (24.04). + - name: Install GCC 14 + if: startsWith(matrix.cxx, 'g++-14') run: | - sudo apt-get update - sudo apt-get install -y g++-${{ matrix.gcc_version }} + sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y + sudo apt-get update -qq + sudo apt-get install -y g++-14 - - name: Configure CMake for benchmarks - working-directory: benchmark + - name: Install Clang 17 + if: matrix.cxx == 'clang++-17' run: | - mkdir -p build - cd build - cmake .. \ - -DCMAKE_CXX_COMPILER=g++-${{ matrix.gcc_version }} \ - -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ - -DCMAKE_CXX_STANDARD=${{ matrix.cxx_version }} - - - name: Build benchmarks - working-directory: benchmark/build - run: cmake --build . --config ${{ matrix.build_type }} - - - name: Run benchmarks - working-directory: benchmark/build + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key \ + | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc + echo "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-17 main" \ + | sudo tee /etc/apt/sources.list.d/llvm-17.list + sudo apt-get update -qq && sudo apt-get install -y clang-17 + + - name: Install Clang 18 + if: matrix.cxx == 'clang++-18' run: | - ./span_bench > benchmark_output.txt 2>&1 - ./span_bench --benchmark_format=json > benchmark_results.json 2>/dev/null - continue-on-error: true + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key \ + | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc + echo "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-18 main" \ + | sudo tee /etc/apt/sources.list.d/llvm-18.list + sudo apt-get update -qq && sudo apt-get install -y clang-18 - - name: Upload benchmark results - uses: actions/upload-artifact@v5 + # ── FetchContent cache ───────────────────────────────────────────── + # Caches the cloned sources for google-benchmark, googletest, and GSL. + # Key is the hash of CMakeLists.txt — busts automatically on any + # GIT_TAG bump without manual intervention. + - name: Cache FetchContent dependencies + uses: actions/cache@v4 with: - name: benchmark-results-gcc-${{ matrix.gcc_version }}-cxx${{ matrix.cxx_version }} - path: | - benchmark/build/benchmark_results.json - benchmark/build/benchmark_output.txt - - - name: Comment PR with benchmark results - if: github.event_name == 'pull_request' - uses: actions/github-script@v7 + path: build/_deps + key: fetchcontent-linux-${{ matrix.cxx }}-${{ hashFiles('CMakeLists.txt') }} + restore-keys: | + fetchcontent-linux-${{ matrix.cxx }}- + + # ── Configure → Build → Run ──────────────────────────────────────── + # FetchContent handles benchmark + GSL — no separate install needed. + # -DCMAKE_CXX_STANDARD overrides the default C++20 set in CMakeLists + # so we can test C++23 configs from the matrix. + - name: Configure + run: | + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \ + -DCMAKE_CXX_STANDARD=${{ matrix.cppstd }} \ + -DCMAKE_CXX_FLAGS="${{ matrix.extra_flags }}" + + - name: Build + run: cmake --build build --target span_bench -j$(nproc) + + # 10 repetitions → mean + stddev aggregates in the JSON output. + # benchmark_report_aggregates_only keeps the file compact. + - name: Run benchmark + run: | + ./build/span_bench \ + --benchmark_format=json \ + --benchmark_repetitions=10 \ + --benchmark_report_aggregates_only=true \ + --benchmark_out=results_${{ matrix.label }}.json + + - name: Upload results + uses: actions/upload-artifact@v4 with: - script: | - const fs = require('fs'); - const benchmark_output = fs.readFileSync('benchmark/build/benchmark_output.txt', 'utf8'); - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: `## Benchmark Results - GCC ${{ matrix.gcc_version }} (C++${{ matrix.cxx_version }})\n\n\`\`\`\n${benchmark_output}\n\`\`\`` - }); - - benchmarks-clang: + name: bench-${{ matrix.label }} + path: results_${{ matrix.label }}.json + retention-days: 7 + +# ───────────────────────────────────────────────────────────────────────────── +# WINDOWS — MSVC 2022 · clang-cl (× C++20 and C++23) +# +# Notes: +# • Uses the Visual Studio 17 2022 generator for both MSVC and clang-cl. +# The -T ClangCL toolset switch selects the clang-cl frontend that ships +# bundled with VS 2022 — no extra install needed. +# • Multi-config generator: output binary is at +# build\Release\span_bench.exe (FetchContent flat layout, no subdir). +# • FetchContent cache is keyed per-toolset because MSVC and clang-cl +# produce ABI-incompatible object files. +# • PowerShell is used throughout (shell: pwsh) for consistent quoting. +# ───────────────────────────────────────────────────────────────────────────── + benchmark-windows: + name: Windows / ${{ matrix.label }} + runs-on: windows-latest strategy: + fail-fast: false matrix: - clang_version: [ 18 ] - build_type: [ Release ] - cxx_version: [ 20 ] - runs-on: ubuntu-latest + include: + # ── MSVC 2022 ────────────────────────────────────────────────── + - label: MSVC-2022-cpp20 + toolset: "" + cppstd: 20 + extra_flags: "" + + - label: MSVC-2022-cpp23 + toolset: "" + cppstd: 23 + extra_flags: "" + + # ── clang-cl (bundled with VS 2022) ──────────────────────────── + - label: clang-cl-cpp20 + toolset: "ClangCL" + cppstd: 20 + extra_flags: "" + + - label: clang-cl-cpp23 + toolset: "ClangCL" + cppstd: 23 + extra_flags: "" + steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 - - name: Install Clang + # ── FetchContent cache ───────────────────────────────────────────── + - name: Cache FetchContent dependencies + uses: actions/cache@v4 + with: + path: build/_deps + key: fetchcontent-windows-${{ matrix.toolset }}-${{ hashFiles('CMakeLists.txt') }} + restore-keys: | + fetchcontent-windows-${{ matrix.toolset }}- + + # ── Configure → Build → Run ──────────────────────────────────────── + - name: Configure + shell: pwsh run: | - sudo apt-get update - sudo apt-get install -y clang-${{ matrix.clang_version }} + $tsArg = if ("${{ matrix.toolset }}" -ne "") { @("-T", "${{ matrix.toolset }}") } else { @() } + + cmake -S . -B build ` + -G "Visual Studio 17 2022" @tsArg ` + -DCMAKE_CXX_STANDARD=${{ matrix.cppstd }} ` + -DCMAKE_CXX_FLAGS="${{ matrix.extra_flags }}" - - name: Configure CMake for benchmarks - working-directory: benchmark + - name: Build + shell: pwsh run: | - mkdir -p build - cd build - cmake .. \ - -DCMAKE_CXX_COMPILER=clang++-${{ matrix.clang_version }} \ - -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ - -DCMAKE_CXX_STANDARD=${{ matrix.cxx_version }} - - - name: Build benchmarks - working-directory: benchmark/build - run: cmake --build . --config ${{ matrix.build_type }} - - - name: Run benchmarks - working-directory: benchmark/build + cmake --build build --target span_bench ` + --config Release -j $env:NUMBER_OF_PROCESSORS + + # Multi-config generator places the binary under build\Release\ + - name: Run benchmark + shell: pwsh run: | - ./span_bench > benchmark_output.txt 2>&1 - ./span_bench --benchmark_format=json > benchmark_results.json 2>/dev/null - continue-on-error: true + .\build\Release\span_bench.exe ` + --benchmark_format=json ` + --benchmark_repetitions=10 ` + --benchmark_report_aggregates_only=true ` + --benchmark_out=results_${{ matrix.label }}.json - - name: Upload benchmark results - uses: actions/upload-artifact@v5 - with: - name: benchmark-results-clang-${{ matrix.clang_version }}-cxx${{ matrix.cxx_version }} - path: | - benchmark/build/benchmark_results.json - benchmark/build/benchmark_output.txt - - - name: Comment PR with benchmark results - if: github.event_name == 'pull_request' - uses: actions/github-script@v7 + - name: Upload results + uses: actions/upload-artifact@v4 with: - script: | - const fs = require('fs'); - const benchmark_output = fs.readFileSync('benchmark/build/benchmark_output.txt', 'utf8'); - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: `## Benchmark Results - Clang ${{ matrix.clang_version }} (C++${{ matrix.cxx_version }})\n\n\`\`\`\n${benchmark_output}\n\`\`\`` - }); - - benchmarks-msvc: + name: bench-${{ matrix.label }} + path: results_${{ matrix.label }}.json + retention-days: 7 + +# ───────────────────────────────────────────────────────────────────────────── +# MACOS — Apple Clang via Xcode (latest) (× C++20 and C++23) +# +# Notes: +# • macos-14 = Apple Silicon (M1). Switch to macos-13 for Intel if needed. +# • No extra compiler install — Apple Clang from Xcode is used directly. +# • -j uses sysctl (macOS equivalent of nproc). +# ───────────────────────────────────────────────────────────────────────────── + benchmark-macos: + name: macOS / ${{ matrix.label }} + runs-on: macos-14 strategy: + fail-fast: false matrix: - image: [ windows-2025 ] - build_type: [ Release ] - cxx_version: [ 20 ] - runs-on: ${{ matrix.image }} + include: + - label: AppleClang-cpp20 + cppstd: 20 + extra_flags: "" + + - label: AppleClang-cpp23 + cppstd: 23 + extra_flags: "" + steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 + + # ── FetchContent cache ───────────────────────────────────────────── + - name: Cache FetchContent dependencies + uses: actions/cache@v4 + with: + path: build/_deps + key: fetchcontent-macos-${{ matrix.cppstd }}-${{ hashFiles('CMakeLists.txt') }} + restore-keys: | + fetchcontent-macos-${{ matrix.cppstd }}- - - name: Configure CMake for benchmarks - working-directory: benchmark + # ── Configure → Build → Run ──────────────────────────────────────── + - name: Configure run: | - mkdir build - cd build - cmake .. -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DCMAKE_CXX_STANDARD=${{ matrix.cxx_version }} + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_STANDARD=${{ matrix.cppstd }} \ + -DCMAKE_CXX_FLAGS="${{ matrix.extra_flags }}" - - name: Build benchmarks - working-directory: benchmark/build - run: cmake --build . --config ${{ matrix.build_type }} + - name: Build + run: cmake --build build --target span_bench -j$(sysctl -n hw.logicalcpu) - - name: Run benchmarks - working-directory: benchmark/build + - name: Run benchmark run: | - $config = "${{ matrix.build_type }}" - & ".\$config\span_bench.exe" | Tee-Object -FilePath benchmark_output.txt | Out-Null - & ".\$config\span_bench.exe" --benchmark_format=json | Out-File -FilePath benchmark_results.json - shell: pwsh + ./build/span_bench \ + --benchmark_format=json \ + --benchmark_repetitions=10 \ + --benchmark_report_aggregates_only=true \ + --benchmark_out=results_${{ matrix.label }}.json + + - name: Upload results + uses: actions/upload-artifact@v4 + with: + name: bench-${{ matrix.label }} + path: results_${{ matrix.label }}.json + retention-days: 7 + +# ───────────────────────────────────────────────────────────────────────────── +# COMMENT — collect every result artifact → one PR comment +# +# Waits for all three OS groups. `if: always()` ensures this runs even when +# some benchmark jobs fail — partial results are always reported. +# The CI check fails at the very end (after posting) if any regression found. +# ───────────────────────────────────────────────────────────────────────────── + comment: + name: Post PR comment + needs: [benchmark-linux, benchmark-windows, benchmark-macos] + runs-on: ubuntu-latest + if: always() && github.event_name == 'pull_request' + permissions: + pull-requests: write + + steps: + - uses: actions/checkout@v4 + + - name: Download all result artifacts + uses: actions/download-artifact@v4 + with: + pattern: bench-* + merge-multiple: true # flatten all artifacts into CWD + + # continue-on-error so the comment step always runs even on regression. + - name: Generate regression report + id: report + run: | + python3 benchmark/check_regression.py \ + --threshold 0.15 \ + --output comment.md \ + results_*.json continue-on-error: true - - name: Upload benchmark results - uses: actions/upload-artifact@v5 + - name: Find existing bot comment + uses: peter-evans/find-comment@v3 + id: find_comment with: - name: benchmark-results-msvc-${{ matrix.image }}-cxx${{ matrix.cxx_version }} - path: | - benchmark/build/benchmark_results.json - benchmark/build/benchmark_output.txt - - - name: Comment PR with benchmark results - if: github.event_name == 'pull_request' - uses: actions/github-script@v7 + issue-number: ${{ github.event.pull_request.number }} + comment-author: github-actions[bot] + body-includes: "" + + # One comment per PR, updated on every push — not a flood of new ones. + - name: Create or update PR comment + uses: peter-evans/create-or-update-comment@v4 with: - script: | - const fs = require('fs'); - const benchmark_output = fs.readFileSync('benchmark/build/benchmark_output.txt', 'utf8'); - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: `## Benchmark Results - MSVC on ${{ matrix.image }} (C++${{ matrix.cxx_version }})\n\n\`\`\`\n${benchmark_output}\n\`\`\`` - }); + comment-id: ${{ steps.find_comment.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body-path: comment.md + edit-mode: replace + + # Surface the failure visibly in the CI check panel. + - name: Fail if regression detected + if: steps.report.outcome == 'failure' + run: | + echo "::error::Performance regression detected. See the PR comment for the full table." + exit 1 \ No newline at end of file diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index b6739512..36183fec 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -1,30 +1,87 @@ +# benchmark/CMakeLists.txt +# +# Self-contained build for the gsl::span vs std::span benchmark. +# All dependencies are fetched automatically via FetchContent: +# - google/benchmark v1.9.0 +# - google/googletest release-1.12.1 (benchmark's internal dep) +# - microsoft/GSL v4.1.0 +# +# Usage (from this directory): +# cmake -S . -B build -DCMAKE_BUILD_TYPE=Release +# cmake --build build --target span_bench +# ./build/span_bench --benchmark_format=json --benchmark_repetitions=10 \ +# --benchmark_report_aggregates_only=true \ +# --benchmark_out=results.json +# +# To override the compiler or standard (as CI does): +# cmake -S . -B build -DCMAKE_BUILD_TYPE=Release \ +# -DCMAKE_CXX_COMPILER=clang++-18 \ +# -DCMAKE_CXX_STANDARD=23 + cmake_minimum_required(VERSION 3.20) -project(myproject C CXX) -set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type") # Default build type +project(gsl_span_benchmark CXX) -set(BENCHMARK_ENABLE_TESTING off) # to suppress benchmark internal tests -set(CMAKE_CXX_STANDARD 20 CACHE STRING "C++ standard for benchmarks") +# ── C++ standard ────────────────────────────────────────────────────────────── +# Default C++20 (minimum for std::span). CI overrides via -DCMAKE_CXX_STANDARD. +set(CMAKE_CXX_STANDARD 20 CACHE STRING "C++ standard") set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_EXTENSIONS OFF) +set(CMAKE_CXX_EXTENSIONS NO) + +set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type") +# ── FetchContent dependencies ────────────────────────────────────────────────── include(FetchContent) -FetchContent_Declare(googletest - GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG release-1.12.1) # or "main" for latest -FetchContent_Declare(googlebenchmark - GIT_REPOSITORY https://github.com/google/benchmark.git - GIT_TAG v1.9.0) # or "main" for latest -FetchContent_Declare(GSL - GIT_REPOSITORY https://github.com/microsoft/GSL.git - GIT_TAG v4.1.0 - GIT_SHALLOW ON) +# Suppress benchmark's own test suite — we don't need it. +set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE) + +FetchContent_Declare( + googletest + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG release-1.12.1 + GIT_SHALLOW ON +) -FetchContent_MakeAvailable(googletest googlebenchmark GSL) +FetchContent_Declare( + googlebenchmark + GIT_REPOSITORY https://github.com/google/benchmark.git + GIT_TAG v1.9.0 + GIT_SHALLOW ON +) +FetchContent_MakeAvailable(googletest googlebenchmark) +# Use the GSL from the local checkout — NOT a pinned release tag. +# The benchmark/ folder sits one level below the repo root, so +# CMAKE_CURRENT_SOURCE_DIR/.. resolves to the repo root which contains +# the real include/gsl/span header being modified by the PR. +# This is what makes the CI actually test the PR's changes. +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/.. gsl_root) + +# ── Benchmark executable ─────────────────────────────────────────────────────── add_executable(span_bench span_bench.cpp) -target_link_libraries(span_bench PRIVATE benchmark::benchmark) -target_link_libraries(span_bench PRIVATE Microsoft.GSL::GSL) +# Link benchmark::benchmark only — NOT benchmark::benchmark_main. +# span_bench.cpp uses the BENCHMARK_MAIN() macro which expands to its own +# main(), so linking benchmark_main would cause a duplicate-symbol linker error. +target_link_libraries(span_bench PRIVATE + benchmark::benchmark # provides the benchmark framework + runner + Microsoft.GSL::GSL # provides gsl::span and friends +) + +# ── Optimisation flags ───────────────────────────────────────────────────────── +# -O3 and -march=native let the compiler apply the same optimisations to both +# gsl::span and std::span, keeping the comparison fair. +# -fno-omit-frame-pointer keeps perf/profiling usable if you need it locally. +target_compile_options(span_bench PRIVATE + $<$: + -O3 + -march=native + -fno-omit-frame-pointer + > + $<$: + /O2 + /GL # whole-program optimisation + > +) \ No newline at end of file diff --git a/benchmark/check_regression.py b/benchmark/check_regression.py new file mode 100644 index 00000000..5d900f62 --- /dev/null +++ b/benchmark/check_regression.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python3 +""" +benchmark/check_regression.py + +Reads one or more Google Benchmark JSON result files (produced with +--benchmark_report_aggregates_only=true), pairs up gsl::span and std::span +benchmarks by name, computes the ratio gsl_ns / std_ns, and writes a +Markdown report to --output (default: stdout). + +Exits with code 1 if any ratio exceeds the threshold so CI fails cleanly. + +Usage: + python3 check_regression.py [--threshold 0.15] [--output report.md] results_*.json +""" + +import argparse +import json +import os +import sys +from pathlib import Path + + +# ─── helpers ────────────────────────────────────────────────────────────────── + +def load_json(path: str) -> dict: + with open(path) as f: + return json.load(f) + + +def label_from_filename(path: str) -> str: + """ + 'results_GCC-14-cpp20.json' → 'GCC-14-cpp20' + Falls back to the bare filename stem if the prefix isn't there. + """ + stem = Path(path).stem # e.g. 'results_GCC-14-cpp20' + if stem.startswith("results_"): + return stem[len("results_"):] + return stem + + +def parse_benchmarks(data: dict) -> dict[str, dict]: + """ + Returns a dict keyed by benchmark name. + For aggregated runs Google Benchmark emits multiple rows per benchmark + (_mean, _median, _stddev, _cv). We pull out mean and stddev. + + { "BM_IsSorted_StdSpan": {"mean": 124.3, "stddev": 2.1}, ... } + """ + result: dict[str, dict] = {} + for bm in data.get("benchmarks", []): + name: str = bm["name"] # e.g. "BM_IsSorted_StdSpan_mean" + agg = bm.get("aggregate_name") # "mean" | "stddev" | "median" | "cv" + + if agg not in ("mean", "stddev"): + continue + + # Strip the trailing _mean / _stddev to get the canonical name + base_name = name[: name.rfind(f"_{agg}")] + + entry = result.setdefault(base_name, {"mean": None, "stddev": None}) + entry[agg] = bm.get("real_time") or bm.get("cpu_time", 0.0) + + return result + + +def pair_benchmarks(bm_dict: dict[str, dict]): + """ + Match GslSpan variants with their StdSpan counterparts. + + Actual names from span_bench.cpp: + BM_IsSortedStdSpan ↔ BM_IsSortedGslSpan + BM_IsSortedRangesStdSpan ↔ BM_IsSortedRangesGslSpan + BM_IsSortedCustomStdSpan ↔ BM_IsSortedCustomGslSpan + BM_MinElementAlgorithmStdSpan ↔ BM_MinElementAlgorithmGslSpan + BM_MinElementRangeForStdSpan ↔ BM_MinElementRangeForGslSpan + + All std variants contain 'StdSpan'; swapping it for 'GslSpan' gives + the paired name. Short display name strips 'BM_' prefix and 'StdSpan' + infix, leaving e.g. 'IsSortedRanges', 'MinElementAlgorithm'. + """ + pairs = [] + for name, vals in sorted(bm_dict.items()): + if "StdSpan" not in name: + continue + gsl_name = name.replace("StdSpan", "GslSpan") + if gsl_name not in bm_dict: + continue + + # Strip BM_ prefix and StdSpan infix for a clean display name. + # e.g. "BM_IsSortedRangesStdSpan" → "IsSortedRanges" + short = name.removeprefix("BM_").replace("StdSpan", "").rstrip("_") + + pairs.append({ + "short": short, + "std_name": name, + "gsl_name": gsl_name, + "std_mean": bm_dict[name]["mean"] or 0.0, + "std_stddev": bm_dict[name]["stddev"] or 0.0, + "gsl_mean": bm_dict[gsl_name]["mean"] or 0.0, + "gsl_stddev": bm_dict[gsl_name]["stddev"] or 0.0, + }) + return pairs + + +def ratio_and_status(gsl: float, std: float, threshold: float): + if std == 0: + return None, "⚠️ div/0" + r = gsl / std + if r > 1 + threshold: + return r, f"🔴 **{r:.2f}×** regression" + if r < 1 - threshold: + return r, f"🟢 **{r:.2f}×** faster" + return r, f"✅ {r:.2f}×" + + +def fmt(ns: float) -> str: + """Format nanoseconds nicely.""" + if ns >= 1_000_000: + return f"{ns/1_000_000:.1f} ms" + if ns >= 1_000: + return f"{ns/1_000:.1f} µs" + return f"{ns:.1f} ns" + + +def fmt_stddev(stddev: float, mean: float) -> str: + if mean == 0: + return "—" + pct = (stddev / mean) * 100 + return f"±{pct:.1f}%" + + +# ─── report builder ─────────────────────────────────────────────────────────── + +def build_report(json_paths: list[str], threshold: float) -> tuple[str, bool]: + """ + Returns (markdown_text, had_regression). + """ + lines = [] + had_regression = False + + lines.append("") + lines.append("## 📊 `gsl::span` vs `std::span` benchmark results") + lines.append("") + lines.append( + f"> Threshold: flag if `gsl_ns / std_ns > {1 + threshold:.2f}` " + f"(+{threshold*100:.0f}%) or `< {1 - threshold:.2f}` " + f"(-{threshold*100:.0f}%) \n" + f"> Each benchmark ran **10 repetitions**; times shown are the mean " + f"± stddev." + ) + lines.append("") + + found_any = False + + for path in sorted(json_paths): + if not os.path.exists(path): + lines.append(f"> ⚠️ Result file not found: `{path}`") + continue + + try: + data = load_json(path) + except Exception as e: + lines.append(f"> ⚠️ Could not parse `{path}`: {e}") + continue + + config_label = label_from_filename(path) + bm_dict = parse_benchmarks(data) + pairs = pair_benchmarks(bm_dict) + + if not pairs: + lines.append(f"### `{config_label}`") + lines.append("> ⚠️ No paired benchmarks found — check naming convention.") + lines.append("") + continue + + found_any = True + + # Extract context info from the JSON if present + ctx = data.get("context", {}) + # cpu_scaling_enabled=True means the OS was allowed to vary frequency + # — bad for stable benchmarks. Google Benchmark warns about this itself + # but we surface it in the report too. + cpu_scaling = ctx.get("cpu_scaling_enabled", None) + num_cpus = ctx.get("num_cpus", "?") + mhz = ctx.get("mhz_per_cpu", "?") + + lines.append(f"### `{config_label}`") + lines.append(f"CPUs: {num_cpus} @ {mhz} MHz") + lines.append("") + + if cpu_scaling is True: + lines.append("> ⚠️ CPU frequency scaling was **enabled** — results may be noisier than usual.") + + lines.append("") + lines.append( + "| Benchmark | std mean | std σ | gsl mean | gsl σ | ratio | status |" + ) + lines.append( + "|-----------|----------|-------|----------|-------|-------|--------|" + ) + + config_regression = False + for p in pairs: + ratio, status = ratio_and_status(p["gsl_mean"], p["std_mean"], threshold) + if "regression" in status: + had_regression = True + config_regression = True + + lines.append( + f"| `{p['short']}` " + f"| {fmt(p['std_mean'])} " + f"| {fmt_stddev(p['std_stddev'], p['std_mean'])} " + f"| {fmt(p['gsl_mean'])} " + f"| {fmt_stddev(p['gsl_stddev'], p['gsl_mean'])} " + f"| {ratio:.2f}× " + f"| {status} |" + ) + + if config_regression: + lines.append("") + lines.append( + f"> 🔴 **Regression detected** in `{config_label}` — " + f"`gsl::span` is more than {threshold*100:.0f}% slower than " + f"`std::span` on one or more benchmarks." + ) + + lines.append("") + + if not found_any: + lines.append("> ❌ No benchmark results could be loaded.") + + # Footer + lines.append("---") + lines.append( + "_Ratio = `gsl_ns / std_ns`. " + "Values close to 1.0 mean performance parity. " + "Run by [span-benchmark CI](.github/workflows/span_benchmark.yml)._" + ) + + return "\n".join(lines), had_regression + + +# ─── entry point ───────────────────────────────────────────────────────────── + +def main(): + parser = argparse.ArgumentParser( + description="Check gsl::span vs std::span benchmark regression." + ) + parser.add_argument( + "json_files", + nargs="+", + metavar="results.json", + help="Google Benchmark JSON output files (one per CI matrix config).", + ) + parser.add_argument( + "--threshold", + type=float, + default=0.15, + help="Fractional slowdown threshold before flagging a regression (default: 0.15 = 15%%).", + ) + parser.add_argument( + "--output", + default=None, + metavar="FILE", + help="Write Markdown report to FILE instead of stdout.", + ) + args = parser.parse_args() + + report, had_regression = build_report(args.json_files, args.threshold) + + if args.output: + Path(args.output).write_text(report, encoding="utf-8") + print(f"Report written to {args.output}") + else: + print(report) + + if had_regression: + print( + "\n[check_regression] ❌ Performance regression detected. " + "See the report above.", + file=sys.stderr, + ) + sys.exit(1) + + print("\n[check_regression] ✅ No regressions detected.", file=sys.stderr) + sys.exit(0) + + +if __name__ == "__main__": + main() \ No newline at end of file From b8fa5a0fec85121ebffaf2a5124e2bbc34c60e94 Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Gafoor <94980075+Gafoor2005@users.noreply.github.com> Date: Fri, 29 May 2026 10:48:05 +0000 Subject: [PATCH 06/15] renamed workflow file --- .github/workflows/{benchmarks.yml => span_benchmark.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{benchmarks.yml => span_benchmark.yml} (100%) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/span_benchmark.yml similarity index 100% rename from .github/workflows/benchmarks.yml rename to .github/workflows/span_benchmark.yml From 7e5d36decb3a18722a5ca8fa3304d95f4858b118 Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Gafoor <94980075+Gafoor2005@users.noreply.github.com> Date: Fri, 29 May 2026 11:14:19 +0000 Subject: [PATCH 07/15] Test commit for PR-based benchmark --- benchmark/test | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 benchmark/test diff --git a/benchmark/test b/benchmark/test new file mode 100644 index 00000000..e69de29b From 822df048ca56078ea028bdbf6b00a373e1a628c1 Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Gafoor <94980075+Gafoor2005@users.noreply.github.com> Date: Fri, 29 May 2026 11:30:15 +0000 Subject: [PATCH 08/15] Fixed cmake. build form benchmark directory --- .github/workflows/span_benchmark.yml | 60 ++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 7 deletions(-) diff --git a/.github/workflows/span_benchmark.yml b/.github/workflows/span_benchmark.yml index 97e2263f..ed0ddefb 100644 --- a/.github/workflows/span_benchmark.yml +++ b/.github/workflows/span_benchmark.yml @@ -19,19 +19,24 @@ # A final `comment` job waits for all three groups, collects every JSON # artifact, runs check_regression.py, and posts (or updates) one PR comment. + name: Span Benchmark + on: pull_request: branches: [main] + # Cancel stale runs when a new commit is pushed to the same PR. concurrency: group: span-bench-${{ github.event.pull_request.number }} cancel-in-progress: true + jobs: + # ───────────────────────────────────────────────────────────────────────────── # LINUX — GCC 13/14 · Clang 17/18 (× C++20 and C++23) # ───────────────────────────────────────────────────────────────────────────── @@ -48,32 +53,38 @@ jobs: cppstd: 20 extra_flags: "" + - label: GCC-14-cpp20 cxx: g++-14 cppstd: 20 extra_flags: "" + - label: GCC-14-cpp23 cxx: g++-14 cppstd: 23 extra_flags: "" + # ── Clang ────────────────────────────────────────────────────── - label: Clang-17-cpp20 cxx: clang++-17 cppstd: 20 extra_flags: "" + - label: Clang-18-cpp20 cxx: clang++-18 cppstd: 20 extra_flags: "" + - label: Clang-18-cpp23 cxx: clang++-18 cppstd: 23 extra_flags: "" + # ── Clang + -fbounds-safety (core motivation of issue #1167) ─── # Safe Buffers enforcement is what triggered this whole tracking effort. - label: Clang-18-cpp20-bounds-safety @@ -81,9 +92,11 @@ jobs: cppstd: 20 extra_flags: "-fbounds-safety" + steps: - uses: actions/checkout@v4 + # ── Compiler install ─────────────────────────────────────────────── # GCC-13 and Clang pre-17 are already on ubuntu-latest (24.04). - name: Install GCC 14 @@ -93,24 +106,27 @@ jobs: sudo apt-get update -qq sudo apt-get install -y g++-14 + - name: Install Clang 17 if: matrix.cxx == 'clang++-17' run: | - wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key \ + wget -qO- [https://apt.llvm.org/llvm-snapshot.gpg.key](https://apt.llvm.org/llvm-snapshot.gpg.key) \ | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc - echo "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-17 main" \ + echo "deb [http://apt.llvm.org/noble/](http://apt.llvm.org/noble/) llvm-toolchain-noble-17 main" \ | sudo tee /etc/apt/sources.list.d/llvm-17.list sudo apt-get update -qq && sudo apt-get install -y clang-17 + - name: Install Clang 18 if: matrix.cxx == 'clang++-18' run: | - wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key \ + wget -qO- [https://apt.llvm.org/llvm-snapshot.gpg.key](https://apt.llvm.org/llvm-snapshot.gpg.key) \ | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc - echo "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-18 main" \ + echo "deb [http://apt.llvm.org/noble/](http://apt.llvm.org/noble/) llvm-toolchain-noble-18 main" \ | sudo tee /etc/apt/sources.list.d/llvm-18.list sudo apt-get update -qq && sudo apt-get install -y clang-18 + # ── FetchContent cache ───────────────────────────────────────────── # Caches the cloned sources for google-benchmark, googletest, and GSL. # Key is the hash of CMakeLists.txt — busts automatically on any @@ -123,21 +139,24 @@ jobs: restore-keys: | fetchcontent-linux-${{ matrix.cxx }}- + # ── Configure → Build → Run ──────────────────────────────────────── # FetchContent handles benchmark + GSL — no separate install needed. # -DCMAKE_CXX_STANDARD overrides the default C++20 set in CMakeLists # so we can test C++23 configs from the matrix. - name: Configure run: | - cmake -S . -B build \ + cmake -S benchmark -B build \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \ -DCMAKE_CXX_STANDARD=${{ matrix.cppstd }} \ -DCMAKE_CXX_FLAGS="${{ matrix.extra_flags }}" + - name: Build run: cmake --build build --target span_bench -j$(nproc) + # 10 repetitions → mean + stddev aggregates in the JSON output. # benchmark_report_aggregates_only keeps the file compact. - name: Run benchmark @@ -148,6 +167,7 @@ jobs: --benchmark_report_aggregates_only=true \ --benchmark_out=results_${{ matrix.label }}.json + - name: Upload results uses: actions/upload-artifact@v4 with: @@ -155,6 +175,7 @@ jobs: path: results_${{ matrix.label }}.json retention-days: 7 + # ───────────────────────────────────────────────────────────────────────────── # WINDOWS — MSVC 2022 · clang-cl (× C++20 and C++23) # @@ -181,25 +202,30 @@ jobs: cppstd: 20 extra_flags: "" + - label: MSVC-2022-cpp23 toolset: "" cppstd: 23 extra_flags: "" + # ── clang-cl (bundled with VS 2022) ──────────────────────────── - label: clang-cl-cpp20 toolset: "ClangCL" cppstd: 20 extra_flags: "" + - label: clang-cl-cpp23 toolset: "ClangCL" cppstd: 23 extra_flags: "" + steps: - uses: actions/checkout@v4 + # ── FetchContent cache ───────────────────────────────────────────── - name: Cache FetchContent dependencies uses: actions/cache@v4 @@ -209,23 +235,27 @@ jobs: restore-keys: | fetchcontent-windows-${{ matrix.toolset }}- + # ── Configure → Build → Run ──────────────────────────────────────── - name: Configure shell: pwsh run: | $tsArg = if ("${{ matrix.toolset }}" -ne "") { @("-T", "${{ matrix.toolset }}") } else { @() } - cmake -S . -B build ` + + cmake -S benchmark -B build ` -G "Visual Studio 17 2022" @tsArg ` -DCMAKE_CXX_STANDARD=${{ matrix.cppstd }} ` -DCMAKE_CXX_FLAGS="${{ matrix.extra_flags }}" + - name: Build shell: pwsh run: | cmake --build build --target span_bench ` --config Release -j $env:NUMBER_OF_PROCESSORS + # Multi-config generator places the binary under build\Release\ - name: Run benchmark shell: pwsh @@ -236,6 +266,7 @@ jobs: --benchmark_report_aggregates_only=true ` --benchmark_out=results_${{ matrix.label }}.json + - name: Upload results uses: actions/upload-artifact@v4 with: @@ -243,6 +274,7 @@ jobs: path: results_${{ matrix.label }}.json retention-days: 7 + # ───────────────────────────────────────────────────────────────────────────── # MACOS — Apple Clang via Xcode (latest) (× C++20 and C++23) # @@ -262,13 +294,16 @@ jobs: cppstd: 20 extra_flags: "" + - label: AppleClang-cpp23 cppstd: 23 extra_flags: "" + steps: - uses: actions/checkout@v4 + # ── FetchContent cache ───────────────────────────────────────────── - name: Cache FetchContent dependencies uses: actions/cache@v4 @@ -278,17 +313,20 @@ jobs: restore-keys: | fetchcontent-macos-${{ matrix.cppstd }}- + # ── Configure → Build → Run ──────────────────────────────────────── - name: Configure run: | - cmake -S . -B build \ + cmake -S benchmark -B build \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CXX_STANDARD=${{ matrix.cppstd }} \ -DCMAKE_CXX_FLAGS="${{ matrix.extra_flags }}" + - name: Build run: cmake --build build --target span_bench -j$(sysctl -n hw.logicalcpu) + - name: Run benchmark run: | ./build/span_bench \ @@ -297,6 +335,7 @@ jobs: --benchmark_report_aggregates_only=true \ --benchmark_out=results_${{ matrix.label }}.json + - name: Upload results uses: actions/upload-artifact@v4 with: @@ -304,6 +343,7 @@ jobs: path: results_${{ matrix.label }}.json retention-days: 7 + # ───────────────────────────────────────────────────────────────────────────── # COMMENT — collect every result artifact → one PR comment # @@ -319,15 +359,18 @@ jobs: permissions: pull-requests: write + steps: - uses: actions/checkout@v4 + - name: Download all result artifacts uses: actions/download-artifact@v4 with: pattern: bench-* merge-multiple: true # flatten all artifacts into CWD + # continue-on-error so the comment step always runs even on regression. - name: Generate regression report id: report @@ -338,6 +381,7 @@ jobs: results_*.json continue-on-error: true + - name: Find existing bot comment uses: peter-evans/find-comment@v3 id: find_comment @@ -346,6 +390,7 @@ jobs: comment-author: github-actions[bot] body-includes: "" + # One comment per PR, updated on every push — not a flood of new ones. - name: Create or update PR comment uses: peter-evans/create-or-update-comment@v4 @@ -355,6 +400,7 @@ jobs: body-path: comment.md edit-mode: replace + # Surface the failure visibly in the CI check panel. - name: Fail if regression detected if: steps.report.outcome == 'failure' From cf497fdeb704f4e9451cf07f520a0212be6aa05a Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Gafoor <94980075+Gafoor2005@users.noreply.github.com> Date: Fri, 29 May 2026 11:39:26 +0000 Subject: [PATCH 09/15] fixed benchmark ci syntax --- .github/workflows/span_benchmark.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/span_benchmark.yml b/.github/workflows/span_benchmark.yml index ed0ddefb..667ac15e 100644 --- a/.github/workflows/span_benchmark.yml +++ b/.github/workflows/span_benchmark.yml @@ -110,9 +110,9 @@ jobs: - name: Install Clang 17 if: matrix.cxx == 'clang++-17' run: | - wget -qO- [https://apt.llvm.org/llvm-snapshot.gpg.key](https://apt.llvm.org/llvm-snapshot.gpg.key) \ + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key \ | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc - echo "deb [http://apt.llvm.org/noble/](http://apt.llvm.org/noble/) llvm-toolchain-noble-17 main" \ + echo "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-17 main" \ | sudo tee /etc/apt/sources.list.d/llvm-17.list sudo apt-get update -qq && sudo apt-get install -y clang-17 @@ -120,9 +120,9 @@ jobs: - name: Install Clang 18 if: matrix.cxx == 'clang++-18' run: | - wget -qO- [https://apt.llvm.org/llvm-snapshot.gpg.key](https://apt.llvm.org/llvm-snapshot.gpg.key) \ + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key \ | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc - echo "deb [http://apt.llvm.org/noble/](http://apt.llvm.org/noble/) llvm-toolchain-noble-18 main" \ + echo "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-18 main" \ | sudo tee /etc/apt/sources.list.d/llvm-18.list sudo apt-get update -qq && sudo apt-get install -y clang-18 From 678771b1547f72692a2570d90218ef040689a5e2 Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Gafoor <94980075+Gafoor2005@users.noreply.github.com> Date: Fri, 29 May 2026 12:16:45 +0000 Subject: [PATCH 10/15] disabled a unsupported matrix entry in span_benchmark.yml --- .github/workflows/span_benchmark.yml | 12 ++++++++---- benchmark/check_regression.py | 12 +++++++++--- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/span_benchmark.yml b/.github/workflows/span_benchmark.yml index 667ac15e..b4f0aeea 100644 --- a/.github/workflows/span_benchmark.yml +++ b/.github/workflows/span_benchmark.yml @@ -87,10 +87,13 @@ jobs: # ── Clang + -fbounds-safety (core motivation of issue #1167) ─── # Safe Buffers enforcement is what triggered this whole tracking effort. - - label: Clang-18-cpp20-bounds-safety - cxx: clang++-18 - cppstd: 20 - extra_flags: "-fbounds-safety" + # - label: Clang-18-cpp20-bounds-safety + # cxx: clang++-18 + # cppstd: 20 + # extra_flags: "-fbounds-safety" + # ---------------------------------------------------------------------------- + # ⚠️ Note: As of early 2026, the feature is not yet fully available or stable in mainline, official releases (like Clang 20, 21, or 22). + # ---------------------------------------------------------------------------- steps: @@ -378,6 +381,7 @@ jobs: python3 benchmark/check_regression.py \ --threshold 0.15 \ --output comment.md \ + --workflow-url "https://github.com/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/span_benchmark.yml" \ results_*.json continue-on-error: true diff --git a/benchmark/check_regression.py b/benchmark/check_regression.py index 5d900f62..f89ed4a2 100644 --- a/benchmark/check_regression.py +++ b/benchmark/check_regression.py @@ -131,7 +131,7 @@ def fmt_stddev(stddev: float, mean: float) -> str: # ─── report builder ─────────────────────────────────────────────────────────── -def build_report(json_paths: list[str], threshold: float) -> tuple[str, bool]: +def build_report(json_paths: list[str], threshold: float, workflow_url: str = None) -> tuple[str, bool]: """ Returns (markdown_text, had_regression). """ @@ -234,7 +234,7 @@ def build_report(json_paths: list[str], threshold: float) -> tuple[str, bool]: lines.append( "_Ratio = `gsl_ns / std_ns`. " "Values close to 1.0 mean performance parity. " - "Run by [span-benchmark CI](.github/workflows/span_benchmark.yml)._" + "Run by [span-benchmark CI]({workflow_url})._" ) return "\n".join(lines), had_regression @@ -264,9 +264,15 @@ def main(): metavar="FILE", help="Write Markdown report to FILE instead of stdout.", ) + parser.add_argument( + "--workflow-url", + default=None, + metavar="URL", + help="GitHub URL to the workflow YAML (shows in the report footer).", + ) args = parser.parse_args() - report, had_regression = build_report(args.json_files, args.threshold) + report, had_regression = build_report(args.json_files, args.threshold, args.workflow_url) if args.output: Path(args.output).write_text(report, encoding="utf-8") From d86a56873cf7f7987476709d815b9648352843f0 Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Gafoor <94980075+Gafoor2005@users.noreply.github.com> Date: Fri, 29 May 2026 16:12:41 +0000 Subject: [PATCH 11/15] added readme --- benchmark/README.md | 165 +++++++++++++++++++++++----------- benchmark/check_regression.py | 2 +- benchmark/test | 0 3 files changed, 114 insertions(+), 53 deletions(-) delete mode 100644 benchmark/test diff --git a/benchmark/README.md b/benchmark/README.md index 8e85dd90..fdda94fe 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -1,83 +1,144 @@ -# GSL Performance Benchmarks +# `gsl::span` vs `std::span` Benchmark -This directory contains performance benchmarks comparing `gsl::span` with `std::span` across various workloads and compiler implementations. +Performance parity tracking between `gsl::span` and `std::span` across all supported compilers, platforms, and C++ standards — as part of [microsoft/GSL#1167](https://github.com/microsoft/GSL/issues/1167) and [microsoft/GSL#1165](https://github.com/microsoft/GSL/issues/1165). + +--- ## Overview -The benchmarks test critical span operations to ensure `gsl::span` maintains **performance parity** with `std::span`, particularly when [Safe Buffers](https://clang.llvm.org/docs/SafeBuffers.html) are enabled. +`gsl::span` should be a zero-overhead abstraction over `std::span`. This benchmark suite verifies that claim continuously — on every PR — so performance regressions are caught before they land in main. -### Tested Operations +The comparison strategy is **in-run ratio** (`gsl_ns / std_ns`): both spans are measured in the same process on the same machine at the same moment, so runner noise cancels out. A ratio close to `1.0` means parity. If `gsl::span` is more than **15% slower** than `std::span` on any benchmark, CI flags it and posts a detailed table in the PR comment. -- **`is_sorted`**: Custom iterator-based check, `std::is_sorted`, and `std::ranges::is_sorted` -- **`min_element`**: Algorithm-based and range-for iteration approaches +--- -Each test compares both `std::span` and `gsl::span` implementations. +## Benchmarks -## Building Locally +All benchmarks run on a sorted vector of 1000 integers. -### Prerequisites +| Benchmark | What it tests | +|---|---| +| `IsSorted` | `std::is_sorted` via span iterators | +| `IsSortedRanges` | `std::ranges::is_sorted` via the span range interface | +| `IsSortedCustom` | Custom hand-rolled `is_sorted` loop via span iterators | +| `MinElementAlgorithm` | `std::min_element` via span iterators | +| `MinElementRangeFor` | Range-for loop with a custom min accumulator | -- CMake 3.20+ -- C++20 compiler or newer (GCC, Clang, or MSVC) -- Google Benchmark (fetched automatically via CMake) -- Google Test (fetched automatically via CMake) +Each benchmark has a `StdSpan` and `GslSpan` variant. The Python script pairs them by name and computes the ratio. -### Build Steps +--- -```bash -# From the repository root -cd benchmark -mkdir -p build -cd build +## CI Matrix + +The benchmark runs on every pull request across 13 configurations: + +| OS | Compiler | C++ Standard | +|---|---|---| +| ubuntu-latest | GCC 13 | C++20 | +| ubuntu-latest | GCC 14 | C++20, C++23 | +| ubuntu-latest | Clang 17 | C++20 | +| ubuntu-latest | Clang 18 | C++20, C++23 | +| windows-latest | MSVC 2022 | C++20, C++23 | +| windows-latest | clang-cl (VS 2022 bundled) | C++20, C++23 | +| macos-14 (Apple Silicon) | Apple Clang (Xcode latest) | C++20, C++23 | + +Results from all 13 jobs are collected and posted as a **single PR comment**, updated on every push. + +--- -# Configure with Release build type (important for accurate benchmarks) -cmake .. -DCMAKE_BUILD_TYPE=Release +## Repository Layout -# Build -cmake --build . --config Release ``` +benchmark/ +├── CMakeLists.txt # self-contained build — fetches benchmark + googletest +├── span_bench.cpp # the benchmark source +├── check_regression.py # parses JSON results, writes the PR comment markdown +└── README.md # this file + +.github/workflows/ +└── span_benchmark.yml # CI workflow +``` + +The benchmark folder is self-contained. `CMakeLists.txt` fetches `google/benchmark` (v1.9.0) and `google/googletest` via `FetchContent`. GSL itself is sourced from the **local repo checkout** — not a pinned tag — so the benchmark always tests the code in the PR, not a release. + +--- -## Running Benchmarks +## Running Locally + +**Prerequisites:** CMake ≥ 3.20, a C++20-capable compiler, internet access for `FetchContent`. ```bash -# Run all benchmarks with default output -./span_bench +# From the benchmark/ directory: +cmake -S . -B build -DCMAKE_BUILD_TYPE=Release +cmake --build build --target span_bench + +# Run with JSON output (matches what CI does): +./build/span_bench \ + --benchmark_format=json \ + --benchmark_repetitions=10 \ + --benchmark_report_aggregates_only=true \ + --benchmark_out=results.json + +# Generate the regression report locally: +python3 check_regression.py --threshold 0.15 results.json +``` -# Run with JSON output (useful for analysis) -./span_bench --benchmark_format=json > results.json +To test a specific compiler or standard: -# Run specific benchmark (regex matching) -./span_bench --benchmark_filter="IsSorted" +```bash +cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_COMPILER=clang++-18 \ + -DCMAKE_CXX_STANDARD=23 +``` + +--- + +## Regression Detection + +The `check_regression.py` script: -# Show detailed statistics -./span_bench --benchmark_report_aggregates_only=true +1. Reads one or more Google Benchmark JSON files (one per CI matrix config) +2. Pairs `*StdSpan` benchmarks with their `*GslSpan` counterparts by name +3. Computes `ratio = gsl_mean / std_mean` using the 10-repetition mean +4. Flags any ratio above `1 + threshold` (default **15%**) as a regression +5. Writes a Markdown table per config, collected into a single PR comment +6. Exits with code `1` if any regression is found — failing the CI check + +``` +python3 check_regression.py [--threshold 0.15] [--output report.md] results_*.json ``` -For more options, see [Google Benchmark documentation](https://github.com/google/benchmark#usage). +### Example PR comment output + +``` +## 📊 gsl::span vs std::span benchmark results + +### `GCC-14-cpp20` +| Benchmark | std mean | std σ | gsl mean | gsl σ | ratio | status | +|----------------------|----------|--------|----------|--------|-------|----------| +| IsSorted | 124.1 ns | ±1.2% | 125.3 ns | ±1.4% | 1.01× | ✅ 1.01× | +| IsSortedRanges | 88.4 ns | ±0.9% | 89.1 ns | ±1.1% | 1.01× | ✅ 1.01× | +| IsSortedCustom | 112.6 ns | ±1.5% | 113.2 ns | ±1.3% | 1.01× | ✅ 1.01× | +| MinElementAlgorithm | 95.2 ns | ±1.0% | 96.0 ns | ±1.2% | 1.01× | ✅ 1.01× | +| MinElementRangeFor | 98.7 ns | ±1.1% | 99.4 ns | ±0.8% | 1.01× | ✅ 1.01× | +``` -## CI Integration +Status icons: +- ✅ — within the threshold (parity) +- 🔴 — `gsl::span` is more than 15% slower (regression, CI fails) +- 🟢 — `gsl::span` is more than 15% faster (improvement, noted but not a failure) -Benchmarks automatically run on every pull request across: -- **Compilers**: GCC 14 (latest), Clang 18 (latest), MSVC (latest) -- **Standards**: C++20 -- **Platforms**: Linux (GCC/Clang), Windows (MSVC on windows-2025) +--- -Results are uploaded as GitHub Actions artifacts for easy download and analysis. +## Noise Considerations -Results are uploaded as GitHub Actions artifacts and can be analyzed for performance regressions. +GitHub-hosted runners are shared VMs with a typical noise floor of **±10–15%** on absolute timing. The ratio strategy mitigates this because both span variants experience the same CPU conditions simultaneously. The 15% threshold is chosen to sit just above the noise floor — tight enough to catch real regressions, loose enough to avoid false positives on every PR. -## Performance Analysis +To further reduce variance, each benchmark runs **10 repetitions** and the script uses the **mean** (not a single sample) for the ratio calculation. The stddev column in the comment table lets reviewers eyeball how stable each measurement was. -When comparing results: -1. Always compare the same compiler, C++ standard, and platform -2. Release builds should be used for accurate measurements -3. Significant differences (>5%) between `std::span` and `gsl::span` warrant investigation -4. Account for noise in CI environments when analyzing small variations +--- -## Contributing +## Background -When adding new benchmarks: -1. Follow the existing pattern in `span_bench.cpp` -2. Compare both `std::span` and `gsl::span` implementations -3. Use `benchmark::DoNotOptimize` to prevent compiler optimizations from skewing results -4. Document what the benchmark tests and why it matters +This benchmark was created in response to [microsoft/GSL#1167](https://github.com/microsoft/GSL/issues/1167), which highlighted the need to maintain performance parity with `std::span`, especially when [Safe Buffers / `-fbounds-safety`](https://clang.llvm.org/docs/SafeBuffers.html) are enabled. The initial benchmark scaffolding was provided by @galenelias. \ No newline at end of file diff --git a/benchmark/check_regression.py b/benchmark/check_regression.py index f89ed4a2..69988c9f 100644 --- a/benchmark/check_regression.py +++ b/benchmark/check_regression.py @@ -234,7 +234,7 @@ def build_report(json_paths: list[str], threshold: float, workflow_url: str = No lines.append( "_Ratio = `gsl_ns / std_ns`. " "Values close to 1.0 mean performance parity. " - "Run by [span-benchmark CI]({workflow_url})._" + f"Run by [span-benchmark CI]({workflow_url})._" ) return "\n".join(lines), had_regression diff --git a/benchmark/test b/benchmark/test deleted file mode 100644 index e69de29b..00000000 From 62d9022d85d09c875474d05aed6dbcbf0fd45e9a Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Gafoor <94980075+Gafoor2005@users.noreply.github.com> Date: Fri, 29 May 2026 16:20:56 +0000 Subject: [PATCH 12/15] fixed PR comment's workflow url path --- .github/workflows/span_benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/span_benchmark.yml b/.github/workflows/span_benchmark.yml index b4f0aeea..cbbc7dfc 100644 --- a/.github/workflows/span_benchmark.yml +++ b/.github/workflows/span_benchmark.yml @@ -381,7 +381,7 @@ jobs: python3 benchmark/check_regression.py \ --threshold 0.15 \ --output comment.md \ - --workflow-url "https://github.com/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/span_benchmark.yml" \ + --workflow-url "https://github.com/${{ github.repository }}/blob/tree/main/.github/workflows/span_benchmark.yml" \ results_*.json continue-on-error: true From ff48618829b9c768fe79c750ead97640fb2cfbbe Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Gafoor <94980075+Gafoor2005@users.noreply.github.com> Date: Mon, 8 Jun 2026 12:38:05 +0000 Subject: [PATCH 13/15] Refactor benchmark CI configuration and add span benchmark comment workflow - Updated span_benchmark.yml to streamline job configurations and improve artifact handling. - Introduced span_benchmark_comment.yml to automate PR comment updates with benchmark results. - Enhanced CMakeLists.txt to support benchmark builds with new GSL_BENCHMARK option. - Removed README.md from benchmark directory and adjusted check_regression.py for cleaner report generation. - Modified span_bench.cpp to include necessary headers for benchmarking. --- .github/workflows/span_benchmark.yml | 333 +++---------------- .github/workflows/span_benchmark_comment.yml | 76 +++++ CMakeLists.txt | 5 + benchmark/CMakeLists.txt | 82 ++--- benchmark/README.md | 144 -------- benchmark/check_regression.py | 21 +- benchmark/span_bench.cpp | 10 +- 7 files changed, 154 insertions(+), 517 deletions(-) create mode 100644 .github/workflows/span_benchmark_comment.yml delete mode 100644 benchmark/README.md diff --git a/.github/workflows/span_benchmark.yml b/.github/workflows/span_benchmark.yml index cbbc7dfc..d1743483 100644 --- a/.github/workflows/span_benchmark.yml +++ b/.github/workflows/span_benchmark.yml @@ -1,256 +1,94 @@ -# .github/workflows/span_benchmark.yml -# -# Benchmarks gsl::span vs std::span on every PR across all supported compilers. -# Strategy: in-run ratio (gsl_ns / std_ns) — noise-resistant because both -# spans run in the same process, same machine, same moment. -# -# All dependencies (google-benchmark v1.9.0, GSL v4.1.0) are pulled via -# FetchContent inside the repo's CMakeLists.txt — no separate install steps. -# The build/_deps directory is cached, keyed on CMakeLists.txt hash, so -# repeated runs don't re-clone anything. -# -# Three separate job groups (one per OS) because `runs-on` can't share a -# matrix with OS-specific shell/path differences cleanly: -# -# benchmark-linux → ubuntu-latest → GCC-13, GCC-14, Clang-17, Clang-18 -# benchmark-windows → windows-latest → MSVC 2022, clang-cl -# benchmark-macos → macos-14 → Apple Clang (Xcode latest) -# -# A final `comment` job waits for all three groups, collects every JSON -# artifact, runs check_regression.py, and posts (or updates) one PR comment. - - name: Span Benchmark - on: pull_request: branches: [main] - # Cancel stale runs when a new commit is pushed to the same PR. concurrency: group: span-bench-${{ github.event.pull_request.number }} cancel-in-progress: true - jobs: - - -# ───────────────────────────────────────────────────────────────────────────── -# LINUX — GCC 13/14 · Clang 17/18 (× C++20 and C++23) -# ───────────────────────────────────────────────────────────────────────────── benchmark-linux: - name: Linux / ${{ matrix.label }} + name: Linux / ${{ format('{0}-cpp{1}', matrix.cxx, matrix.cppstd) }} runs-on: ubuntu-latest strategy: fail-fast: false matrix: - include: - # ── GCC ──────────────────────────────────────────────────────── - - label: GCC-13-cpp20 - cxx: g++-13 - cppstd: 20 - extra_flags: "" - - - - label: GCC-14-cpp20 - cxx: g++-14 - cppstd: 20 - extra_flags: "" - - - - label: GCC-14-cpp23 - cxx: g++-14 - cppstd: 23 - extra_flags: "" - - - # ── Clang ────────────────────────────────────────────────────── - - label: Clang-17-cpp20 - cxx: clang++-17 - cppstd: 20 - extra_flags: "" - - - - label: Clang-18-cpp20 - cxx: clang++-18 - cppstd: 20 - extra_flags: "" - - - - label: Clang-18-cpp23 - cxx: clang++-18 - cppstd: 23 - extra_flags: "" - - - # ── Clang + -fbounds-safety (core motivation of issue #1167) ─── - # Safe Buffers enforcement is what triggered this whole tracking effort. - # - label: Clang-18-cpp20-bounds-safety - # cxx: clang++-18 - # cppstd: 20 - # extra_flags: "-fbounds-safety" - # ---------------------------------------------------------------------------- - # ⚠️ Note: As of early 2026, the feature is not yet fully available or stable in mainline, official releases (like Clang 20, 21, or 22). - # ---------------------------------------------------------------------------- - - + cxx: [g++-13, g++-14, clang++-16, clang++-17, clang++-18] + cppstd: [20, 23] + exclude: + - cxx: g++-13 + cppstd: 23 + - cxx: clang++-17 + cppstd: 23 steps: - uses: actions/checkout@v4 - - # ── Compiler install ─────────────────────────────────────────────── - # GCC-13 and Clang pre-17 are already on ubuntu-latest (24.04). - - name: Install GCC 14 - if: startsWith(matrix.cxx, 'g++-14') - run: | - sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y - sudo apt-get update -qq - sudo apt-get install -y g++-14 - - - - name: Install Clang 17 - if: matrix.cxx == 'clang++-17' - run: | - wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key \ - | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc - echo "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-17 main" \ - | sudo tee /etc/apt/sources.list.d/llvm-17.list - sudo apt-get update -qq && sudo apt-get install -y clang-17 - - - - name: Install Clang 18 - if: matrix.cxx == 'clang++-18' - run: | - wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key \ - | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc - echo "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-18 main" \ - | sudo tee /etc/apt/sources.list.d/llvm-18.list - sudo apt-get update -qq && sudo apt-get install -y clang-18 - - - # ── FetchContent cache ───────────────────────────────────────────── - # Caches the cloned sources for google-benchmark, googletest, and GSL. - # Key is the hash of CMakeLists.txt — busts automatically on any - # GIT_TAG bump without manual intervention. - name: Cache FetchContent dependencies uses: actions/cache@v4 with: path: build/_deps - key: fetchcontent-linux-${{ matrix.cxx }}-${{ hashFiles('CMakeLists.txt') }} + key: fetchcontent-linux-${{ matrix.cxx }}-${{ hashFiles('benchmark/CMakeLists.txt') }} restore-keys: | fetchcontent-linux-${{ matrix.cxx }}- - - # ── Configure → Build → Run ──────────────────────────────────────── - # FetchContent handles benchmark + GSL — no separate install needed. - # -DCMAKE_CXX_STANDARD overrides the default C++20 set in CMakeLists - # so we can test C++23 configs from the matrix. - name: Configure run: | - cmake -S benchmark -B build \ + cmake -S . -B build \ + -DGSL_BENCHMARK=ON \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \ - -DCMAKE_CXX_STANDARD=${{ matrix.cppstd }} \ - -DCMAKE_CXX_FLAGS="${{ matrix.extra_flags }}" - + -DCMAKE_CXX_STANDARD=${{ matrix.cppstd }}" - name: Build run: cmake --build build --target span_bench -j$(nproc) - - # 10 repetitions → mean + stddev aggregates in the JSON output. - # benchmark_report_aggregates_only keeps the file compact. - name: Run benchmark run: | ./build/span_bench \ --benchmark_format=json \ --benchmark_repetitions=10 \ --benchmark_report_aggregates_only=true \ - --benchmark_out=results_${{ matrix.label }}.json - + --benchmark_out=results_${{ format('{0}-cpp{1}', matrix.cxx, matrix.cppstd) }}.json - name: Upload results uses: actions/upload-artifact@v4 with: - name: bench-${{ matrix.label }} - path: results_${{ matrix.label }}.json + name: bench-${{ format('{0}-cpp{1}', matrix.cxx, matrix.cppstd) }} + path: results_${{ format('{0}-cpp{1}', matrix.cxx, matrix.cppstd) }}.json retention-days: 7 - -# ───────────────────────────────────────────────────────────────────────────── -# WINDOWS — MSVC 2022 · clang-cl (× C++20 and C++23) -# -# Notes: -# • Uses the Visual Studio 17 2022 generator for both MSVC and clang-cl. -# The -T ClangCL toolset switch selects the clang-cl frontend that ships -# bundled with VS 2022 — no extra install needed. -# • Multi-config generator: output binary is at -# build\Release\span_bench.exe (FetchContent flat layout, no subdir). -# • FetchContent cache is keyed per-toolset because MSVC and clang-cl -# produce ABI-incompatible object files. -# • PowerShell is used throughout (shell: pwsh) for consistent quoting. -# ───────────────────────────────────────────────────────────────────────────── benchmark-windows: - name: Windows / ${{ matrix.label }} + name: Windows / ${{ format('{0}-cpp{1}', matrix.toolset || 'MSVC', matrix.cppstd) }} runs-on: windows-latest strategy: fail-fast: false matrix: - include: - # ── MSVC 2022 ────────────────────────────────────────────────── - - label: MSVC-2022-cpp20 - toolset: "" - cppstd: 20 - extra_flags: "" - - - - label: MSVC-2022-cpp23 - toolset: "" - cppstd: 23 - extra_flags: "" - - - # ── clang-cl (bundled with VS 2022) ──────────────────────────── - - label: clang-cl-cpp20 - toolset: "ClangCL" - cppstd: 20 - extra_flags: "" - - - - label: clang-cl-cpp23 - toolset: "ClangCL" - cppstd: 23 - extra_flags: "" - - + generator: [ 'Visual Studio 17 2022' ] + toolset: [ '', 'ClangCL' ] + cppstd: [ 20, 23 ] steps: - uses: actions/checkout@v4 - - # ── FetchContent cache ───────────────────────────────────────────── - name: Cache FetchContent dependencies uses: actions/cache@v4 with: path: build/_deps - key: fetchcontent-windows-${{ matrix.toolset }}-${{ hashFiles('CMakeLists.txt') }} + key: fetchcontent-windows-${{ matrix.toolset }}-${{ hashFiles('benchmark/CMakeLists.txt') }} restore-keys: | fetchcontent-windows-${{ matrix.toolset }}- - - # ── Configure → Build → Run ──────────────────────────────────────── - name: Configure shell: pwsh run: | $tsArg = if ("${{ matrix.toolset }}" -ne "") { @("-T", "${{ matrix.toolset }}") } else { @() } - - cmake -S benchmark -B build ` - -G "Visual Studio 17 2022" @tsArg ` - -DCMAKE_CXX_STANDARD=${{ matrix.cppstd }} ` - -DCMAKE_CXX_FLAGS="${{ matrix.extra_flags }}" - + cmake -S . -B build ` + -DGSL_BENCHMARK=ON ` + -G "${{ matrix.generator }}" @tsArg ` + -DCMAKE_CXX_STANDARD=${{ matrix.cppstd }}" - name: Build shell: pwsh @@ -258,8 +96,6 @@ jobs: cmake --build build --target span_bench ` --config Release -j $env:NUMBER_OF_PROCESSORS - - # Multi-config generator places the binary under build\Release\ - name: Run benchmark shell: pwsh run: | @@ -267,147 +103,54 @@ jobs: --benchmark_format=json ` --benchmark_repetitions=10 ` --benchmark_report_aggregates_only=true ` - --benchmark_out=results_${{ matrix.label }}.json - + --benchmark_out=results_${{ format('{0}-cpp{1}', matrix.toolset || 'MSVC', matrix.cppstd) }}.json - name: Upload results uses: actions/upload-artifact@v4 with: - name: bench-${{ matrix.label }} - path: results_${{ matrix.label }}.json + name: bench-${{ format('{0}-cpp{1}', matrix.toolset || 'MSVC', matrix.cppstd) }} + path: results_${{ format('{0}-cpp{1}', matrix.toolset || 'MSVC', matrix.cppstd) }}.json retention-days: 7 - -# ───────────────────────────────────────────────────────────────────────────── -# MACOS — Apple Clang via Xcode (latest) (× C++20 and C++23) -# -# Notes: -# • macos-14 = Apple Silicon (M1). Switch to macos-13 for Intel if needed. -# • No extra compiler install — Apple Clang from Xcode is used directly. -# • -j uses sysctl (macOS equivalent of nproc). -# ───────────────────────────────────────────────────────────────────────────── benchmark-macos: - name: macOS / ${{ matrix.label }} - runs-on: macos-14 + name: macOS / ${{ format('AppleClang-cpp{0}', matrix.cppstd) }} + runs-on: macos-latest strategy: fail-fast: false matrix: - include: - - label: AppleClang-cpp20 - cppstd: 20 - extra_flags: "" - - - - label: AppleClang-cpp23 - cppstd: 23 - extra_flags: "" - - + cppstd: [ 20, 23 ] steps: - uses: actions/checkout@v4 - - # ── FetchContent cache ───────────────────────────────────────────── - name: Cache FetchContent dependencies uses: actions/cache@v4 with: path: build/_deps - key: fetchcontent-macos-${{ matrix.cppstd }}-${{ hashFiles('CMakeLists.txt') }} + key: fetchcontent-macos-${{ matrix.cppstd }}-${{ hashFiles('benchmark/CMakeLists.txt') }} restore-keys: | fetchcontent-macos-${{ matrix.cppstd }}- - - # ── Configure → Build → Run ──────────────────────────────────────── - name: Configure run: | - cmake -S benchmark -B build \ + cmake -S . -B build \ + -DGSL_BENCHMARK=ON \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_STANDARD=${{ matrix.cppstd }} \ - -DCMAKE_CXX_FLAGS="${{ matrix.extra_flags }}" - + -DCMAKE_CXX_STANDARD=${{ matrix.cppstd }}" - name: Build run: cmake --build build --target span_bench -j$(sysctl -n hw.logicalcpu) - - name: Run benchmark run: | ./build/span_bench \ --benchmark_format=json \ --benchmark_repetitions=10 \ --benchmark_report_aggregates_only=true \ - --benchmark_out=results_${{ matrix.label }}.json - + --benchmark_out=results_${{ format('AppleClang-cpp{0}', matrix.cppstd) }}.json - name: Upload results uses: actions/upload-artifact@v4 with: - name: bench-${{ matrix.label }} - path: results_${{ matrix.label }}.json - retention-days: 7 - - -# ───────────────────────────────────────────────────────────────────────────── -# COMMENT — collect every result artifact → one PR comment -# -# Waits for all three OS groups. `if: always()` ensures this runs even when -# some benchmark jobs fail — partial results are always reported. -# The CI check fails at the very end (after posting) if any regression found. -# ───────────────────────────────────────────────────────────────────────────── - comment: - name: Post PR comment - needs: [benchmark-linux, benchmark-windows, benchmark-macos] - runs-on: ubuntu-latest - if: always() && github.event_name == 'pull_request' - permissions: - pull-requests: write - - - steps: - - uses: actions/checkout@v4 - - - - name: Download all result artifacts - uses: actions/download-artifact@v4 - with: - pattern: bench-* - merge-multiple: true # flatten all artifacts into CWD - - - # continue-on-error so the comment step always runs even on regression. - - name: Generate regression report - id: report - run: | - python3 benchmark/check_regression.py \ - --threshold 0.15 \ - --output comment.md \ - --workflow-url "https://github.com/${{ github.repository }}/blob/tree/main/.github/workflows/span_benchmark.yml" \ - results_*.json - continue-on-error: true - - - - name: Find existing bot comment - uses: peter-evans/find-comment@v3 - id: find_comment - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: github-actions[bot] - body-includes: "" - - - # One comment per PR, updated on every push — not a flood of new ones. - - name: Create or update PR comment - uses: peter-evans/create-or-update-comment@v4 - with: - comment-id: ${{ steps.find_comment.outputs.comment-id }} - issue-number: ${{ github.event.pull_request.number }} - body-path: comment.md - edit-mode: replace - - - # Surface the failure visibly in the CI check panel. - - name: Fail if regression detected - if: steps.report.outcome == 'failure' - run: | - echo "::error::Performance regression detected. See the PR comment for the full table." - exit 1 \ No newline at end of file + name: bench-${{ format('AppleClang-cpp{0}', matrix.cppstd) }} + path: results_${{ format('AppleClang-cpp{0}', matrix.cppstd) }}.json + retention-days: 7 \ No newline at end of file diff --git a/.github/workflows/span_benchmark_comment.yml b/.github/workflows/span_benchmark_comment.yml new file mode 100644 index 00000000..a7ab68c2 --- /dev/null +++ b/.github/workflows/span_benchmark_comment.yml @@ -0,0 +1,76 @@ +name: Span Benchmark Comment + +on: + workflow_run: + workflows: ["Span Benchmark"] + types: [completed] + +permissions: + pull-requests: write + +jobs: + comment: + name: Post PR comment + runs-on: ubuntu-latest + if: always() + + steps: + - uses: actions/checkout@v6 + + # Download all bench-* artifacts from the triggering workflow run. + - name: Download all result artifacts + uses: actions/download-artifact@v4 + with: + pattern: bench-* + merge-multiple: true + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ github.event.workflow_run.id }} + + - name: Generate regression report + id: report + run: | + python3 benchmark/check_regression.py \ + --threshold 0.15 \ + --output comment.md \ + results_*.json + continue-on-error: true + + # Resolve the PR number from the triggering workflow run. + - name: Get PR number + id: pr + uses: actions/github-script@v7 + with: + script: | + const runs = await github.rest.pulls.list({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + }); + const pr = runs.data.find(p => + p.head.sha === context.payload.workflow_run.head_sha + ); + return pr ? pr.number : null; + + - name: Find existing bot comment + if: steps.pr.outputs.result != 'null' + uses: peter-evans/find-comment@v3 + id: find_comment + with: + issue-number: ${{ steps.pr.outputs.result }} + comment-author: github-actions[bot] + body-includes: "" + + - name: Create or update PR comment + if: steps.pr.outputs.result != 'null' + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.find_comment.outputs.comment-id }} + issue-number: ${{ steps.pr.outputs.result }} + body-path: comment.md + edit-mode: replace + + - name: Fail if regression detected + if: steps.report.outcome == 'failure' + run: | + echo "::error::Performance regression detected. See the PR comment for the full table." + exit 1 \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index b306dc1c..1cd1c148 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,6 +10,7 @@ string(COMPARE EQUAL ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_SOURCE_DIR} PROJECT_IS_ option(GSL_INSTALL "Generate and install GSL target" ${PROJECT_IS_TOP_LEVEL}) option(GSL_TEST "Build and perform GSL tests" ${PROJECT_IS_TOP_LEVEL}) +option(GSL_BENCHMARK "Build span benchmarks" ${PROJECT_IS_TOP_LEVEL}) # The implementation generally assumes a platform that implements C++14 support target_compile_features(GSL INTERFACE "cxx_std_14") @@ -19,6 +20,10 @@ add_subdirectory(include) target_sources(GSL INTERFACE $) +if(GSL_BENCHMARK) + add_subdirectory(benchmark) +endif() + if (GSL_TEST) enable_testing() add_subdirectory(tests) diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 36183fec..2bdf7da4 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -1,87 +1,57 @@ -# benchmark/CMakeLists.txt -# -# Self-contained build for the gsl::span vs std::span benchmark. -# All dependencies are fetched automatically via FetchContent: -# - google/benchmark v1.9.0 -# - google/googletest release-1.12.1 (benchmark's internal dep) -# - microsoft/GSL v4.1.0 -# -# Usage (from this directory): -# cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -# cmake --build build --target span_bench -# ./build/span_bench --benchmark_format=json --benchmark_repetitions=10 \ -# --benchmark_report_aggregates_only=true \ -# --benchmark_out=results.json -# -# To override the compiler or standard (as CI does): -# cmake -S . -B build -DCMAKE_BUILD_TYPE=Release \ -# -DCMAKE_CXX_COMPILER=clang++-18 \ -# -DCMAKE_CXX_STANDARD=23 +cmake_minimum_required(VERSION 3.14...3.16) -cmake_minimum_required(VERSION 3.20) +project(GSLBenchmarks LANGUAGES CXX) -project(gsl_span_benchmark CXX) +# ── C++ standard ─────────────────────────────────────────────────────────────── +# Minimum is 20 because std::span (required for this benchmark) is C++20 only. +# If a consumer passes GSL_CXX_STANDARD < 20 we clamp and warn. +set(GSL_CXX_STANDARD "20" CACHE STRING "Use c++ standard") -# ── C++ standard ────────────────────────────────────────────────────────────── -# Default C++20 (minimum for std::span). CI overrides via -DCMAKE_CXX_STANDARD. -set(CMAKE_CXX_STANDARD 20 CACHE STRING "C++ standard") +if(GSL_CXX_STANDARD LESS 20) + message(WARNING + "GSL_CXX_STANDARD=${GSL_CXX_STANDARD} is too old for the span benchmark " + "(std::span requires C++20). Overriding to 20.") + set(GSL_CXX_STANDARD "20" CACHE STRING "Use c++ standard" FORCE) +endif() + +set(CMAKE_CXX_STANDARD ${GSL_CXX_STANDARD}) +set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_EXTENSIONS NO) -set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type") +# Makes Visual Studio organise targets into folders +set_property(GLOBAL PROPERTY USE_FOLDERS ON) -# ── FetchContent dependencies ────────────────────────────────────────────────── include(FetchContent) -# Suppress benchmark's own test suite — we don't need it. +# Suppress benchmark's own test suite set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE) - -FetchContent_Declare( - googletest - GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG release-1.12.1 - GIT_SHALLOW ON -) +set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "" FORCE) FetchContent_Declare( googlebenchmark GIT_REPOSITORY https://github.com/google/benchmark.git - GIT_TAG v1.9.0 + GIT_TAG v1.9.5 GIT_SHALLOW ON ) +FetchContent_MakeAvailable(googlebenchmark) -FetchContent_MakeAvailable(googletest googlebenchmark) - -# Use the GSL from the local checkout — NOT a pinned release tag. -# The benchmark/ folder sits one level below the repo root, so -# CMAKE_CURRENT_SOURCE_DIR/.. resolves to the repo root which contains -# the real include/gsl/span header being modified by the PR. -# This is what makes the CI actually test the PR's changes. -add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/.. gsl_root) - -# ── Benchmark executable ─────────────────────────────────────────────────────── add_executable(span_bench span_bench.cpp) -# Link benchmark::benchmark only — NOT benchmark::benchmark_main. -# span_bench.cpp uses the BENCHMARK_MAIN() macro which expands to its own -# main(), so linking benchmark_main would cause a duplicate-symbol linker error. target_link_libraries(span_bench PRIVATE - benchmark::benchmark # provides the benchmark framework + runner - Microsoft.GSL::GSL # provides gsl::span and friends + benchmark::benchmark + Microsoft.GSL::GSL ) +set_target_properties(span_bench PROPERTIES FOLDER "benchmarks") + # ── Optimisation flags ───────────────────────────────────────────────────────── -# -O3 and -march=native let the compiler apply the same optimisations to both -# gsl::span and std::span, keeping the comparison fair. -# -fno-omit-frame-pointer keeps perf/profiling usable if you need it locally. target_compile_options(span_bench PRIVATE $<$: -O3 -march=native - -fno-omit-frame-pointer > $<$: /O2 - /GL # whole-program optimisation + /GL > ) \ No newline at end of file diff --git a/benchmark/README.md b/benchmark/README.md deleted file mode 100644 index fdda94fe..00000000 --- a/benchmark/README.md +++ /dev/null @@ -1,144 +0,0 @@ -# `gsl::span` vs `std::span` Benchmark - -Performance parity tracking between `gsl::span` and `std::span` across all supported compilers, platforms, and C++ standards — as part of [microsoft/GSL#1167](https://github.com/microsoft/GSL/issues/1167) and [microsoft/GSL#1165](https://github.com/microsoft/GSL/issues/1165). - ---- - -## Overview - -`gsl::span` should be a zero-overhead abstraction over `std::span`. This benchmark suite verifies that claim continuously — on every PR — so performance regressions are caught before they land in main. - -The comparison strategy is **in-run ratio** (`gsl_ns / std_ns`): both spans are measured in the same process on the same machine at the same moment, so runner noise cancels out. A ratio close to `1.0` means parity. If `gsl::span` is more than **15% slower** than `std::span` on any benchmark, CI flags it and posts a detailed table in the PR comment. - ---- - -## Benchmarks - -All benchmarks run on a sorted vector of 1000 integers. - -| Benchmark | What it tests | -|---|---| -| `IsSorted` | `std::is_sorted` via span iterators | -| `IsSortedRanges` | `std::ranges::is_sorted` via the span range interface | -| `IsSortedCustom` | Custom hand-rolled `is_sorted` loop via span iterators | -| `MinElementAlgorithm` | `std::min_element` via span iterators | -| `MinElementRangeFor` | Range-for loop with a custom min accumulator | - -Each benchmark has a `StdSpan` and `GslSpan` variant. The Python script pairs them by name and computes the ratio. - ---- - -## CI Matrix - -The benchmark runs on every pull request across 13 configurations: - -| OS | Compiler | C++ Standard | -|---|---|---| -| ubuntu-latest | GCC 13 | C++20 | -| ubuntu-latest | GCC 14 | C++20, C++23 | -| ubuntu-latest | Clang 17 | C++20 | -| ubuntu-latest | Clang 18 | C++20, C++23 | -| windows-latest | MSVC 2022 | C++20, C++23 | -| windows-latest | clang-cl (VS 2022 bundled) | C++20, C++23 | -| macos-14 (Apple Silicon) | Apple Clang (Xcode latest) | C++20, C++23 | - -Results from all 13 jobs are collected and posted as a **single PR comment**, updated on every push. - ---- - -## Repository Layout - -``` -benchmark/ -├── CMakeLists.txt # self-contained build — fetches benchmark + googletest -├── span_bench.cpp # the benchmark source -├── check_regression.py # parses JSON results, writes the PR comment markdown -└── README.md # this file - -.github/workflows/ -└── span_benchmark.yml # CI workflow -``` - -The benchmark folder is self-contained. `CMakeLists.txt` fetches `google/benchmark` (v1.9.0) and `google/googletest` via `FetchContent`. GSL itself is sourced from the **local repo checkout** — not a pinned tag — so the benchmark always tests the code in the PR, not a release. - ---- - -## Running Locally - -**Prerequisites:** CMake ≥ 3.20, a C++20-capable compiler, internet access for `FetchContent`. - -```bash -# From the benchmark/ directory: -cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -cmake --build build --target span_bench - -# Run with JSON output (matches what CI does): -./build/span_bench \ - --benchmark_format=json \ - --benchmark_repetitions=10 \ - --benchmark_report_aggregates_only=true \ - --benchmark_out=results.json - -# Generate the regression report locally: -python3 check_regression.py --threshold 0.15 results.json -``` - -To test a specific compiler or standard: - -```bash -cmake -S . -B build \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_COMPILER=clang++-18 \ - -DCMAKE_CXX_STANDARD=23 -``` - ---- - -## Regression Detection - -The `check_regression.py` script: - -1. Reads one or more Google Benchmark JSON files (one per CI matrix config) -2. Pairs `*StdSpan` benchmarks with their `*GslSpan` counterparts by name -3. Computes `ratio = gsl_mean / std_mean` using the 10-repetition mean -4. Flags any ratio above `1 + threshold` (default **15%**) as a regression -5. Writes a Markdown table per config, collected into a single PR comment -6. Exits with code `1` if any regression is found — failing the CI check - -``` -python3 check_regression.py [--threshold 0.15] [--output report.md] results_*.json -``` - -### Example PR comment output - -``` -## 📊 gsl::span vs std::span benchmark results - -### `GCC-14-cpp20` -| Benchmark | std mean | std σ | gsl mean | gsl σ | ratio | status | -|----------------------|----------|--------|----------|--------|-------|----------| -| IsSorted | 124.1 ns | ±1.2% | 125.3 ns | ±1.4% | 1.01× | ✅ 1.01× | -| IsSortedRanges | 88.4 ns | ±0.9% | 89.1 ns | ±1.1% | 1.01× | ✅ 1.01× | -| IsSortedCustom | 112.6 ns | ±1.5% | 113.2 ns | ±1.3% | 1.01× | ✅ 1.01× | -| MinElementAlgorithm | 95.2 ns | ±1.0% | 96.0 ns | ±1.2% | 1.01× | ✅ 1.01× | -| MinElementRangeFor | 98.7 ns | ±1.1% | 99.4 ns | ±0.8% | 1.01× | ✅ 1.01× | -``` - -Status icons: -- ✅ — within the threshold (parity) -- 🔴 — `gsl::span` is more than 15% slower (regression, CI fails) -- 🟢 — `gsl::span` is more than 15% faster (improvement, noted but not a failure) - ---- - -## Noise Considerations - -GitHub-hosted runners are shared VMs with a typical noise floor of **±10–15%** on absolute timing. The ratio strategy mitigates this because both span variants experience the same CPU conditions simultaneously. The 15% threshold is chosen to sit just above the noise floor — tight enough to catch real regressions, loose enough to avoid false positives on every PR. - -To further reduce variance, each benchmark runs **10 repetitions** and the script uses the **mean** (not a single sample) for the ratio calculation. The stddev column in the comment table lets reviewers eyeball how stable each measurement was. - ---- - -## Background - -This benchmark was created in response to [microsoft/GSL#1167](https://github.com/microsoft/GSL/issues/1167), which highlighted the need to maintain performance parity with `std::span`, especially when [Safe Buffers / `-fbounds-safety`](https://clang.llvm.org/docs/SafeBuffers.html) are enabled. The initial benchmark scaffolding was provided by @galenelias. \ No newline at end of file diff --git a/benchmark/check_regression.py b/benchmark/check_regression.py index 69988c9f..4ff859a5 100644 --- a/benchmark/check_regression.py +++ b/benchmark/check_regression.py @@ -131,7 +131,7 @@ def fmt_stddev(stddev: float, mean: float) -> str: # ─── report builder ─────────────────────────────────────────────────────────── -def build_report(json_paths: list[str], threshold: float, workflow_url: str = None) -> tuple[str, bool]: +def build_report(json_paths: list[str], threshold: float) -> tuple[str, bool]: """ Returns (markdown_text, had_regression). """ @@ -202,6 +202,7 @@ def build_report(json_paths: list[str], threshold: float, workflow_url: str = No config_regression = False for p in pairs: ratio, status = ratio_and_status(p["gsl_mean"], p["std_mean"], threshold) + ratio_str = "—" if ratio is None else f"{ratio:.2f}×" if "regression" in status: had_regression = True config_regression = True @@ -212,7 +213,7 @@ def build_report(json_paths: list[str], threshold: float, workflow_url: str = No f"| {fmt_stddev(p['std_stddev'], p['std_mean'])} " f"| {fmt(p['gsl_mean'])} " f"| {fmt_stddev(p['gsl_stddev'], p['gsl_mean'])} " - f"| {ratio:.2f}× " + f"| {ratio_str} " f"| {status} |" ) @@ -229,14 +230,6 @@ def build_report(json_paths: list[str], threshold: float, workflow_url: str = No if not found_any: lines.append("> ❌ No benchmark results could be loaded.") - # Footer - lines.append("---") - lines.append( - "_Ratio = `gsl_ns / std_ns`. " - "Values close to 1.0 mean performance parity. " - f"Run by [span-benchmark CI]({workflow_url})._" - ) - return "\n".join(lines), had_regression @@ -264,15 +257,9 @@ def main(): metavar="FILE", help="Write Markdown report to FILE instead of stdout.", ) - parser.add_argument( - "--workflow-url", - default=None, - metavar="URL", - help="GitHub URL to the workflow YAML (shows in the report footer).", - ) args = parser.parse_args() - report, had_regression = build_report(args.json_files, args.threshold, args.workflow_url) + report, had_regression = build_report(args.json_files, args.threshold) if args.output: Path(args.output).write_text(report, encoding="utf-8") diff --git a/benchmark/span_bench.cpp b/benchmark/span_bench.cpp index a5f9e4a0..f8a92ad5 100644 --- a/benchmark/span_bench.cpp +++ b/benchmark/span_bench.cpp @@ -1,10 +1,10 @@ -// Setup: -// 1) cmake . -DCMAKE_BUILD_TYPE=Release -// 2) cmake --build . --config Release - #include "gsl/span" +#include #include -#include +#include +#include +#include +#include static std::vector make_vector() { From f5c2eb0e5f479944404aeeb86c5ee9c091714ce7 Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Gafoor <94980075+Gafoor2005@users.noreply.github.com> Date: Mon, 8 Jun 2026 13:28:54 +0000 Subject: [PATCH 14/15] Update set GSL_BENCHMARK option and fix CMake command syntax --- .github/workflows/span_benchmark.yml | 6 +++--- CMakeLists.txt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/span_benchmark.yml b/.github/workflows/span_benchmark.yml index d1743483..aaccae78 100644 --- a/.github/workflows/span_benchmark.yml +++ b/.github/workflows/span_benchmark.yml @@ -40,7 +40,7 @@ jobs: -DGSL_BENCHMARK=ON \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \ - -DCMAKE_CXX_STANDARD=${{ matrix.cppstd }}" + -DCMAKE_CXX_STANDARD=${{ matrix.cppstd }} \ - name: Build run: cmake --build build --target span_bench -j$(nproc) @@ -88,7 +88,7 @@ jobs: cmake -S . -B build ` -DGSL_BENCHMARK=ON ` -G "${{ matrix.generator }}" @tsArg ` - -DCMAKE_CXX_STANDARD=${{ matrix.cppstd }}" + -DCMAKE_CXX_STANDARD=${{ matrix.cppstd }} ` - name: Build shell: pwsh @@ -135,7 +135,7 @@ jobs: cmake -S . -B build \ -DGSL_BENCHMARK=ON \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_STANDARD=${{ matrix.cppstd }}" + -DCMAKE_CXX_STANDARD=${{ matrix.cppstd }} \ - name: Build run: cmake --build build --target span_bench -j$(sysctl -n hw.logicalcpu) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1cd1c148..f0f9366b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ string(COMPARE EQUAL ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_SOURCE_DIR} PROJECT_IS_ option(GSL_INSTALL "Generate and install GSL target" ${PROJECT_IS_TOP_LEVEL}) option(GSL_TEST "Build and perform GSL tests" ${PROJECT_IS_TOP_LEVEL}) -option(GSL_BENCHMARK "Build span benchmarks" ${PROJECT_IS_TOP_LEVEL}) +option(GSL_BENCHMARK "Build span benchmarks" OFF) # The implementation generally assumes a platform that implements C++14 support target_compile_features(GSL INTERFACE "cxx_std_14") From 8d73c5688ba8390f084f80fa8cbaf967647b40de Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Gafoor <94980075+Gafoor2005@users.noreply.github.com> Date: Wed, 1 Jul 2026 04:58:32 +0000 Subject: [PATCH 15/15] fix: add a safe gaurd for parser --- benchmark/check_regression.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmark/check_regression.py b/benchmark/check_regression.py index 4ff859a5..f7a20123 100644 --- a/benchmark/check_regression.py +++ b/benchmark/check_regression.py @@ -55,7 +55,9 @@ def parse_benchmarks(data: dict) -> dict[str, dict]: continue # Strip the trailing _mean / _stddev to get the canonical name - base_name = name[: name.rfind(f"_{agg}")] + suffix = f"_{agg}" + assert name.endswith(suffix) + base_name = name[:-len(suffix)] entry = result.setdefault(base_name, {"mean": None, "stddev": None}) entry[agg] = bm.get("real_time") or bm.get("cpu_time", 0.0)