feat(ci): parallelize reference tests via bucketing

.github/workflows/_backend-tests.yml

@@ -0,0 +1,120 @@
+name: Backend tests
+on:
+  workflow_call:
+    inputs:
+      julia-version:
+        required: true
+        type: string
+      backend:
+        required: true
+        type: string
+        description: 'Backend name, e.g. CairoMakie, GLMakie, WGLMakie'
+      nbuckets:
+        required: false
+        type: number
+        default: 2
+      system-packages:
+        required: false
+        type: string
+        default: ''
+        description: 'apt packages to install (space-separated)'
+      test-prefix:
+        required: false
+        type: string
+        default: ''
+        description: 'Command prefix for test execution, e.g. "DISPLAY=:0 xvfb-run -s \"-screen 0 1024x768x24\""'
+
+jobs:
+  setup:
+    name: Setup ${{ inputs.backend }} Julia ${{ inputs.julia-version }}
+    runs-on: ubuntu-latest
+    outputs:
+      bucket-list: ${{ steps.buckets.outputs.list }}
+    steps:
+      - id: buckets
+        run: echo "list=$(seq 1 ${{ inputs.nbuckets }} | jq -cs .)" >> $GITHUB_OUTPUT
+      - uses: actions/checkout@v6
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: ${{ inputs.julia-version }}
+      - uses: julia-actions/cache@v3
+        with:
+          cache-name: julia-cache-${{ inputs.backend }}-${{ inputs.julia-version }}
+          include-matrix: false
+      - name: Install system dependencies
+        if: inputs.system-packages != ''
+        run: sudo apt-get update && sudo apt-get install -y ${{ inputs.system-packages }}
+      - name: Install Julia dependencies
+        shell: julia --project=monorepo {0}
+        env:
+          # Don't precompile here; Pkg.test below handles it with the correct
+          # test environment and flags.
+          JULIA_PKG_PRECOMPILE_AUTO: '0'
+        run: |
+          using Pkg;
+          pkg"registry up"
+          Pkg.update()
+          pkg"dev ./Makie ./${{ inputs.backend }} ./ReferenceTests ./ComputePipeline"
+      - name: Precompile via Pkg.test
+        env:
+          PRECOMPILE_ONLY: 'true'
+          # Portable CPU target so pkgimages work across GitHub's heterogeneous
+          # runner fleet (mix of AMD Zen3, Intel Haswell, etc.).
+          JULIA_CPU_TARGET: 'generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)'
+        # NOTE: coverage is intentionally omitted. On Julia 1.10, coverage=true
+        # disables pkgimages (use_pkgimages=false), causing cache flag mismatches
+        # between setup and bucket jobs. If coverage is re-enabled, this must be
+        # set identically here and in the bucket test step.
+        run: ${{ inputs.test-prefix }} julia --color=yes --project=monorepo -e 'using Pkg; Pkg.test("${{ inputs.backend }}")'
+      - name: Package depot and Manifest
+        run: >
+          cp monorepo/Manifest.toml ~/.julia/monorepo-Manifest.toml &&
+          tar -cf /tmp/depot.tar.zst --use-compress-program zstdmt -C ~
+          .julia/compiled .julia/packages .julia/artifacts .julia/scratchspaces .julia/monorepo-Manifest.toml
+      - name: Upload depot tarball
+        uses: actions/upload-artifact@v7
+        with:
+          name: julia-depot-${{ inputs.backend }}-${{ inputs.julia-version }}
+          path: /tmp/depot.tar.zst
+          retention-days: 1
+
+  test:
+    name: ${{ inputs.backend }} Julia ${{ inputs.julia-version }} bucket ${{ matrix.bucket }}/${{ inputs.nbuckets }}
+    needs: setup
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        bucket: ${{ fromJSON(needs.setup.outputs.bucket-list) }}
+    steps:
+      - uses: actions/checkout@v6
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: ${{ inputs.julia-version }}
+      - name: Download depot tarball
+        uses: actions/download-artifact@v8
+        with:
+          name: julia-depot-${{ inputs.backend }}-${{ inputs.julia-version }}
+      - name: Extract depot and Manifest
+        run: tar -xf depot.tar.zst --use-compress-program unzstd -C ~ && mkdir -p monorepo && cp ~/.julia/monorepo-Manifest.toml monorepo/Manifest.toml
+      - name: Install system dependencies
+        if: inputs.system-packages != ''
+        run: sudo apt-get update && sudo apt-get install -y ${{ inputs.system-packages }}
+      - name: Run the tests
+        continue-on-error: true
+        env:
+          REFTEST_BUCKET: ${{ matrix.bucket }}
+          REFTEST_NBUCKETS: ${{ inputs.nbuckets }}
+          JULIA_CPU_TARGET: 'generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)'
+        run: >
+          ${{ inputs.test-prefix }}
+          julia --color=yes --project=monorepo -e 'using Pkg; Pkg.test("${{ inputs.backend }}")'
+          && echo "TESTS_SUCCESSFUL=true" >> $GITHUB_ENV
+      - name: Upload test Artifacts
+        uses: actions/upload-artifact@v7
+        with:
+          name: ReferenceImages_${{ inputs.backend }}_${{ inputs.julia-version }}_${{ matrix.bucket }}
+          path: ./${{ inputs.backend }}/test/reference_images/
+      - name: Fail after artifacts if tests failed
+        if: ${{ env.TESTS_SUCCESSFUL != 'true' }}
+        run: exit 1

.github/workflows/_compute-pipeline.yml

@@ -26,6 +26,8 @@ jobs:
       - uses: julia-actions/cache@v3
       - name: Develop and test ComputePipeline
         shell: julia --project=monorepo {0}
+        env:
+          JULIA_PKG_PRECOMPILE_AUTO: '0'
         run: |
           using Pkg
           # dev mono repo versions

.github/workflows/_makie.yml

@@ -26,6 +26,8 @@ jobs:
       - uses: julia-actions/cache@v3
       - name: Develop and test Makie
         shell: julia --project=monorepo {0}
+        env:
+          JULIA_PKG_PRECOMPILE_AUTO: '0'
         run: |
           using Pkg
           # dev mono repo versions

.github/workflows/_reference-tests.yml

@@ -4,56 +4,98 @@ on:
 
 jobs:
   cairomakie:
-    uses: ./.github/workflows/_cairomakie.yml
+    strategy:
+      matrix:
+        julia-version: ['1.10', '1']
+    uses: ./.github/workflows/_backend-tests.yml
+    with:
+      julia-version: ${{ matrix.julia-version }}
+      backend: CairoMakie
 
   glmakie:
-    uses: ./.github/workflows/_glmakie.yml
+    strategy:
+      matrix:
+        julia-version: ['1.10', '1']
+    uses: ./.github/workflows/_backend-tests.yml
+    with:
+      julia-version: ${{ matrix.julia-version }}
+      backend: GLMakie
+      system-packages: xorg-dev mesa-utils xvfb libgl1 freeglut3-dev libxrandr-dev libxinerama-dev libxcursor-dev libxi-dev libxext-dev xsettingsd x11-xserver-utils
+      test-prefix: 'DISPLAY=:0 xvfb-run -s "-screen 0 1024x768x24"'
 
   wglmakie:
-    uses: ./.github/workflows/_wglmakie.yml
+    strategy:
+      matrix:
+        julia-version: ['1.10', '1']
+    uses: ./.github/workflows/_backend-tests.yml
+    with:
+      julia-version: ${{ matrix.julia-version }}
+      backend: WGLMakie
+      nbuckets: 4
+      system-packages: xorg-dev libosmesa6 libgl1-mesa-dri mesa-utils xvfb libgl1 freeglut3-dev libxrandr-dev libxinerama-dev libxcursor-dev libxi-dev libxext-dev
+      test-prefix: 'DISPLAY=:0 xvfb-run -s "-screen 0 2048x2048x24"'
 
   consolidation:
     name: Merge artifacts
     runs-on: ubuntu-latest
     if: always()
     needs: [cairomakie, glmakie, wglmakie]
     steps:
-      - uses: actions/download-artifact@v8
-        with:
-          name: ReferenceImages_WGLMakie_1
-          path: ./ReferenceImages/WGLMakie
-      - uses: actions/download-artifact@v8
+      - name: Download all bucket artifacts for Julia 1
+        uses: actions/download-artifact@v8
         with:
-          name: ReferenceImages_CairoMakie_1
-          path: ./ReferenceImages/CairoMakie
-      - uses: actions/download-artifact@v8
-        with:
-          name: ReferenceImages_GLMakie_1
-          path: ./ReferenceImages/GLMakie
+          pattern: ReferenceImages_*_1_*
+          path: ./BucketArtifacts/
       - name: Consolidate reference image folders
         run: |
-          baseDir="./ReferenceImages"
+          baseDir="./BucketArtifacts"
+          outDir="./ReferenceImagesCombined"
+          mkdir -p "$outDir"
+
+          # Copy reference images from the first available artifact (same across all buckets/backends)
+          first_artifact=$(ls -d ${baseDir}/ReferenceImages_*_1_*/ 2>/dev/null | head -1)
+          if [ -n "$first_artifact" ] && [ -d "${first_artifact}/reference" ]; then
+            cp -r "${first_artifact}/reference/." "$outDir/reference/"
+          fi
+
+          # Initialize output files
+          > "$outDir/scores.tsv"
+          > "$outDir/new_files.txt"
+          > "$outDir/missing_files.txt"
+
+          for backend in CairoMakie GLMakie WGLMakie; do
+              mkdir -p "$outDir/recorded/${backend}/"
+
+              # Merge all buckets for this backend
+              for bucket_dir in ${baseDir}/ReferenceImages_${backend}_1_*/; do
+                  [ -d "$bucket_dir" ] || continue
+
+                  # Copy recorded images (no overlap between buckets)
+                  if [ -d "${bucket_dir}/recorded/${backend}/" ]; then
+                      cp -r "${bucket_dir}/recorded/${backend}/." "$outDir/recorded/${backend}/"
+                  fi
 
-          # Create new top-level directory for combined files
-          mkdir -p "./ReferenceImagesCombined"
+                  # Concatenate scores and new_files from each bucket
+                  [ -f "${bucket_dir}/scores.tsv" ] && cat "${bucket_dir}/scores.tsv" >> "$outDir/scores.tsv"
+                  [ -f "${bucket_dir}/new_files.txt" ] && cat "${bucket_dir}/new_files.txt" >> "$outDir/new_files.txt"
+              done
 
-          # Copy the reference folder from GLMakie, it's the same for all backends
-          cp -r "${baseDir}/GLMakie/reference/." "./ReferenceImagesCombined/reference/"
+              # Compute missing_files: reference files that were neither recorded nor skipped
+              if [ -d "$outDir/reference/${backend}" ]; then
+                  first_bucket=$(ls -d ${baseDir}/ReferenceImages_${backend}_1_*/ 2>/dev/null | head -1)
 
-          # Initialize empty files for concatenation
-          > "./ReferenceImagesCombined/scores.tsv"
-          > "./ReferenceImagesCombined/new_files.txt"
+                  # Build sorted list of files that are accounted for (recorded + skipped)
+                  {
+                      [ -d "$outDir/recorded/${backend}" ] && (cd "$outDir/recorded" && find "${backend}" -type f)
+                      [ -n "$first_bucket" ] && [ -f "${first_bucket}/skipped_names.txt" ] && cat "${first_bucket}/skipped_names.txt"
+                  } | sort -u > /tmp/have_or_skip.txt
 
-          # Loop through the directories and concatenate the files, and copy recorded folders
-          for dir in WGLMakie CairoMakie GLMakie; do
-              # Concatenate scores.tsv, new_files.txt and missing_files.txt
-              cat "${baseDir}/${dir}/scores.tsv" >> "./ReferenceImagesCombined/scores.tsv"
-              cat "${baseDir}/${dir}/new_files.txt" >> "./ReferenceImagesCombined/new_files.txt"
-              cat "${baseDir}/${dir}/missing_files.txt" >> "./ReferenceImagesCombined/missing_files.txt"
+                  # Reference files
+                  (cd "$outDir/reference" && find "${backend}" -type f) | sort > /tmp/ref_files.txt
 
-              # Copy recorded folder
-              mkdir -p "./ReferenceImagesCombined/recorded/${dir}/"
-              cp -r "${baseDir}/${dir}/recorded/${dir}/." "./ReferenceImagesCombined/recorded/${dir}/"
+                  # Missing = reference - (recorded ∪ skipped)
+                  comm -23 /tmp/ref_files.txt /tmp/have_or_skip.txt >> "$outDir/missing_files.txt"
+              fi
           done
 
           echo "Files and folders have been successfully combined into ReferenceImagesCombined."

.github/workflows/_rprmakie.yml

@@ -25,6 +25,8 @@ jobs:
       - uses: julia-actions/cache@v3
       - name: Install Julia dependencies
         shell: julia --project=monorepo {0}
+        env:
+          JULIA_PKG_PRECOMPILE_AUTO: '0'
         run: |
           using Pkg;
           pkg"registry up"