diff --git a/.codecov.yml b/.codecov.yml deleted file mode 100644 index 6adca0c9580..00000000000 --- a/.codecov.yml +++ /dev/null @@ -1,7 +0,0 @@ -comment: false -coverage: - status: - project: false - patch: false -github_checks: - annotations: false diff --git a/.github/workflows/_backend-tests.yml b/.github/workflows/_backend-tests.yml new file mode 100644 index 00000000000..b7f49164916 --- /dev/null +++ b/.github/workflows/_backend-tests.yml @@ -0,0 +1,120 @@ +name: Backend tests +on: + workflow_call: + inputs: + julia-version: + required: true + type: string + backend: + required: true + type: string + description: 'Backend name, e.g. CairoMakie, GLMakie, WGLMakie' + nbuckets: + required: false + type: number + default: 2 + system-packages: + required: false + type: string + default: '' + description: 'apt packages to install (space-separated)' + test-prefix: + required: false + type: string + default: '' + description: 'Command prefix for test execution, e.g. "DISPLAY=:0 xvfb-run -s \"-screen 0 1024x768x24\""' + +jobs: + setup: + name: Setup ${{ inputs.backend }} Julia ${{ inputs.julia-version }} + runs-on: ubuntu-latest + outputs: + bucket-list: ${{ steps.buckets.outputs.list }} + steps: + - id: buckets + run: echo "list=$(seq 1 ${{ inputs.nbuckets }} | jq -cs .)" >> $GITHUB_OUTPUT + - uses: actions/checkout@v6 + - uses: julia-actions/setup-julia@v2 + with: + version: ${{ inputs.julia-version }} + - uses: julia-actions/cache@v3 + with: + cache-name: julia-cache-${{ inputs.backend }}-${{ inputs.julia-version }} + include-matrix: false + - name: Install system dependencies + if: inputs.system-packages != '' + run: sudo apt-get update && sudo apt-get install -y ${{ inputs.system-packages }} + - name: Install Julia dependencies + shell: julia --project=monorepo {0} + env: + # Don't precompile here; Pkg.test below handles it with the correct + # test environment and flags. + JULIA_PKG_PRECOMPILE_AUTO: '0' + run: | + using Pkg; + pkg"registry up" + Pkg.update() + pkg"dev ./Makie ./${{ inputs.backend }} ./ReferenceTests ./ComputePipeline" + - name: Precompile via Pkg.test + env: + PRECOMPILE_ONLY: 'true' + # Portable CPU target so pkgimages work across GitHub's heterogeneous + # runner fleet (mix of AMD Zen3, Intel Haswell, etc.). + JULIA_CPU_TARGET: 'generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)' + # NOTE: coverage is intentionally omitted. On Julia 1.10, coverage=true + # disables pkgimages (use_pkgimages=false), causing cache flag mismatches + # between setup and bucket jobs. If coverage is re-enabled, this must be + # set identically here and in the bucket test step. + run: ${{ inputs.test-prefix }} julia --color=yes --project=monorepo -e 'using Pkg; Pkg.test("${{ inputs.backend }}")' + - name: Package depot and Manifest + run: > + cp monorepo/Manifest.toml ~/.julia/monorepo-Manifest.toml && + tar -cf /tmp/depot.tar.zst --use-compress-program zstdmt -C ~ + .julia/compiled .julia/packages .julia/artifacts .julia/scratchspaces .julia/monorepo-Manifest.toml + - name: Upload depot tarball + uses: actions/upload-artifact@v7 + with: + name: julia-depot-${{ inputs.backend }}-${{ inputs.julia-version }} + path: /tmp/depot.tar.zst + retention-days: 1 + + test: + name: ${{ inputs.backend }} Julia ${{ inputs.julia-version }} bucket ${{ matrix.bucket }}/${{ inputs.nbuckets }} + needs: setup + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + bucket: ${{ fromJSON(needs.setup.outputs.bucket-list) }} + steps: + - uses: actions/checkout@v6 + - uses: julia-actions/setup-julia@v2 + with: + version: ${{ inputs.julia-version }} + - name: Download depot tarball + uses: actions/download-artifact@v8 + with: + name: julia-depot-${{ inputs.backend }}-${{ inputs.julia-version }} + - name: Extract depot and Manifest + run: tar -xf depot.tar.zst --use-compress-program unzstd -C ~ && mkdir -p monorepo && cp ~/.julia/monorepo-Manifest.toml monorepo/Manifest.toml + - name: Install system dependencies + if: inputs.system-packages != '' + run: sudo apt-get update && sudo apt-get install -y ${{ inputs.system-packages }} + - name: Run the tests + continue-on-error: true + env: + REFTEST_BUCKET: ${{ matrix.bucket }} + REFTEST_NBUCKETS: ${{ inputs.nbuckets }} + JULIA_CPU_TARGET: 'generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)' + run: > + ${{ inputs.test-prefix }} + julia --color=yes --project=monorepo -e 'using Pkg; Pkg.test("${{ inputs.backend }}")' + && echo "TESTS_SUCCESSFUL=true" >> $GITHUB_ENV + - name: Upload test Artifacts + uses: actions/upload-artifact@v7 + with: + name: ReferenceImages_${{ inputs.backend }}_${{ inputs.julia-version }}_${{ matrix.bucket }} + path: ./${{ inputs.backend }}/test/reference_images/ + - name: Fail after artifacts if tests failed + if: ${{ env.TESTS_SUCCESSFUL != 'true' }} + run: exit 1 diff --git a/.github/workflows/_cairomakie.yml b/.github/workflows/_cairomakie.yml deleted file mode 100644 index afce54308a2..00000000000 --- a/.github/workflows/_cairomakie.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: CairoMakie -on: - workflow_call: - -jobs: - test: - name: CairoMakie Julia ${{ matrix.version }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - version: - - '1.10' - - '1' - os: - - ubuntu-latest - arch: - - x64 - steps: - - name: Checkout - uses: actions/checkout@v6 - - uses: julia-actions/setup-julia@v2 - with: - version: ${{ matrix.version }} - arch: ${{ matrix.arch }} - - uses: julia-actions/cache@v3 - - name: Install Julia dependencies - shell: julia --project=monorepo {0} - run: | - using Pkg; - # dev mono repo versions - pkg"registry up" - Pkg.update() - pkg"dev ./Makie ./CairoMakie ./ReferenceTests ./ComputePipeline" - - name: Run the tests - continue-on-error: true - run: > - julia --color=yes --project=monorepo -e 'using Pkg; Pkg.test("CairoMakie", coverage=true)' - && echo "TESTS_SUCCESSFUL=true" >> $GITHUB_ENV - - name: Upload test Artifacts - uses: actions/upload-artifact@v7 - with: - name: ReferenceImages_CairoMakie_${{ matrix.version }} - path: ./CairoMakie/test/reference_images/ - - name: Fail after artifacts if tests failed - if: ${{ env.TESTS_SUCCESSFUL != 'true' }} - run: exit 1 - - uses: julia-actions/julia-processcoverage@v1 - with: - directories: Makie/src - - uses: codecov/codecov-action@v6 - with: - file: lcov.info diff --git a/.github/workflows/_compute-pipeline.yml b/.github/workflows/_compute-pipeline.yml index 93d2830665c..d923db80995 100644 --- a/.github/workflows/_compute-pipeline.yml +++ b/.github/workflows/_compute-pipeline.yml @@ -26,6 +26,8 @@ jobs: - uses: julia-actions/cache@v3 - name: Develop and test ComputePipeline shell: julia --project=monorepo {0} + env: + JULIA_PKG_PRECOMPILE_AUTO: '0' run: | using Pkg # dev mono repo versions diff --git a/.github/workflows/_glmakie.yml b/.github/workflows/_glmakie.yml deleted file mode 100644 index 6572dba7828..00000000000 --- a/.github/workflows/_glmakie.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: GLMakie -on: - workflow_call: - -jobs: - test: - name: GLMakie Julia ${{ matrix.version }} - env: - MODERNGL_DEBUGGING: "true" - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - version: - - '1.10' - - '1' - os: - - ubuntu-latest - arch: - - x64 - steps: - - name: Checkout - uses: actions/checkout@v6 - - uses: julia-actions/setup-julia@v2 - with: - version: ${{ matrix.version }} - arch: ${{ matrix.arch }} - - uses: julia-actions/cache@v3 - - run: sudo apt-get update && sudo apt-get install -y xorg-dev mesa-utils xvfb libgl1 freeglut3-dev libxrandr-dev libxinerama-dev libxcursor-dev libxi-dev libxext-dev xsettingsd x11-xserver-utils - - name: Install Julia dependencies - shell: julia --project=monorepo {0} - run: | - using Pkg; - # dev mono repo versions - pkg"registry up" - Pkg.update() - pkg"dev ./Makie ./GLMakie ./ReferenceTests ./ComputePipeline" - - name: Run the tests - id: referencetests - continue-on-error: true - run: > - DISPLAY=:0 xvfb-run -s '-screen 0 1024x768x24' julia --color=yes --project=monorepo -e 'using Pkg; Pkg.test("GLMakie", coverage=true)' - && echo "TESTS_SUCCESSFUL=true" >> $GITHUB_ENV - - name: Upload test Artifacts - uses: actions/upload-artifact@v7 - with: - name: ReferenceImages_GLMakie_${{ matrix.version }} - path: | - ./GLMakie/test/reference_images/ - - name: Fail after artifacts if tests failed - if: ${{ env.TESTS_SUCCESSFUL != 'true' }} - run: exit 1 - - uses: julia-actions/julia-processcoverage@v1 - with: - directories: Makie/src - - uses: codecov/codecov-action@v6 - with: - file: lcov.info diff --git a/.github/workflows/_makie.yml b/.github/workflows/_makie.yml index 1a488243aaa..df9d7a8e20c 100644 --- a/.github/workflows/_makie.yml +++ b/.github/workflows/_makie.yml @@ -26,6 +26,8 @@ jobs: - uses: julia-actions/cache@v3 - name: Develop and test Makie shell: julia --project=monorepo {0} + env: + JULIA_PKG_PRECOMPILE_AUTO: '0' run: | using Pkg # dev mono repo versions diff --git a/.github/workflows/_reference-tests.yml b/.github/workflows/_reference-tests.yml index 6076b788d0b..80ca90ee547 100644 --- a/.github/workflows/_reference-tests.yml +++ b/.github/workflows/_reference-tests.yml @@ -4,13 +4,36 @@ on: jobs: cairomakie: - uses: ./.github/workflows/_cairomakie.yml + strategy: + matrix: + julia-version: ['1.10', '1'] + uses: ./.github/workflows/_backend-tests.yml + with: + julia-version: ${{ matrix.julia-version }} + backend: CairoMakie glmakie: - uses: ./.github/workflows/_glmakie.yml + strategy: + matrix: + julia-version: ['1.10', '1'] + uses: ./.github/workflows/_backend-tests.yml + with: + julia-version: ${{ matrix.julia-version }} + backend: GLMakie + system-packages: xorg-dev mesa-utils xvfb libgl1 freeglut3-dev libxrandr-dev libxinerama-dev libxcursor-dev libxi-dev libxext-dev xsettingsd x11-xserver-utils + test-prefix: 'DISPLAY=:0 xvfb-run -s "-screen 0 1024x768x24"' wglmakie: - uses: ./.github/workflows/_wglmakie.yml + strategy: + matrix: + julia-version: ['1.10', '1'] + uses: ./.github/workflows/_backend-tests.yml + with: + julia-version: ${{ matrix.julia-version }} + backend: WGLMakie + nbuckets: 4 + system-packages: xorg-dev libosmesa6 libgl1-mesa-dri mesa-utils xvfb libgl1 freeglut3-dev libxrandr-dev libxinerama-dev libxcursor-dev libxi-dev libxext-dev + test-prefix: 'DISPLAY=:0 xvfb-run -s "-screen 0 2048x2048x24"' consolidation: name: Merge artifacts @@ -18,42 +41,61 @@ jobs: if: always() needs: [cairomakie, glmakie, wglmakie] steps: - - uses: actions/download-artifact@v8 - with: - name: ReferenceImages_WGLMakie_1 - path: ./ReferenceImages/WGLMakie - - uses: actions/download-artifact@v8 + - name: Download all bucket artifacts for Julia 1 + uses: actions/download-artifact@v8 with: - name: ReferenceImages_CairoMakie_1 - path: ./ReferenceImages/CairoMakie - - uses: actions/download-artifact@v8 - with: - name: ReferenceImages_GLMakie_1 - path: ./ReferenceImages/GLMakie + pattern: ReferenceImages_*_1_* + path: ./BucketArtifacts/ - name: Consolidate reference image folders run: | - baseDir="./ReferenceImages" + baseDir="./BucketArtifacts" + outDir="./ReferenceImagesCombined" + mkdir -p "$outDir" + + # Copy reference images from the first available artifact (same across all buckets/backends) + first_artifact=$(ls -d ${baseDir}/ReferenceImages_*_1_*/ 2>/dev/null | head -1) + if [ -n "$first_artifact" ] && [ -d "${first_artifact}/reference" ]; then + cp -r "${first_artifact}/reference/." "$outDir/reference/" + fi + + # Initialize output files + > "$outDir/scores.tsv" + > "$outDir/new_files.txt" + > "$outDir/missing_files.txt" + + for backend in CairoMakie GLMakie WGLMakie; do + mkdir -p "$outDir/recorded/${backend}/" + + # Merge all buckets for this backend + for bucket_dir in ${baseDir}/ReferenceImages_${backend}_1_*/; do + [ -d "$bucket_dir" ] || continue + + # Copy recorded images (no overlap between buckets) + if [ -d "${bucket_dir}/recorded/${backend}/" ]; then + cp -r "${bucket_dir}/recorded/${backend}/." "$outDir/recorded/${backend}/" + fi - # Create new top-level directory for combined files - mkdir -p "./ReferenceImagesCombined" + # Concatenate scores and new_files from each bucket + [ -f "${bucket_dir}/scores.tsv" ] && cat "${bucket_dir}/scores.tsv" >> "$outDir/scores.tsv" + [ -f "${bucket_dir}/new_files.txt" ] && cat "${bucket_dir}/new_files.txt" >> "$outDir/new_files.txt" + done - # Copy the reference folder from GLMakie, it's the same for all backends - cp -r "${baseDir}/GLMakie/reference/." "./ReferenceImagesCombined/reference/" + # Compute missing_files: reference files that were neither recorded nor skipped + if [ -d "$outDir/reference/${backend}" ]; then + first_bucket=$(ls -d ${baseDir}/ReferenceImages_${backend}_1_*/ 2>/dev/null | head -1) - # Initialize empty files for concatenation - > "./ReferenceImagesCombined/scores.tsv" - > "./ReferenceImagesCombined/new_files.txt" + # Build sorted list of files that are accounted for (recorded + skipped) + { + [ -d "$outDir/recorded/${backend}" ] && (cd "$outDir/recorded" && find "${backend}" -type f) + [ -n "$first_bucket" ] && [ -f "${first_bucket}/skipped_names.txt" ] && cat "${first_bucket}/skipped_names.txt" + } | sort -u > /tmp/have_or_skip.txt - # Loop through the directories and concatenate the files, and copy recorded folders - for dir in WGLMakie CairoMakie GLMakie; do - # Concatenate scores.tsv, new_files.txt and missing_files.txt - cat "${baseDir}/${dir}/scores.tsv" >> "./ReferenceImagesCombined/scores.tsv" - cat "${baseDir}/${dir}/new_files.txt" >> "./ReferenceImagesCombined/new_files.txt" - cat "${baseDir}/${dir}/missing_files.txt" >> "./ReferenceImagesCombined/missing_files.txt" + # Reference files + (cd "$outDir/reference" && find "${backend}" -type f) | sort > /tmp/ref_files.txt - # Copy recorded folder - mkdir -p "./ReferenceImagesCombined/recorded/${dir}/" - cp -r "${baseDir}/${dir}/recorded/${dir}/." "./ReferenceImagesCombined/recorded/${dir}/" + # Missing = reference - (recorded ∪ skipped) + comm -23 /tmp/ref_files.txt /tmp/have_or_skip.txt >> "$outDir/missing_files.txt" + fi done echo "Files and folders have been successfully combined into ReferenceImagesCombined." diff --git a/.github/workflows/_rprmakie.yml b/.github/workflows/_rprmakie.yml index 2fd252a6720..7370eac3905 100644 --- a/.github/workflows/_rprmakie.yml +++ b/.github/workflows/_rprmakie.yml @@ -25,6 +25,8 @@ jobs: - uses: julia-actions/cache@v3 - name: Install Julia dependencies shell: julia --project=monorepo {0} + env: + JULIA_PKG_PRECOMPILE_AUTO: '0' run: | using Pkg; pkg"registry up" diff --git a/.github/workflows/_wglmakie.yml b/.github/workflows/_wglmakie.yml deleted file mode 100644 index bc037485a87..00000000000 --- a/.github/workflows/_wglmakie.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: WGLMakie -on: - workflow_call: - -jobs: - test: - name: WGLMakie Julia ${{ matrix.version }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - version: - - '1.10' - - '1' - os: - - ubuntu-latest - arch: - - x64 - steps: - - name: Checkout - uses: actions/checkout@v6 - - uses: julia-actions/setup-julia@v2 - with: - version: ${{ matrix.version }} - arch: ${{ matrix.arch }} - - uses: julia-actions/cache@v3 - - run: sudo apt-get update && sudo apt-get install -y xorg-dev libosmesa6 libgl1-mesa-dri mesa-utils xvfb libgl1 freeglut3-dev libxrandr-dev libxinerama-dev libxcursor-dev libxi-dev libxext-dev - - name: Install Julia dependencies - shell: julia --project=monorepo {0} - run: | - using Pkg; - # dev mono repo versions - pkg"registry up" - Pkg.update() - pkg"dev ./Makie ./WGLMakie ./ReferenceTests ./ComputePipeline" - - name: Run the tests - continue-on-error: true - run: > - DISPLAY=:0 xvfb-run -s '-screen 0 2048x2048x24' julia --color=yes --project=monorepo -e 'using Pkg; Pkg.test("WGLMakie", coverage=true)' - && echo "TESTS_SUCCESSFUL=true" >> $GITHUB_ENV - - name: Upload test Artifacts - uses: actions/upload-artifact@v7 - with: - name: ReferenceImages_WGLMakie_${{ matrix.version }} - path: ./WGLMakie/test/reference_images/ - - name: Upload test Electron logs - uses: actions/upload-artifact@v7 - with: - name: Electron_Logs_WGLMakie_${{ matrix.version }} - path: ./WGLMakie/test/electron.log - - name: Fail after artifacts if tests failed - if: ${{ env.TESTS_SUCCESSFUL != 'true' }} - run: exit 1 - - uses: julia-actions/julia-processcoverage@v1 - with: - directories: Makie/src - - uses: codecov/codecov-action@v6 - with: - file: lcov.info diff --git a/.github/workflows/compilation-benchmark.yaml b/.github/workflows/compilation-benchmark.yaml index aa0dde4e33c..86b2375cb9b 100644 --- a/.github/workflows/compilation-benchmark.yaml +++ b/.github/workflows/compilation-benchmark.yaml @@ -1,19 +1,21 @@ name: Benchmark on: - pull_request: - paths-ignore: - - 'docs/**' - - '*.md' - branches: - - master + issue_comment: + types: [created] concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.event.issue.number }} cancel-in-progress: true jobs: benchmark: name: ${{ matrix.package }} + if: > + github.event.issue.pull_request && + contains(github.event.comment.body, '/benchmark') && + (github.event.comment.author_association == 'MEMBER' || + github.event.comment.author_association == 'OWNER' || + github.event.comment.author_association == 'COLLABORATOR') runs-on: ubuntu-latest strategy: fail-fast: false @@ -23,8 +25,19 @@ jobs: - GLMakie - WGLMakie steps: + - name: Get PR head ref + id: pr + env: + GH_TOKEN: ${{ github.token }} + run: | + pr_data=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.issue.number }}) + echo "ref=$(echo $pr_data | jq -r .head.ref)" >> $GITHUB_OUTPUT + echo "sha=$(echo $pr_data | jq -r .head.sha)" >> $GITHUB_OUTPUT + echo "base_ref=$(echo $pr_data | jq -r .base.ref)" >> $GITHUB_OUTPUT - name: Checkout uses: actions/checkout@v6 + with: + ref: ${{ steps.pr.outputs.ref }} - name: Install xvfb run: sudo apt-get update && sudo apt-get install -y xorg-dev mesa-utils xvfb libgl1 freeglut3-dev libxrandr-dev libxinerama-dev libxcursor-dev libxi-dev libxext-dev - uses: julia-actions/setup-julia@v2 @@ -35,9 +48,9 @@ jobs: - name: Benchmark env: GITHUB_TOKEN: ${{ secrets.BENCHMARK_KEY }} - PR_NUMBER: ${{ github.event.number }} + PR_NUMBER: ${{ github.event.issue.number }} run: > - DISPLAY=:0 xvfb-run -s '-screen 0 1024x768x24' julia --project=./metrics/ttfp/ ./metrics/ttfp/run-benchmark.jl ${{ matrix.package }} 20 ${{ github.event.pull_request.base.ref }} + DISPLAY=:0 xvfb-run -s '-screen 0 1024x768x24' julia --project=./metrics/ttfp/ ./metrics/ttfp/run-benchmark.jl ${{ matrix.package }} 60 ${{ steps.pr.outputs.base_ref }} - name: Upload plots as artifact uses: actions/upload-artifact@v7 with: @@ -45,11 +58,19 @@ jobs: path: ./benchmark_results post-gist: name: Post Benchmark Gist - needs: benchmark # Wait for all benchmark jobs to complete + needs: benchmark runs-on: ubuntu-latest permissions: - statuses: write # Permission to post workflow status + statuses: write steps: + - name: Get PR head SHA + id: pr + env: + GH_TOKEN: ${{ github.token }} + run: | + sha=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.issue.number }} --jq .head.sha) + echo "sha=$sha" >> $GITHUB_OUTPUT + - name: Check token validity id: check_token env: @@ -57,7 +78,6 @@ jobs: run: | if [ -z "$BENCHMARK_KEY" ]; then echo "Benchmark key missing — skipping benchmark upload." - echo "NOTE: Benchmark key is not accessible for outside contributors" echo "has_token=false" >> $GITHUB_OUTPUT else echo "has_token=true" >> $GITHUB_OUTPUT @@ -93,7 +113,7 @@ jobs: --method POST \ -H "Accept: application/vnd.github+json" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - /repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }} \ + /repos/${{ github.repository }}/statuses/${{ steps.pr.outputs.sha }} \ -f "state=success" \ -f "context=Benchmark Results" \ -f "description=Plots are available under Details" \ @@ -104,11 +124,12 @@ jobs: uses: thollander/actions-comment-pull-request@v3 with: github-token: ${{ secrets.BENCHMARK_KEY }} - comment-tag: benchmark # this allows to update the same post with new data + pr-number: ${{ github.event.issue.number }} + comment-tag: benchmark message: | # Benchmark Results - SHA: [${{ github.event.pull_request.head.sha }}](https://github.com/${{ github.repository }}/commit/${{ github.event.pull_request.head.sha }}) + SHA: [${{ steps.pr.outputs.sha }}](https://github.com/${{ github.repository }}/commit/${{ steps.pr.outputs.sha }}) > [!WARNING] > These results are subject to substantial noise because GitHub's CI runs on shared machines that are not ideally suited for benchmarking. @@ -116,4 +137,3 @@ jobs: ![GLMakie](${{ env.GIST_URL_USERCONTENT }}/raw/GLMakie.svg) ![CairoMakie](${{ env.GIST_URL_USERCONTENT }}/raw/CairoMakie.svg) ![WGLMakie](${{ env.GIST_URL_USERCONTENT }}/raw/WGLMakie.svg) - diff --git a/CairoMakie/test/runtests.jl b/CairoMakie/test/runtests.jl index 7d2d2e7d2e4..bf38203cb11 100644 --- a/CairoMakie/test/runtests.jl +++ b/CairoMakie/test/runtests.jl @@ -1,5 +1,8 @@ ENV["ENABLE_COMPUTE_CHECKS"] = "true" +# Allow Pkg.test to be used purely for precompilation (setup job in CI) +get(ENV, "PRECOMPILE_ONLY", nothing) == "true" && exit(0) + using Test using CairoMakie using Makie.FileIO diff --git a/GLMakie/test/runtests.jl b/GLMakie/test/runtests.jl index 47dfbc3de7b..f47d1edc424 100644 --- a/GLMakie/test/runtests.jl +++ b/GLMakie/test/runtests.jl @@ -1,5 +1,8 @@ ENV["ENABLE_COMPUTE_CHECKS"] = "true" +# Allow Pkg.test to be used purely for precompilation (setup job in CI) +get(ENV, "PRECOMPILE_ONLY", nothing) == "true" && exit(0) + using Makie using GLMakie, Test using FileIO diff --git a/ReferenceTests/src/database.jl b/ReferenceTests/src/database.jl index 73372ca5baa..45ebd279492 100644 --- a/ReferenceTests/src/database.jl +++ b/ReferenceTests/src/database.jl @@ -21,8 +21,59 @@ const RECORDING_DIR = Base.RefValue{String}() const SKIP_TITLES = Set{String}() const SKIP_FUNCTIONS = Set{Symbol}() const COUNTER = Ref(0) +const TOTAL_COUNTER = Ref(0) # counts all @reference_test invocations, used for bucket assignment +const TOTAL_TESTS = Ref(0) # total number of tests across all files, set before running for chunked bucketing const SKIPPED_NAMES = Set{String}() # names skipped due to title exclusion or function exclusion +""" + count_reference_tests(paths) + +Count `@reference_test` occurrences by reading files as text, recursively +following `include("...")` directives relative to each file's directory. +""" +function count_reference_tests(paths) + n = 0 + for path in paths + n += _count_in_file(path) + end + return n +end + +function _count_in_file(path) + text = read(path, String) + n = count(r"@reference_test\b", text) + dir = dirname(path) + for m in eachmatch(r"include\(\"([^\"]+)\"\)", text) + child = joinpath(dir, m.captures[1]) + isfile(child) && (n += _count_in_file(child)) + end + return n +end + +""" + should_run_in_bucket(test_index::Int) + +Check whether a test with the given index should run in the current bucket. +Bucketing is controlled by the `REFTEST_BUCKET` and `REFTEST_NBUCKETS` environment variables. +If either is unset, all tests run (no bucketing). +""" +function should_run_in_bucket(test_index::Int) + nbuckets_str = get(ENV, "REFTEST_NBUCKETS", nothing) + bucket_str = get(ENV, "REFTEST_BUCKET", nothing) + (nbuckets_str === nothing || bucket_str === nothing) && return true + nbuckets = parse(Int, nbuckets_str) + bucket = parse(Int, bucket_str) + # Use contiguous chunks rather than interleaving so that each bucket + # touches fewer distinct code paths, reducing JIT/compilation overhead. + ntotal = TOTAL_TESTS[] + if ntotal <= 0 + # Total unknown, fall back to interleaved assignment + return mod1(test_index, nbuckets) == bucket + end + chunk_size = cld(ntotal, nbuckets) # ceil division + return (bucket - 1) * chunk_size < test_index <= min(bucket * chunk_size, ntotal) +end + """ @reference_test(name, code) @@ -34,35 +85,40 @@ macro reference_test(name, code) funcs = used_functions(code) skip = (title in SKIP_TITLES) || any(x -> x in funcs, SKIP_FUNCTIONS) return quote - @testset $(title) begin - if $skip - @test_broken false - mark_skipped!($title) - else - t1 = time() - if $title in $REGISTERED_TESTS - error("title must be unique. Duplicate title: $($title)") - end - println("running $(lpad(COUNTER[] += 1, 3)): $($title)") - Makie.set_theme!(; - size = (500, 500), - CairoMakie = (; px_per_unit = 1), - GLMakie = (; scalefactor = 1, px_per_unit = 1), - WGLMakie = (; scalefactor = 1, px_per_unit = 1) - ) - ReferenceTests.RNG.seed_rng!() - result = let - $(esc(code)) + ReferenceTests.TOTAL_COUNTER[] += 1 + if !ReferenceTests.should_run_in_bucket(ReferenceTests.TOTAL_COUNTER[]) + # Not in this bucket, skip entirely + else + @testset $(title) begin + if $skip + @test_broken false + mark_skipped!($title) + else + t1 = time() + if $title in $REGISTERED_TESTS + error("title must be unique. Duplicate title: $($title)") + end + println("running $(lpad(COUNTER[] += 1, 3)): $($title)") + Makie.set_theme!(; + size = (500, 500), + CairoMakie = (; px_per_unit = 1), + GLMakie = (; scalefactor = 1, px_per_unit = 1), + WGLMakie = (; scalefactor = 1, px_per_unit = 1) + ) + ReferenceTests.RNG.seed_rng!() + result = let + $(esc(code)) + end + @test save_result(joinpath(RECORDING_DIR[], $title), result) + push!($REGISTERED_TESTS, $title) + elapsed = round(time() - t1; digits = 5) + total = Sys.total_memory() + mem = round((total - Sys.free_memory()) / 10^9; digits = 3) + # TODO, write to file and create an overview in the end, similar to the benchmark results! + println("Used $(mem)gb of $(round(total / 10^9; digits = 3))gb RAM, time: $(elapsed)s") end - @test save_result(joinpath(RECORDING_DIR[], $title), result) - push!($REGISTERED_TESTS, $title) - elapsed = round(time() - t1; digits = 5) - total = Sys.total_memory() - mem = round((total - Sys.free_memory()) / 10^9; digits = 3) - # TODO, write to file and create an overview in the end, similar to the benchmark results! - println("Used $(mem)gb of $(round(total / 10^9; digits = 3))gb RAM, time: $(elapsed)s") + GC.gc(true) # Run GC, to catch accumulating memory early on (used RAM) end - GC.gc(true) # Run GC, to catch accumulating memory early on (used RAM) end end end @@ -115,6 +171,20 @@ macro include_reference_tests(backend::Symbol, path, paths...) # prefix the recordings with the backend name so that each backend has its own versions ReferenceTests.RECORDING_DIR[] = joinpath(recording_dir, "recorded", $(string(backend))) mkpath(ReferenceTests.RECORDING_DIR[]) + + # Count total @reference_test occurrences for chunked bucket assignment + ReferenceTests.TOTAL_TESTS[] = ReferenceTests.count_reference_tests(include_paths) + + # Log bucket configuration + let + nb = get(ENV, "REFTEST_NBUCKETS", nothing) + b = get(ENV, "REFTEST_BUCKET", nothing) + if nb !== nothing && b !== nothing + ntotal = ReferenceTests.TOTAL_TESTS[] + @info "Running reference test bucket $(b)/$(nb) ($(ntotal) total tests)" + end + end + @testset "Reference tests $($(string(backend)))" begin empty!(ReferenceTests.REGISTERED_TESTS) for include_path in include_paths @@ -125,6 +195,7 @@ macro include_reference_tests(backend::Symbol, path, paths...) recording_dir = recording_dir empty!(ReferenceTests.REGISTERED_TESTS) ReferenceTests.COUNTER[] = 0 + ReferenceTests.TOTAL_COUNTER[] = 0 (recorded_files, recording_dir) end ) diff --git a/ReferenceTests/src/runtests.jl b/ReferenceTests/src/runtests.jl index e00effcdb96..16a88636c15 100644 --- a/ReferenceTests/src/runtests.jl +++ b/ReferenceTests/src/runtests.jl @@ -37,6 +37,12 @@ function record_comparison(base_folder::String, backend::String; record_folder_n end end + open(joinpath(base_folder, "skipped_names.txt"), "w") do file + for name in sort!(collect(SKIPPED_NAMES)) + println(file, joinpath(backend, "$name.png")) + end + end + return missing_refimages, scores end diff --git a/ReferenceTests/src/tests/updating.jl b/ReferenceTests/src/tests/updating.jl index a8089c411bf..17ff041c6e2 100644 --- a/ReferenceTests/src/tests/updating.jl +++ b/ReferenceTests/src/tests/updating.jl @@ -108,7 +108,11 @@ end @reference_test "deletion" begin f = Figure() l = Legend(f[1, 1], [LineElement(color = :red)], ["Line"]) - s = display(f) + # Use colorbuffer to initialize the screen with a properly established session. + # The generic `display(f)` inline path doesn't reliably wait for the WGLMakie session. + Makie.colorbuffer(f) + s = Makie.getscreen(f.scene) + @test !isnothing(s) @test f.scene.current_screens[1] === s @test f.scene.children[1].current_screens[1] === s @test f.scene.children[1].children[1].current_screens[1] === s @@ -123,7 +127,9 @@ end @reference_test "deletion and observable args" begin obs = Observable(1:5) f, ax, pl = scatter(obs; markersize = 150) - s = display(f) + # Use colorbuffer to initialize the screen with a properly established session. + # The generic `display(f)` inline path doesn't reliably wait for the WGLMakie session. + Makie.colorbuffer(f) # So, for GLMakie it will be 2, since we register an additional listener for # State changes for the on demand renderloop @test length(obs.listeners) in (1, 2) diff --git a/WGLMakie/test/runtests.jl b/WGLMakie/test/runtests.jl index 5f468350376..d24b3e94cc4 100644 --- a/WGLMakie/test/runtests.jl +++ b/WGLMakie/test/runtests.jl @@ -1,4 +1,8 @@ ENV["ENABLE_COMPUTE_CHECKS"] = "true" + +# Allow Pkg.test to be used purely for precompilation (setup job in CI) +get(ENV, "PRECOMPILE_ONLY", nothing) == "true" && exit(0) + ENV["ELECTRON_LOG_FILE"] = joinpath(@__DIR__, "electron.log") ENV["ELECTRON_ENABLE_LOGGING"] = "true"