From acf60d71d756e08e2f406ca603fe1b5b030f46da Mon Sep 17 00:00:00 2001 From: wallstop Date: Tue, 1 Sep 2026 06:13:12 +0000 Subject: [PATCH 1/2] Test all shipping stripping levels --- ...ipping-fidelity-il2cpp-low-profile.v1.json | 30 +++++++ ...ing-fidelity-il2cpp-medium-profile.v1.json | 30 +++++++ ...ng-fidelity-il2cpp-minimal-profile.v1.json | 30 +++++++ .github/workflows/unity-tests.yml | 46 +++++----- .../il2cpp-build-configuration/SKILL.md | 15 ++-- ...erf-config-il2cpp-release-netstandard21.md | 26 +++--- .llm/skills/unity-editor-ci/SKILL.md | 8 +- .../references/unity-ci-matrix.md | 13 +-- docs/runbooks/perf-benchmark-methodology.md | 14 ++-- .../unity/__tests__/il2cpp-profile.test.ps1 | 5 +- .../__tests__/shipping-fidelity.test.ps1 | 84 ++++++++++++++++++- scripts/unity/run-ci-tests.ps1 | 35 +++++--- scripts/unity/validate-il2cpp-profile.ps1 | 12 +++ scripts/validate-unity-pr-policy.py | 11 ++- 14 files changed, 291 insertions(+), 68 deletions(-) create mode 100644 .github/perf/shipping-fidelity-il2cpp-low-profile.v1.json create mode 100644 .github/perf/shipping-fidelity-il2cpp-medium-profile.v1.json create mode 100644 .github/perf/shipping-fidelity-il2cpp-minimal-profile.v1.json diff --git a/.github/perf/shipping-fidelity-il2cpp-low-profile.v1.json b/.github/perf/shipping-fidelity-il2cpp-low-profile.v1.json new file mode 100644 index 00000000..71f26fe6 --- /dev/null +++ b/.github/perf/shipping-fidelity-il2cpp-low-profile.v1.json @@ -0,0 +1,30 @@ +{ + "schemaVersion": 1, + "profileId": "shipping-fidelity-il2cpp-low-player-v1", + "configuration": { + "buildTarget": "StandaloneWindows64", + "scriptingBackend": "IL2CPP", + "apiCompatibilityLevel": "NET_Standard_2_0", + "codeOptimization": "Release", + "il2cppCompilerConfiguration": "Release", + "il2cppCodeGeneration": "OptimizeSpeed", + "managedStrippingLevel": "Low", + "incrementalGc": true, + "stripEngineCode": true + }, + "buildOptions": { + "developmentBuild": false, + "allowDebugging": false, + "deepProfiling": false, + "enableAssertions": false, + "includeTestAssemblies": false, + "autoRunPlayer": false, + "connectToHost": false, + "connectWithProfiler": false, + "cleanBuildCache": true, + "detailedBuildReport": true + }, + "runtime": { + "debugBuild": false + } +} diff --git a/.github/perf/shipping-fidelity-il2cpp-medium-profile.v1.json b/.github/perf/shipping-fidelity-il2cpp-medium-profile.v1.json new file mode 100644 index 00000000..6659244e --- /dev/null +++ b/.github/perf/shipping-fidelity-il2cpp-medium-profile.v1.json @@ -0,0 +1,30 @@ +{ + "schemaVersion": 1, + "profileId": "shipping-fidelity-il2cpp-medium-player-v1", + "configuration": { + "buildTarget": "StandaloneWindows64", + "scriptingBackend": "IL2CPP", + "apiCompatibilityLevel": "NET_Standard_2_0", + "codeOptimization": "Release", + "il2cppCompilerConfiguration": "Release", + "il2cppCodeGeneration": "OptimizeSpeed", + "managedStrippingLevel": "Medium", + "incrementalGc": true, + "stripEngineCode": true + }, + "buildOptions": { + "developmentBuild": false, + "allowDebugging": false, + "deepProfiling": false, + "enableAssertions": false, + "includeTestAssemblies": false, + "autoRunPlayer": false, + "connectToHost": false, + "connectWithProfiler": false, + "cleanBuildCache": true, + "detailedBuildReport": true + }, + "runtime": { + "debugBuild": false + } +} diff --git a/.github/perf/shipping-fidelity-il2cpp-minimal-profile.v1.json b/.github/perf/shipping-fidelity-il2cpp-minimal-profile.v1.json new file mode 100644 index 00000000..449cd33e --- /dev/null +++ b/.github/perf/shipping-fidelity-il2cpp-minimal-profile.v1.json @@ -0,0 +1,30 @@ +{ + "schemaVersion": 1, + "profileId": "shipping-fidelity-il2cpp-minimal-player-v1", + "configuration": { + "buildTarget": "StandaloneWindows64", + "scriptingBackend": "IL2CPP", + "apiCompatibilityLevel": "NET_Standard_2_0", + "codeOptimization": "Release", + "il2cppCompilerConfiguration": "Release", + "il2cppCodeGeneration": "OptimizeSpeed", + "managedStrippingLevel": "Minimal", + "incrementalGc": true, + "stripEngineCode": true + }, + "buildOptions": { + "developmentBuild": false, + "allowDebugging": false, + "deepProfiling": false, + "enableAssertions": false, + "includeTestAssemblies": false, + "autoRunPlayer": false, + "connectToHost": false, + "connectWithProfiler": false, + "cleanBuildCache": true, + "detailedBuildReport": true + }, + "runtime": { + "debugBuild": false + } +} diff --git a/.github/workflows/unity-tests.yml b/.github/workflows/unity-tests.yml index 22ec132a..fd6b386d 100644 --- a/.github/workflows/unity-tests.yml +++ b/.github/workflows/unity-tests.yml @@ -469,11 +469,11 @@ jobs: -ReleaseCodeOptimization ` -ReleasePlayerBuild - # Run the shipping-fidelity slice on the oldest and newest supported - # editors. Keep it last because it builds a second IL2CPP player, and the - # central cleanup action below returns the final activation before the - # editor-scoped organization lock is released. - - name: Run stripped shipping-fidelity player + # Run all shipping-fidelity stripping levels on the oldest and newest + # supported editors. Keep the group last because each cell builds another + # IL2CPP player, and the central cleanup action below returns the final + # activation before the editor-scoped organization lock is released. + - name: Run stripped shipping-fidelity players id: run_shipping if: >- ${{ @@ -490,22 +490,30 @@ jobs: UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_ACCELERATOR_ENDPOINT: ${{ secrets.UNITY_ACCELERATOR_ENDPOINT }} run: | - $projectPath = Join-Path $env:RUNNER_WORKSPACE 'dxm-u\t\${{ matrix.unity-version }}-shipping' $cachePath = Join-Path $env:RUNNER_WORKSPACE 'dxm-c\${{ matrix.unity-version }}' - ./scripts/unity/run-ci-tests.ps1 ` - -UnityVersion '${{ matrix.unity-version }}' ` - -UnityInstallRoot (Join-Path $env:RUNNER_TOOL_CACHE 'u6-v3') ` - -TestMode shipping ` - -AssemblyNames '' ` - -ArtifactsPath '.artifacts/unity/${{ matrix.unity-version }}-shipping' ` - -ProjectPath $projectPath ` - -CachePath $cachePath ` - -CanonicalProfilePath '.github/perf/shipping-fidelity-il2cpp-profile.v1.json' ` - -LicenseReturnOwner Central ` - -ReleaseCodeOptimization ` - -ReleasePlayerBuild + $shippingProfiles = @( + @{ Level = 'minimal'; Path = '.github/perf/shipping-fidelity-il2cpp-minimal-profile.v1.json' }, + @{ Level = 'low'; Path = '.github/perf/shipping-fidelity-il2cpp-low-profile.v1.json' }, + @{ Level = 'medium'; Path = '.github/perf/shipping-fidelity-il2cpp-medium-profile.v1.json' }, + @{ Level = 'high'; Path = '.github/perf/shipping-fidelity-il2cpp-profile.v1.json' } + ) + foreach ($shippingProfile in $shippingProfiles) { + $projectPath = Join-Path $env:RUNNER_WORKSPACE "dxm-u\t\${{ matrix.unity-version }}-shipping-$($shippingProfile.Level)" + ./scripts/unity/run-ci-tests.ps1 ` + -UnityVersion '${{ matrix.unity-version }}' ` + -UnityInstallRoot (Join-Path $env:RUNNER_TOOL_CACHE 'u6-v3') ` + -TestMode shipping ` + -AssemblyNames '' ` + -ArtifactsPath ".artifacts/unity/${{ matrix.unity-version }}-shipping/$($shippingProfile.Level)" ` + -ProjectPath $projectPath ` + -CachePath $cachePath ` + -CanonicalProfilePath $shippingProfile.Path ` + -LicenseReturnOwner Central ` + -ReleaseCodeOptimization ` + -ReleasePlayerBuild + } - # A mode failure is tolerated until the test modes and shipping slice have + # A mode failure is tolerated until the test modes and shipping group have # had a chance to run. Preserve diagnostics and result verification, then fail # the editor-scoped job from one summary gate before central cleanup. - name: Dump EditMode log tail on failure or cancellation diff --git a/.llm/skills/il2cpp-build-configuration/SKILL.md b/.llm/skills/il2cpp-build-configuration/SKILL.md index 60cfa855..9ce8fb43 100644 --- a/.llm/skills/il2cpp-build-configuration/SKILL.md +++ b/.llm/skills/il2cpp-build-configuration/SKILL.md @@ -65,9 +65,11 @@ fidelity leg additionally takes `-TestMode shipping` and the shipping profile. is mandatory for test players. Default stripping deletes the benchmark assemblies and the `[Preserve]` standalone test-run callback, and the player then runs nothing. -- `.github/perf/shipping-fidelity-il2cpp-profile.v1.json` pins High stripping - and `includeTestAssemblies=false` for the separate shipping consumer. That - player builds through `BuildPipeline` and must not use Unity Test Framework. +- The four shipping-fidelity profiles under `.github/perf/` pin Minimal, Low, + Medium, and High stripping with `includeTestAssemblies=false`. The existing + `shipping-fidelity-il2cpp-profile.v1.json` is the High profile. Each separate + shipping consumer builds through `BuildPipeline` and must not use Unity Test + Framework. - Pin `Il2CppCodeGeneration.OptimizeSpeed`, incremental GC, and engine-code stripping from the canonical profile. Do not rely on an ephemeral project's defaults. @@ -93,9 +95,10 @@ fidelity leg additionally takes `-TestMode shipping` and the shipping profile. - Never publish a Debug number. `perf-numbers.yml` publishes only the Standalone IL2CPP Release leg; manually dispatched `unity-benchmarks.yml` supplies per-version editor coverage and allocation evidence. -- The Unity Tests workflow runs the shipping-fidelity player on the oldest and - newest supported editors. Its positive and missing-root runs use the same - binary and publish correctness evidence only, never throughput rows. +- The Unity Tests workflow builds every shipping-fidelity stripping level on + the oldest and newest supported editors. Each level's positive and + missing-root runs use the same binary and publish correctness evidence only, + never throughput rows. ### Backend split diff --git a/.llm/skills/il2cpp-build-configuration/references/perf-config-il2cpp-release-netstandard21.md b/.llm/skills/il2cpp-build-configuration/references/perf-config-il2cpp-release-netstandard21.md index 6140d969..a2b0f956 100644 --- a/.llm/skills/il2cpp-build-configuration/references/perf-config-il2cpp-release-netstandard21.md +++ b/.llm/skills/il2cpp-build-configuration/references/perf-config-il2cpp-release-netstandard21.md @@ -95,20 +95,23 @@ execute and write results. The runner's `StandaloneScriptingBackend` parameter defaults to `IL2CPP` and accepts `Mono2x`, so the same script can build either backend; the published leg pins IL2CPP. -The separate shipping-fidelity player uses High managed stripping and excludes -test assemblies. It builds through `BuildPipeline.BuildPlayer` instead of Unity -Test Framework, then runs positive generated-root coverage and an expected -missing-root control from the same binary. +The separate shipping-fidelity cells use Minimal, Low, Medium, and High managed +stripping and exclude test assemblies. Each cell builds through +`BuildPipeline.BuildPlayer` instead of Unity Test Framework, then runs positive +generated-root coverage and an expected missing-root control from the same +binary. ## Canonical profile `.github/perf/canonical-il2cpp-profile.v1.json` is the reviewed source of truth for the published standalone IL2CPP leg. -`.github/perf/shipping-fidelity-il2cpp-profile.v1.json` is the reviewed source -of truth for the stripped correctness player. Each workflow passes its selected -file through `-CanonicalProfilePath`. The runner copies the exact bytes under -the source filename and writes a SHA-256 file to the run artifacts, then embeds -the profile ID and hash in generated editor and player code. +The four shipping-fidelity profiles under `.github/perf/` are the reviewed +sources of truth for the stripped correctness players. The existing +`.github/perf/shipping-fidelity-il2cpp-profile.v1.json` remains the High profile. +Each workflow passes its selected file through `-CanonicalProfilePath`. The +runner copies the exact bytes under the source filename and writes a SHA-256 +file to the run artifacts, then embeds the profile ID and hash in generated +editor and player code. The profile also pins `Il2CppCodeGeneration.OptimizeSpeed`, incremental GC, engine-code stripping, and the final `BuildOptions` flags. Keep these values in @@ -158,8 +161,9 @@ manually dispatched `unity-benchmarks.yml` runs the EditMode and PlayMode benchmark tests across Unity versions with `-ReleaseCodeOptimization` for coverage only. -The Unity Tests workflow runs the shipping-fidelity profile on the oldest and -newest supported editors. Its evidence is correctness-only and never enters the +The Unity Tests workflow runs all four shipping-fidelity profiles on the oldest +and newest supported editors. Each profile uses an isolated project and +artifact directory. Its evidence is correctness-only and never enters the published benchmark baseline. ## Common Pitfalls diff --git a/.llm/skills/unity-editor-ci/SKILL.md b/.llm/skills/unity-editor-ci/SKILL.md index 8f9b3c2c..454b4bdf 100644 --- a/.llm/skills/unity-editor-ci/SKILL.md +++ b/.llm/skills/unity-editor-ci/SKILL.md @@ -12,8 +12,8 @@ The active Unity workflows run `scripts/unity/run-ci-tests.ps1` directly on self Windows runners. `unity-tests.yml` is a four-version matrix. Each editor-scoped job runs `editmode`, `playmode`, and `standalone` as separate invocations under one lock and cleanup window. The oldest and current endpoint jobs also run the -stripped shipping-fidelity player. Both player modes build `StandaloneWindows64` -IL2CPP players from runner-local projects under +Minimal, Low, Medium, and High shipping-fidelity cells. All player modes build +`StandaloneWindows64` IL2CPP players from runner-local projects under `$RUNNER_WORKSPACE/dxm-u/t/-/`. ## When to use @@ -54,7 +54,7 @@ IL2CPP players from runner-local projects under - Timeout invariant: every step before and including the cleanup gate has an explicit positive timeout. Editor validation is capped at `10`, and the acquire step cap (`305`) exceeds its internal wait (`300`). Grouped correctness invocations use `90`/`90`/`150` caps, and the - endpoint-only shipping invocation uses `150`. Cleanup uses `5`/`2`/`5`/`2` for + endpoint-only shipping group uses `150`. Cleanup uses `5`/`2`/`5`/`2` for return/classify/release/gate. `unity-tests.yml` uses a `1050`-minute job cap. Other licensed jobs retain their `900`-minute cap. Each cap must retain at least 60 minutes beyond the sum of its enforced step caps. @@ -77,7 +77,7 @@ IL2CPP players from runner-local projects under - CI must pass `-CiManagedOnly -RequireHealthyExisting` plus `-ProvisioningProfile` explicitly. The grouped correctness job validates `StandaloneWindowsIl2Cpp` once because - that superset serves all test modes and the endpoint-only shipping invocation. Other jobs use + that superset serves all test modes and the endpoint-only shipping cells. Other jobs use `EditorOnly` for editmode, playmode, benchmarks, and release checks, or `StandaloneWindowsIl2Cpp` for standalone (verifies `windows-il2cpp`); `Android` and `Full` remain manual-maintenance profiles. diff --git a/.llm/skills/unity-editor-ci/references/unity-ci-matrix.md b/.llm/skills/unity-editor-ci/references/unity-ci-matrix.md index 40a46bcc..35f44941 100644 --- a/.llm/skills/unity-editor-ci/references/unity-ci-matrix.md +++ b/.llm/skills/unity-editor-ci/references/unity-ci-matrix.md @@ -26,9 +26,10 @@ Four jobs each run three independent test modes. `editmode`/`playmode` run in-editor on Mono; `standalone` builds and runs a `StandaloneWindows64` IL2CPP test player. -The oldest and current endpoint jobs also build and run the stripped -`StandaloneWindows64` IL2CPP shipping-fidelity player. Each invocation keeps its -own result verification, artifact, timeout, and runner-local project. The direct runner +The oldest and current endpoint jobs also build and run Minimal, Low, Medium, +and High `StandaloneWindows64` IL2CPP shipping-fidelity players. Each level +keeps its own result evidence and runner-local project inside the grouped +shipping step and artifact. The direct runner generates a package host project under `$RUNNER_WORKSPACE/dxm-u/t/-/`, imports the repo package with a `file:` dependency, and configures IL2CPP before either player build. Test hosts @@ -70,13 +71,13 @@ With both controls, a run consumes at most one seat while another repository can job timeout-minutes >= sum(all capped steps through cleanup gate) + 60 ``` -Editor validation is capped at 10 minutes. The acquire input `timeout-minutes: "300"` is the internal lock-poll budget. Its enclosing step has `timeout-minutes: 305`, so the action can finish and report a timeout before GitHub terminates the step. Grouped correctness test modes use 90/90/150-minute caps, and the endpoint-only shipping invocation uses 150 minutes. Return/classify/release/gate use 5/2/5/2 minutes. `unity-tests.yml` uses `timeout-minutes: 1050`; other licensed jobs retain `timeout-minutes: 900`. `scripts/validate-unity-pr-policy.py` sums every enforced step cap through the cleanup gate and requires at least 60 minutes of remaining job time. +Editor validation is capped at 10 minutes. The acquire input `timeout-minutes: "300"` is the internal lock-poll budget. Its enclosing step has `timeout-minutes: 305`, so the action can finish and report a timeout before GitHub terminates the step. Grouped correctness test modes use 90/90/150-minute caps, and the endpoint-only shipping group uses 150 minutes. Return/classify/release/gate use 5/2/5/2 minutes. `unity-tests.yml` uses `timeout-minutes: 1050`; other licensed jobs retain `timeout-minutes: 900`. `scripts/validate-unity-pr-policy.py` sums every enforced step cap through the cleanup gate and requires at least 60 minutes of remaining job time. The step-level caps protect the in-use seat from a hung editor or ancillary action. They must remain strictly below the job timeout so the step fails first and the cleanup chain still runs. This matters because `stuck-job-watchdog.yml` ignores any `in_progress` job; without step caps, a wedged action can squat the seat until the whole job is cancelled. Runner administrators manually install every exact editor and required module under `RUNNER_TOOL_CACHE/u6-v3`. Workflows validate that root with `-CiManagedOnly -RequireHealthyExisting` before acquiring the organization lock; they never install or repair editors. Release the lock only after the always-run return and cleanup-classification steps. -**Operator note (standalone IL2CPP):** the grouped correctness jobs require the Windows IL2CPP Unity module and the host build toolchain needed by Unity for Windows players. They validate the `StandaloneWindowsIl2Cpp` profile once before the lock, then reuse that editor for all test modes and the endpoint-only shipping invocation. Other workflows still select `EditorOnly` or `StandaloneWindowsIl2Cpp` according to their one mode. See [unity-editor-cli-bootstrap](./unity-editor-cli-bootstrap.md) for manual maintenance details. +**Operator note (standalone IL2CPP):** the grouped correctness jobs require the Windows IL2CPP Unity module and the host build toolchain needed by Unity for Windows players. They validate the `StandaloneWindowsIl2Cpp` profile once before the lock, then reuse that editor for all test modes and the endpoint-only shipping cells. Other workflows still select `EditorOnly` or `StandaloneWindowsIl2Cpp` according to their one mode. See [unity-editor-cli-bootstrap](./unity-editor-cli-bootstrap.md) for manual maintenance details. `unity-benchmarks.yml` (active; manual-only, NEVER on PRs): @@ -89,7 +90,7 @@ The active `unity-benchmarks.yml` explicitly omits `pull_request` and `push` per ## compute-unity-assemblies is-empty Gate -Every workflow that consumes `./.github/actions/compute-unity-assemblies` gives each Unity test invocation a stable compute id. Grouped correctness uses `compute`, `compute_playmode`, and `compute_standalone`; each test run receives only its matching assembly output through an environment variable. Test work skips when its selection is empty. The endpoint-only shipping invocation is not test-assembly-discovery gated. Lock acquisition remains unconditional because the static matrix is structurally non-empty and the organization analyzer must prove each acquisition. When asmdef discovery resolves no owned assemblies, verification treats the empty selection as an intentional skip while the terminal return/classify/release/gate chain still proves cleanup. +Every workflow that consumes `./.github/actions/compute-unity-assemblies` gives each Unity test invocation a stable compute id. Grouped correctness uses `compute`, `compute_playmode`, and `compute_standalone`; each test run receives only its matching assembly output through an environment variable. Test work skips when its selection is empty. The endpoint-only shipping group is not test-assembly-discovery gated. Lock acquisition remains unconditional because the static matrix is structurally non-empty and the organization analyzer must prove each acquisition. When asmdef discovery resolves no owned assemblies, verification treats the empty selection as an intentional skip while the terminal return/classify/release/gate chain still proves cleanup. The `Verify tests actually ran` step keeps a cancellation-safe gate and must also require `steps.compute.outcome == 'success'` plus either `steps.compute.outputs.is-empty == 'true'` or a non-skipped Unity run step (never an is-empty gate alone). It receives `expected-empty: ${{ steps.compute.outputs.is-empty }}`, so an intentional skip reads as success rather than a "tests did not run" failure, while checkout/cache/setup/editor-validation/lock failures that prevent Unity from launching are not obscured by a generic missing-results annotation. The skip path does not fire for the current asmdef set; it is the robustness path for a target whose assemblies are all filtered out, such as a runtime-only standalone run when every DxMessaging test asmdef is editor-only. diff --git a/docs/runbooks/perf-benchmark-methodology.md b/docs/runbooks/perf-benchmark-methodology.md index 598e241a..77081110 100644 --- a/docs/runbooks/perf-benchmark-methodology.md +++ b/docs/runbooks/perf-benchmark-methodology.md @@ -285,12 +285,14 @@ metrics because the Release player strips the required profiler recorder (see renderer omits the all-unmeasured memory columns from the Standalone table and the all-unmeasured memory matrices entirely. -- **Shipping-fidelity leg (not published)** builds a separate IL2CPP Release - player with High managed stripping and no test assemblies. It uses - `.github/perf/shipping-fidelity-il2cpp-profile.v1.json` and calls - `BuildPipeline.BuildPlayer` directly, without Unity Test Framework or a - PlayerConnection. The oldest and newest supported Unity editors each run the - same built binary twice: the positive run checks generated AOT roots across +- **Shipping-fidelity leg (not published)** builds separate IL2CPP Release + players with Minimal, Low, Medium, and High managed stripping and no test + assemblies. Each level has a reviewed profile under `.github/perf/`; the + existing `shipping-fidelity-il2cpp-profile.v1.json` remains the High profile. + The harness calls `BuildPipeline.BuildPlayer` directly, without Unity Test + Framework or a PlayerConnection. The oldest and newest supported Unity + editors build each level in an isolated project and artifact directory, then + run each built binary twice. The positive run checks generated AOT roots across every public top-level, public nested, and private nested class and readonly struct message shape across all three message kinds, while the missing-root run checks the expected failure for a private manual message. diff --git a/scripts/unity/__tests__/il2cpp-profile.test.ps1 b/scripts/unity/__tests__/il2cpp-profile.test.ps1 index 800be128..de013a2e 100644 --- a/scripts/unity/__tests__/il2cpp-profile.test.ps1 +++ b/scripts/unity/__tests__/il2cpp-profile.test.ps1 @@ -297,8 +297,11 @@ try { $badProfile = Copy-JsonValue -Value $profile $badProfile.profileId = 'unsupported-il2cpp-profile-v1' Write-TestJson -Path $badProfilePath -Value $badProfile - Assert-Fails 'unsupported profile ID lists both accepted profiles' -ExpectedMessage ( + Assert-Fails 'unsupported profile ID lists every accepted profile' -ExpectedMessage ( "Supported profileIds: 'canonical-il2cpp-verdict-player-v1', " + + "'shipping-fidelity-il2cpp-minimal-player-v1', " + + "'shipping-fidelity-il2cpp-low-player-v1', " + + "'shipping-fidelity-il2cpp-medium-player-v1', " + "'shipping-fidelity-il2cpp-player-v1'." ) { & $validatorPath -ProfilePath $badProfilePath -ProfileOnly diff --git a/scripts/unity/__tests__/shipping-fidelity.test.ps1 b/scripts/unity/__tests__/shipping-fidelity.test.ps1 index c7cb14a8..0e8c4b75 100644 --- a/scripts/unity/__tests__/shipping-fidelity.test.ps1 +++ b/scripts/unity/__tests__/shipping-fidelity.test.ps1 @@ -10,6 +10,28 @@ $unityScriptsRoot = Split-Path -Parent $PSScriptRoot $runnerPath = Join-Path $unityScriptsRoot 'run-ci-tests.ps1' $validatorPath = Join-Path $unityScriptsRoot 'validate-il2cpp-profile.ps1' $profilePath = Join-Path $repoRoot '.github/perf/shipping-fidelity-il2cpp-profile.v1.json' +$profileCases = @( + [ordered]@{ + FileName = 'shipping-fidelity-il2cpp-minimal-profile.v1.json' + ProfileId = 'shipping-fidelity-il2cpp-minimal-player-v1' + ManagedStrippingLevel = 'Minimal' + }, + [ordered]@{ + FileName = 'shipping-fidelity-il2cpp-low-profile.v1.json' + ProfileId = 'shipping-fidelity-il2cpp-low-player-v1' + ManagedStrippingLevel = 'Low' + }, + [ordered]@{ + FileName = 'shipping-fidelity-il2cpp-medium-profile.v1.json' + ProfileId = 'shipping-fidelity-il2cpp-medium-player-v1' + ManagedStrippingLevel = 'Medium' + }, + [ordered]@{ + FileName = 'shipping-fidelity-il2cpp-profile.v1.json' + ProfileId = 'shipping-fidelity-il2cpp-player-v1' + ManagedStrippingLevel = 'High' + } +) $fixtureRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("dxm-shipping-fidelity-{0}" -f [guid]::NewGuid().ToString('N')) function Assert-That { @@ -86,6 +108,65 @@ try { Invoke-Expression $definition.Extent.Text } + foreach ($profileCase in $profileCases) { + $caseProfilePath = Join-Path $repoRoot ".github/perf/$($profileCase.FileName)" + Assert-That "$($profileCase.ManagedStrippingLevel) shipping profile exists" ( + Test-Path -LiteralPath $caseProfilePath -PathType Leaf + ) + $caseProfile = Get-Content -LiteralPath $caseProfilePath -Raw | ConvertFrom-Json + Assert-That "$($profileCase.ManagedStrippingLevel) shipping profile has an exact identity" ( + $caseProfile.profileId -ceq $profileCase.ProfileId -and + $caseProfile.configuration.managedStrippingLevel -ceq $profileCase.ManagedStrippingLevel -and + -not [bool]$caseProfile.buildOptions.includeTestAssemblies + ) + & $validatorPath -ProfilePath $caseProfilePath -ProfileOnly + + $mismatchedProfile = Copy-JsonValue -Value $caseProfile + $mismatchedProfile.configuration.managedStrippingLevel = if ( + $profileCase.ManagedStrippingLevel -ceq 'High' + ) { + 'Medium' + } else { + 'High' + } + $mismatchedProfilePath = Join-Path $fixtureRoot ( + "mismatched-$($profileCase.ManagedStrippingLevel.ToLowerInvariant())-profile.json" + ) + [System.IO.File]::WriteAllText( + $mismatchedProfilePath, + ($mismatchedProfile | ConvertTo-Json -Depth 10) + ) + Assert-Fails "$($profileCase.ManagedStrippingLevel) profile rejects a different level" { + & $validatorPath -ProfilePath $mismatchedProfilePath -ProfileOnly + } 'differs' + + $caseName = $profileCase.ManagedStrippingLevel.ToLowerInvariant() + $caseProjectPath = Join-Path $fixtureRoot "profile-$caseName-project" + & $runnerPath ` + -UnityVersion '6000.3.16f1' ` + -TestMode shipping ` + -AssemblyNames '' ` + -ArtifactsPath (Join-Path $fixtureRoot "profile-$caseName-artifacts") ` + -RepoRoot $repoRoot ` + -ProjectPath $caseProjectPath ` + -CachePath (Join-Path $fixtureRoot "profile-$caseName-cache") ` + -CanonicalProfilePath $caseProfilePath ` + -GenerateOnly + $caseConfiguratorText = Get-Content -LiteralPath ( + Join-Path $caseProjectPath 'Assets/Editor/DxmCiTestConfigurator.cs' + ) -Raw + $caseBuilderText = Get-Content -LiteralPath ( + Join-Path $caseProjectPath 'Assets/Editor/DxmShippingFidelityBuilder.cs' + ) -Raw + $expectedLevelCall = "ManagedStrippingLevel.$($profileCase.ManagedStrippingLevel));" + Assert-That "$($profileCase.ManagedStrippingLevel) profile reaches generated configuration" ( + $caseConfiguratorText.Contains($expectedLevelCall) -and + $caseBuilderText.Contains($expectedLevelCall) -and + $caseBuilderText.Contains($profileCase.ProfileId) -and + $caseConfiguratorText.Contains('PlayerSettings.stripEngineCode = false;') + ) + } + $projectPath = Join-Path $fixtureRoot 'project' $artifactsPath = Join-Path $fixtureRoot 'artifacts' $cachePath = Join-Path $fixtureRoot 'cache' @@ -362,8 +443,7 @@ try { ) ) Assert-That 'shipping configuration defers engine stripping until the builder assembly has loaded' ( - $configuratorText.Contains('PlayerSettings.stripEngineCode = !string.Equals(') -and - $configuratorText.Contains('"shipping-fidelity-il2cpp-player-v1"') + $configuratorText.Contains('PlayerSettings.stripEngineCode = false;') ) Assert-That 'shipping builder invokes BuildPipeline directly' ( $builderText.Contains('BuildPipeline.BuildPlayer(options)') diff --git a/scripts/unity/run-ci-tests.ps1 b/scripts/unity/run-ci-tests.ps1 index 78f8c8f6..e22b3fc0 100644 --- a/scripts/unity/run-ci-tests.ps1 +++ b/scripts/unity/run-ci-tests.ps1 @@ -1423,6 +1423,12 @@ function New-ConfiguratorSource { [string]$CanonicalProfileSha256 = '' ) + $stripEngineCodeDuringConfigure = if ($ManagedStrippingLevel -ceq 'Disabled') { + 'true' + } else { + 'false' + } + # NOTE: this is a DOUBLE-quoted here-string so $Backend interpolates into the # generated C#. Every LITERAL C# dollar sign (the Debug.Log interpolated # string) is therefore backtick-escaped (`$). The LIVE code uses the @@ -1507,7 +1513,8 @@ public static class DxmCiTestConfigurator // member are intentionally NOT used. PlayerSettings.SetApiCompatibilityLevel(standalone, ApiCompatibilityLevel.NET_Standard); // Apply the reviewed profile's stripping level. Test players select - // Disabled so their callbacks survive; shipping fidelity selects High. + // Disabled so their callbacks survive; shipping-fidelity cells select a + // non-Disabled level. PlayerSettings.SetManagedStrippingLevel(standalone, ManagedStrippingLevel.$ManagedStrippingLevel); // Pin the IL2CPP C++ compiler configuration to Release explicitly. An // ephemeral CI project has no committed default for this setting, and @@ -1524,10 +1531,7 @@ public static class DxmCiTestConfigurator // editor-only package code when engine stripping is already enabled. // Keep it disabled while the shipping builder assembly loads; that // builder applies the reviewed value immediately before BuildPlayer. - PlayerSettings.stripEngineCode = !string.Equals( - "$CanonicalProfileId", - "shipping-fidelity-il2cpp-player-v1", - StringComparison.Ordinal); + PlayerSettings.stripEngineCode = $stripEngineCodeDuringConfigure; #if UNITY_2022_1_OR_NEWER PlayerSettings.SetIl2CppCodeGeneration(standalone, Il2CppCodeGeneration.OptimizeSpeed); #else @@ -2591,7 +2595,10 @@ public sealed partial class DxmShippingFidelityPlayer function New-ShippingFidelityBuilderSource { param( [Parameter(Mandatory = $true)][string]$CanonicalProfileId, - [Parameter(Mandatory = $true)][string]$CanonicalProfileSha256 + [Parameter(Mandatory = $true)][string]$CanonicalProfileSha256, + [Parameter(Mandatory = $true)] + [ValidateSet('Minimal', 'Low', 'Medium', 'High')] + [string]$ManagedStrippingLevel ) @" @@ -2688,7 +2695,7 @@ public static class DxmShippingFidelityBuilder UnityEditor.Compilation.CodeOptimization.Release; PlayerSettings.SetScriptingBackend(standalone, ScriptingImplementation.IL2CPP); PlayerSettings.SetApiCompatibilityLevel(standalone, ApiCompatibilityLevel.NET_Standard); - PlayerSettings.SetManagedStrippingLevel(standalone, ManagedStrippingLevel.High); + PlayerSettings.SetManagedStrippingLevel(standalone, ManagedStrippingLevel.$ManagedStrippingLevel); PlayerSettings.SetIl2CppCompilerConfiguration( standalone, Il2CppCompilerConfiguration.Release); @@ -3141,7 +3148,7 @@ EditorSettings: throw 'Shipping-fidelity generation requires a validated IL2CPP profile identity.' } $shippingFiles = @( - @{ Path = ([System.IO.Path]::Combine($project, 'Assets', 'Editor', 'DxmShippingFidelityBuilder.cs')); Content = (New-ShippingFidelityBuilderSource -CanonicalProfileId $CanonicalProfileId -CanonicalProfileSha256 $CanonicalProfileSha256) }, + @{ Path = ([System.IO.Path]::Combine($project, 'Assets', 'Editor', 'DxmShippingFidelityBuilder.cs')); Content = (New-ShippingFidelityBuilderSource -CanonicalProfileId $CanonicalProfileId -CanonicalProfileSha256 $CanonicalProfileSha256 -ManagedStrippingLevel $ManagedStrippingLevel) }, @{ Path = ([System.IO.Path]::Combine($project, 'Assets', 'DxmShippingFidelityPlayer.cs')); Content = (New-ShippingFidelityPlayerSource -CanonicalProfileId $CanonicalProfileId -CanonicalProfileSha256 $CanonicalProfileSha256) } ) foreach ($file in $shippingFiles) { @@ -5424,12 +5431,18 @@ if (-not [string]::IsNullOrWhiteSpace($CanonicalProfilePath)) { $managedStrippingLevel = [string]$canonicalProfile.configuration.managedStrippingLevel $includeTestAssemblies = [bool]$canonicalProfile.buildOptions.includeTestAssemblies if ($isShippingFidelity) { + $shippingProfileLevels = [ordered]@{ + 'shipping-fidelity-il2cpp-minimal-player-v1' = 'Minimal' + 'shipping-fidelity-il2cpp-low-player-v1' = 'Low' + 'shipping-fidelity-il2cpp-medium-player-v1' = 'Medium' + 'shipping-fidelity-il2cpp-player-v1' = 'High' + } if ( - $canonicalProfileId -cne 'shipping-fidelity-il2cpp-player-v1' -or - $managedStrippingLevel -cne 'High' -or + -not $shippingProfileLevels.Contains($canonicalProfileId) -or + $managedStrippingLevel -cne $shippingProfileLevels[$canonicalProfileId] -or $includeTestAssemblies ) { - throw 'Shipping fidelity requires its reviewed profile with High stripping and includeTestAssemblies=false.' + throw 'Shipping fidelity requires a reviewed Minimal, Low, Medium, or High profile with includeTestAssemblies=false.' } } elseif ( $canonicalProfileId -cne 'canonical-il2cpp-verdict-player-v1' -or diff --git a/scripts/unity/validate-il2cpp-profile.ps1 b/scripts/unity/validate-il2cpp-profile.ps1 index eed41023..cb27746e 100644 --- a/scripts/unity/validate-il2cpp-profile.ps1 +++ b/scripts/unity/validate-il2cpp-profile.ps1 @@ -187,6 +187,18 @@ $profileVariants = [ordered]@{ managedStrippingLevel = 'Disabled' includeTestAssemblies = $true } + 'shipping-fidelity-il2cpp-minimal-player-v1' = [ordered]@{ + managedStrippingLevel = 'Minimal' + includeTestAssemblies = $false + } + 'shipping-fidelity-il2cpp-low-player-v1' = [ordered]@{ + managedStrippingLevel = 'Low' + includeTestAssemblies = $false + } + 'shipping-fidelity-il2cpp-medium-player-v1' = [ordered]@{ + managedStrippingLevel = 'Medium' + includeTestAssemblies = $false + } 'shipping-fidelity-il2cpp-player-v1' = [ordered]@{ managedStrippingLevel = 'High' includeTestAssemblies = $false diff --git a/scripts/validate-unity-pr-policy.py b/scripts/validate-unity-pr-policy.py index aef5deb4..9ffb3f82 100644 --- a/scripts/validate-unity-pr-policy.py +++ b/scripts/validate-unity-pr-policy.py @@ -2605,7 +2605,7 @@ def validate_grouped_unity_correctness() -> None: ) require(run_positions == sorted(run_positions), "grouped Unity modes must run in order") - shipping = step_block(job, "Run stripped shipping-fidelity player") + shipping = step_block(job, "Run stripped shipping-fidelity players") run_positions.append(job.index(shipping)) for fragment in ( "id: run_shipping", @@ -2615,9 +2615,16 @@ def validate_grouped_unity_correctness() -> None: "matrix.unity-version == '6000.5.2f1'", "continue-on-error: true", "timeout-minutes: 150", + "$shippingProfiles = @(", + "Level = 'minimal'; Path = '.github/perf/shipping-fidelity-il2cpp-minimal-profile.v1.json'", + "Level = 'low'; Path = '.github/perf/shipping-fidelity-il2cpp-low-profile.v1.json'", + "Level = 'medium'; Path = '.github/perf/shipping-fidelity-il2cpp-medium-profile.v1.json'", + "Level = 'high'; Path = '.github/perf/shipping-fidelity-il2cpp-profile.v1.json'", + "foreach ($shippingProfile in $shippingProfiles)", "-TestMode shipping", "-AssemblyNames ''", - "-CanonicalProfilePath '.github/perf/shipping-fidelity-il2cpp-profile.v1.json'", + "-ArtifactsPath \".artifacts/unity/${{ matrix.unity-version }}-shipping/$($shippingProfile.Level)\"", + "-CanonicalProfilePath $shippingProfile.Path", "-LicenseReturnOwner Central", ): require(fragment in shipping, f"shipping: grouped run missing {fragment!r}") From 9c0b576168009c41cb05b2d99228de9e81716904 Mon Sep 17 00:00:00 2001 From: wallstop Date: Tue, 1 Sep 2026 06:45:13 +0000 Subject: [PATCH 2/2] Keep shipping diagnostics after profile failures --- .github/workflows/unity-tests.yml | 28 ++--- .../references/unity-ci-matrix.md | 7 +- .../__tests__/shipping-fidelity.test.ps1 | 103 ++++++++++++++++++ .../unity/run-shipping-fidelity-matrix.ps1 | 64 +++++++++++ .../run-shipping-fidelity-matrix.ps1.meta | 7 ++ scripts/validate-unity-pr-policy.py | 63 +++++++++-- 6 files changed, 237 insertions(+), 35 deletions(-) create mode 100644 scripts/unity/run-shipping-fidelity-matrix.ps1 create mode 100644 scripts/unity/run-shipping-fidelity-matrix.ps1.meta diff --git a/.github/workflows/unity-tests.yml b/.github/workflows/unity-tests.yml index fd6b386d..26e4c43b 100644 --- a/.github/workflows/unity-tests.yml +++ b/.github/workflows/unity-tests.yml @@ -490,28 +490,14 @@ jobs: UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_ACCELERATOR_ENDPOINT: ${{ secrets.UNITY_ACCELERATOR_ENDPOINT }} run: | + $projectPathRoot = Join-Path $env:RUNNER_WORKSPACE 'dxm-u\t' $cachePath = Join-Path $env:RUNNER_WORKSPACE 'dxm-c\${{ matrix.unity-version }}' - $shippingProfiles = @( - @{ Level = 'minimal'; Path = '.github/perf/shipping-fidelity-il2cpp-minimal-profile.v1.json' }, - @{ Level = 'low'; Path = '.github/perf/shipping-fidelity-il2cpp-low-profile.v1.json' }, - @{ Level = 'medium'; Path = '.github/perf/shipping-fidelity-il2cpp-medium-profile.v1.json' }, - @{ Level = 'high'; Path = '.github/perf/shipping-fidelity-il2cpp-profile.v1.json' } - ) - foreach ($shippingProfile in $shippingProfiles) { - $projectPath = Join-Path $env:RUNNER_WORKSPACE "dxm-u\t\${{ matrix.unity-version }}-shipping-$($shippingProfile.Level)" - ./scripts/unity/run-ci-tests.ps1 ` - -UnityVersion '${{ matrix.unity-version }}' ` - -UnityInstallRoot (Join-Path $env:RUNNER_TOOL_CACHE 'u6-v3') ` - -TestMode shipping ` - -AssemblyNames '' ` - -ArtifactsPath ".artifacts/unity/${{ matrix.unity-version }}-shipping/$($shippingProfile.Level)" ` - -ProjectPath $projectPath ` - -CachePath $cachePath ` - -CanonicalProfilePath $shippingProfile.Path ` - -LicenseReturnOwner Central ` - -ReleaseCodeOptimization ` - -ReleasePlayerBuild - } + ./scripts/unity/run-shipping-fidelity-matrix.ps1 ` + -UnityVersion '${{ matrix.unity-version }}' ` + -UnityInstallRoot (Join-Path $env:RUNNER_TOOL_CACHE 'u6-v3') ` + -ArtifactsPath '.artifacts/unity/${{ matrix.unity-version }}-shipping' ` + -ProjectPathRoot $projectPathRoot ` + -CachePath $cachePath # A mode failure is tolerated until the test modes and shipping group have # had a chance to run. Preserve diagnostics and result verification, then fail diff --git a/.llm/skills/unity-editor-ci/references/unity-ci-matrix.md b/.llm/skills/unity-editor-ci/references/unity-ci-matrix.md index 35f44941..6764d5b2 100644 --- a/.llm/skills/unity-editor-ci/references/unity-ci-matrix.md +++ b/.llm/skills/unity-editor-ci/references/unity-ci-matrix.md @@ -29,8 +29,11 @@ on Mono; `standalone` builds and runs a `StandaloneWindows64` IL2CPP test player The oldest and current endpoint jobs also build and run Minimal, Low, Medium, and High `StandaloneWindows64` IL2CPP shipping-fidelity players. Each level keeps its own result evidence and runner-local project inside the grouped -shipping step and artifact. The direct runner -generates a package host project under +shipping step and artifact. A failed level is recorded while the later levels +continue only when the wrapper catches its terminating failure, then the group +fails after the remaining levels run. Workflow cancellation or the 150-minute +step timeout still aborts the group. The direct runner generates a package host +project under `$RUNNER_WORKSPACE/dxm-u/t/-/`, imports the repo package with a `file:` dependency, and configures IL2CPP before either player build. Test hosts set `testables`; the shipping host omits test packages and `testables`. Dispatch diff --git a/scripts/unity/__tests__/shipping-fidelity.test.ps1 b/scripts/unity/__tests__/shipping-fidelity.test.ps1 index 0e8c4b75..61bfb649 100644 --- a/scripts/unity/__tests__/shipping-fidelity.test.ps1 +++ b/scripts/unity/__tests__/shipping-fidelity.test.ps1 @@ -8,6 +8,7 @@ $ErrorActionPreference = 'Stop' $repoRoot = Split-Path -Parent (Split-Path -Parent (Split-Path -Parent $PSScriptRoot)) $unityScriptsRoot = Split-Path -Parent $PSScriptRoot $runnerPath = Join-Path $unityScriptsRoot 'run-ci-tests.ps1' +$matrixRunnerPath = Join-Path $unityScriptsRoot 'run-shipping-fidelity-matrix.ps1' $validatorPath = Join-Path $unityScriptsRoot 'validate-il2cpp-profile.ps1' $profilePath = Join-Path $repoRoot '.github/perf/shipping-fidelity-il2cpp-profile.v1.json' $profileCases = @( @@ -70,6 +71,108 @@ function Copy-JsonValue { try { New-Item -ItemType Directory -Force -Path $fixtureRoot | Out-Null + + $matrixInvocationLogPath = Join-Path $fixtureRoot 'matrix-invocations.txt' + $mockRunnerPath = Join-Path $fixtureRoot 'mock-run-ci-tests.ps1' + $escapedMatrixInvocationLogPath = $matrixInvocationLogPath.Replace("'", "''") + [System.IO.File]::WriteAllText( + $mockRunnerPath, + @" +param( + [string]`$UnityVersion, + [string]`$UnityInstallRoot, + [string]`$TestMode, + [string]`$AssemblyNames, + [string]`$ArtifactsPath, + [string]`$RepoRoot, + [string]`$ProjectPath, + [string]`$CachePath, + [string]`$CanonicalProfilePath, + [string]`$LicenseReturnOwner, + [switch]`$ReleaseCodeOptimization, + [switch]`$ReleasePlayerBuild +) + +Add-Content -LiteralPath '$escapedMatrixInvocationLogPath' -Value (([ordered]@{ + unityVersion = `$UnityVersion + unityInstallRoot = `$UnityInstallRoot + testMode = `$TestMode + assemblyNames = `$AssemblyNames + artifactsPath = `$ArtifactsPath + repoRoot = `$RepoRoot + projectPath = `$ProjectPath + cachePath = `$CachePath + canonicalProfilePath = `$CanonicalProfilePath + licenseReturnOwner = `$LicenseReturnOwner + releaseCodeOptimization = `$ReleaseCodeOptimization.IsPresent + releasePlayerBuild = `$ReleasePlayerBuild.IsPresent +} | ConvertTo-Json -Compress)) +if (`$CanonicalProfilePath -clike '*-minimal-profile.v1.json') { + throw 'synthetic Minimal failure' +} +"@ + ) + Assert-Fails 'shipping matrix aggregates a first-cell failure after every profile runs' { + & $matrixRunnerPath ` + -UnityVersion '6000.3.16f1' ` + -UnityInstallRoot (Join-Path $fixtureRoot 'unity') ` + -ArtifactsPath (Join-Path $fixtureRoot 'matrix-artifacts') ` + -ProjectPathRoot (Join-Path $fixtureRoot 'matrix-projects') ` + -CachePath (Join-Path $fixtureRoot 'matrix-cache') ` + -RepoRoot $repoRoot ` + -RunnerPath $mockRunnerPath + } 'Shipping-fidelity profile failures: minimal' + $matrixInvocations = @( + Get-Content -LiteralPath $matrixInvocationLogPath | ConvertFrom-Json + ) + $expectedMatrixCases = @( + [ordered]@{ + Level = 'minimal' + Profile = 'shipping-fidelity-il2cpp-minimal-profile.v1.json' + }, + [ordered]@{ + Level = 'low' + Profile = 'shipping-fidelity-il2cpp-low-profile.v1.json' + }, + [ordered]@{ + Level = 'medium' + Profile = 'shipping-fidelity-il2cpp-medium-profile.v1.json' + }, + [ordered]@{ + Level = 'high' + Profile = 'shipping-fidelity-il2cpp-profile.v1.json' + } + ) + Assert-That 'shipping matrix continues through every profile after an early failure' ( + $matrixInvocations.Count -eq $expectedMatrixCases.Count + ) + for ($matrixIndex = 0; $matrixIndex -lt $expectedMatrixCases.Count; $matrixIndex++) { + $matrixInvocation = $matrixInvocations[$matrixIndex] + $expectedMatrixCase = $expectedMatrixCases[$matrixIndex] + Assert-That "$($expectedMatrixCase.Level) shipping matrix delegates the exact runner contract" ( + $matrixInvocation.unityVersion -ceq '6000.3.16f1' -and + $matrixInvocation.unityInstallRoot -ceq (Join-Path $fixtureRoot 'unity') -and + $matrixInvocation.testMode -ceq 'shipping' -and + $matrixInvocation.assemblyNames -ceq '' -and + $matrixInvocation.artifactsPath -ceq ( + Join-Path (Join-Path $fixtureRoot 'matrix-artifacts') $expectedMatrixCase.Level + ) -and + $matrixInvocation.repoRoot -ceq $repoRoot -and + $matrixInvocation.projectPath -ceq ( + Join-Path (Join-Path $fixtureRoot 'matrix-projects') ( + "6000.3.16f1-shipping-$($expectedMatrixCase.Level)" + ) + ) -and + $matrixInvocation.cachePath -ceq (Join-Path $fixtureRoot 'matrix-cache') -and + $matrixInvocation.canonicalProfilePath -ceq ( + Join-Path $repoRoot ".github/perf/$($expectedMatrixCase.Profile)" + ) -and + $matrixInvocation.licenseReturnOwner -ceq 'Central' -and + $matrixInvocation.releaseCodeOptimization -eq $true -and + $matrixInvocation.releasePlayerBuild -eq $true + ) + } + $tokens = $null $parseErrors = $null $runnerAst = [System.Management.Automation.Language.Parser]::ParseFile( diff --git a/scripts/unity/run-shipping-fidelity-matrix.ps1 b/scripts/unity/run-shipping-fidelity-matrix.ps1 new file mode 100644 index 00000000..f791b756 --- /dev/null +++ b/scripts/unity/run-shipping-fidelity-matrix.ps1 @@ -0,0 +1,64 @@ +#!/usr/bin/env pwsh +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)][string]$UnityVersion, + [Parameter(Mandatory = $true)][string]$UnityInstallRoot, + [Parameter(Mandatory = $true)][string]$ArtifactsPath, + [Parameter(Mandatory = $true)][string]$ProjectPathRoot, + [Parameter(Mandatory = $true)][string]$CachePath, + [string]$RepoRoot = (Split-Path -Parent (Split-Path -Parent $PSScriptRoot)), + [string]$RunnerPath = (Join-Path $PSScriptRoot 'run-ci-tests.ps1') +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +if (-not (Test-Path -LiteralPath $RunnerPath -PathType Leaf)) { + throw "Unity CI runner not found: $RunnerPath" +} + +$shippingProfiles = @( + [ordered]@{ + Level = 'minimal' + Path = '.github/perf/shipping-fidelity-il2cpp-minimal-profile.v1.json' + }, + [ordered]@{ + Level = 'low' + Path = '.github/perf/shipping-fidelity-il2cpp-low-profile.v1.json' + }, + [ordered]@{ + Level = 'medium' + Path = '.github/perf/shipping-fidelity-il2cpp-medium-profile.v1.json' + }, + [ordered]@{ + Level = 'high' + Path = '.github/perf/shipping-fidelity-il2cpp-profile.v1.json' + } +) +$failures = [System.Collections.Generic.List[string]]::new() + +foreach ($shippingProfile in $shippingProfiles) { + try { + & $RunnerPath ` + -UnityVersion $UnityVersion ` + -UnityInstallRoot $UnityInstallRoot ` + -TestMode shipping ` + -AssemblyNames '' ` + -ArtifactsPath (Join-Path $ArtifactsPath $shippingProfile.Level) ` + -RepoRoot $RepoRoot ` + -ProjectPath (Join-Path $ProjectPathRoot "$UnityVersion-shipping-$($shippingProfile.Level)") ` + -CachePath $CachePath ` + -CanonicalProfilePath (Join-Path $RepoRoot $shippingProfile.Path) ` + -LicenseReturnOwner Central ` + -ReleaseCodeOptimization ` + -ReleasePlayerBuild + } catch { + $failure = "{0}: {1}" -f $shippingProfile.Level, $_.Exception.Message + $failures.Add($failure) + Write-Warning "Shipping-fidelity profile failed; continuing to preserve later evidence. $failure" + } +} + +if ($failures.Count -gt 0) { + throw "Shipping-fidelity profile failures: $($failures -join '; ')" +} diff --git a/scripts/unity/run-shipping-fidelity-matrix.ps1.meta b/scripts/unity/run-shipping-fidelity-matrix.ps1.meta new file mode 100644 index 00000000..745c6c68 --- /dev/null +++ b/scripts/unity/run-shipping-fidelity-matrix.ps1.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 2f10c6d059e5312f9c726fa4824d8fc5 +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/scripts/validate-unity-pr-policy.py b/scripts/validate-unity-pr-policy.py index 9ffb3f82..40782e4a 100644 --- a/scripts/validate-unity-pr-policy.py +++ b/scripts/validate-unity-pr-policy.py @@ -16,6 +16,7 @@ WORKFLOW = Path(".github/workflows/unity-tests.yml") WATCHDOG = Path(".github/workflows/stuck-job-watchdog.yml") +SHIPPING_MATRIX = Path("scripts/unity/run-shipping-fidelity-matrix.ps1") LOCK_ACTION_PREFIX = "Ambiguous-Interactive/ambiguous-organization-build-lock/.github/actions/" REGISTERED_UNITY_AUTOMATION = { ".github/actions/validate-unity-license/action.yml", @@ -2615,23 +2616,58 @@ def validate_grouped_unity_correctness() -> None: "matrix.unity-version == '6000.5.2f1'", "continue-on-error: true", "timeout-minutes: 150", - "$shippingProfiles = @(", - "Level = 'minimal'; Path = '.github/perf/shipping-fidelity-il2cpp-minimal-profile.v1.json'", - "Level = 'low'; Path = '.github/perf/shipping-fidelity-il2cpp-low-profile.v1.json'", - "Level = 'medium'; Path = '.github/perf/shipping-fidelity-il2cpp-medium-profile.v1.json'", - "Level = 'high'; Path = '.github/perf/shipping-fidelity-il2cpp-profile.v1.json'", - "foreach ($shippingProfile in $shippingProfiles)", - "-TestMode shipping", - "-AssemblyNames ''", - "-ArtifactsPath \".artifacts/unity/${{ matrix.unity-version }}-shipping/$($shippingProfile.Level)\"", - "-CanonicalProfilePath $shippingProfile.Path", - "-LicenseReturnOwner Central", + "./scripts/unity/run-shipping-fidelity-matrix.ps1", + "-UnityVersion '${{ matrix.unity-version }}'", + "-UnityInstallRoot (Join-Path $env:RUNNER_TOOL_CACHE 'u6-v3')", + "-ArtifactsPath '.artifacts/unity/${{ matrix.unity-version }}-shipping'", + "-ProjectPathRoot $projectPathRoot", + "-CachePath $cachePath", ): require(fragment in shipping, f"shipping: grouped run missing {fragment!r}") require( run_positions == sorted(run_positions), "shipping fidelity must run after the grouped Unity test modes", ) + shipping_matrix = SHIPPING_MATRIX.read_text(encoding="utf-8") + for fragment in ( + "shipping-fidelity-il2cpp-minimal-profile.v1.json", + "shipping-fidelity-il2cpp-low-profile.v1.json", + "shipping-fidelity-il2cpp-medium-profile.v1.json", + "shipping-fidelity-il2cpp-profile.v1.json", + "foreach ($shippingProfile in $shippingProfiles)", + "catch {", + "$failures.Add($failure)", + 'throw "Shipping-fidelity profile failures:', + ): + require( + fragment in shipping_matrix, + f"shipping matrix: missing {fragment!r}", + ) + runner_start = shipping_matrix.find("& $RunnerPath") + runner_end = shipping_matrix.find(" } catch {", runner_start) + require( + runner_start >= 0 and runner_end > runner_start, + "shipping matrix: could not isolate the delegated Unity runner invocation", + ) + runner_invocation = shipping_matrix[runner_start:runner_end] + for fragment in ( + "-UnityVersion $UnityVersion", + "-UnityInstallRoot $UnityInstallRoot", + "-TestMode shipping", + "-AssemblyNames ''", + "-ArtifactsPath (Join-Path $ArtifactsPath $shippingProfile.Level)", + "-RepoRoot $RepoRoot", + '-ProjectPath (Join-Path $ProjectPathRoot "$UnityVersion-shipping-$($shippingProfile.Level)")', + "-CachePath $CachePath", + "-CanonicalProfilePath (Join-Path $RepoRoot $shippingProfile.Path)", + "-LicenseReturnOwner Central", + "-ReleaseCodeOptimization", + "-ReleasePlayerBuild", + ): + require( + fragment in runner_invocation, + f"shipping matrix runner invocation: missing {fragment!r}", + ) shipping_upload = step_block(job, "Upload shipping-fidelity artifacts") require( "if-no-files-found: error" in shipping_upload @@ -2641,7 +2677,10 @@ def validate_grouped_unity_correctness() -> None: and ".artifacts/unity/${{ matrix.unity-version }}-shipping" in shipping_upload, "shipping fidelity must upload isolated evidence after lock acquisition, skip cancellation, and fail when it is absent", ) - require(job.count("-LicenseReturnOwner Central") == 4, "every grouped mode needs central cleanup") + require( + job.count("-LicenseReturnOwner Central") == 3, + "each directly invoked grouped test mode needs central cleanup", + ) gate = step_block(job, "Require every Unity mode to pass") require(