Common Test Strategy

.github/actions/run-ledger-tests/action.yml

@@ -0,0 +1,182 @@
+name: Run Ledger clear-signing tests
+description: Execute clear-signing tests against the Ledger Speculos emulator using cs-tester from device-sdk-ts.
+
+inputs:
+  registry-path:
+    description: Absolute path to the registry checkout (contains the descriptor and tests files).
+    required: true
+  descriptor:
+    description: Path to the descriptor JSON file, relative to registry-path.
+    required: true
+  test-file:
+    description: Path to the .tests.json file, relative to registry-path.
+    required: true
+  test-type:
+    description: Test type — "calldata" or "eip712".
+    required: true
+  device:
+    description: Device target (e.g. flex).
+    required: true
+  entity:
+    description: Entity name, used for artifact naming.
+    required: true
+  descriptor-name:
+    description: Descriptor base name, used for artifact naming.
+    required: true
+  gh-bot-app-id:
+    description: GitHub App ID for accessing the private coin-apps repository.
+    required: true
+  gh-bot-private-key:
+    description: GitHub App private key.
+    required: true
+  gating-token:
+    description: Gating token for cs-tester.
+    required: true
+
+outputs:
+  exit-code:
+    description: Exit code of the cs-tester run (0 = all tests passed / clear signed).
+    value: ${{ steps.run-tests.outputs.exit_code }}
+
+runs:
+  using: composite
+  steps:
+    - name: Checkout device-management-kit
+      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      with:
+        repository: LedgerHQ/device-sdk-ts
+        ref: 2e35ed527c1a9aee852809a39ed2eaf76a415af1 # develop @ 2026-05-28
+        path: dmk
+
+    - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
+      with:
+        node-version: '20'
+
+    - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5
+      with:
+        version: 9
+
+    - name: Install and build
+      shell: bash
+      working-directory: dmk
+      run: |
+        pnpm install
+        pnpm build:libs
+
+    - name: Generate token for coin-apps
+      id: generate-token
+      uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3
+      with:
+        app-id: ${{ inputs.gh-bot-app-id }}
+        private-key: ${{ inputs.gh-bot-private-key }}
+        owner: LedgerHQ
+        repositories: coin-apps
+
+    - name: Checkout coin-apps
+      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      with:
+        repository: LedgerHQ/coin-apps
+        # TODO: pin to a specific commit SHA (coin-apps is private — pin
+        # using a SHA resolved with the CI bot's token).
+        ref: master
+        token: ${{ steps.generate-token.outputs.token }}
+        path: coin-apps
+        sparse-checkout: ${{ inputs.device }}/*/Ethereum/*.elf
+        sparse-checkout-cone-mode: false
+
+    - name: Pull Speculos Docker image
+      shell: bash
+      run: docker pull ghcr.io/ledgerhq/speculos@sha256:9b414c3bcaecb7638224156d36a702d66af812a0aa59ca203ce7b34cf4d590ca
+
+    - name: Prepare test inputs
+      shell: bash
+      env:
+        REGISTRY_PATH: ${{ inputs.registry-path }}
+        TEST_FILE: ${{ inputs.test-file }}
+      run: |
+        mkdir -p test-inputs screenshots
+        jq '.tests' "$REGISTRY_PATH/$TEST_FILE" > test-inputs/tests.json
+
+    - name: Resolve descriptor includes
+      shell: bash
+      env:
+        REGISTRY_PATH: ${{ inputs.registry-path }}
+        DESCRIPTOR: ${{ inputs.descriptor }}
+        WORKSPACE: ${{ github.workspace }}
+        ACTION_PATH: ${{ github.action_path }}
+      run: |
+        node "$ACTION_PATH/../../scripts/resolve-erc7730-includes.js" \
+          "$REGISTRY_PATH/$DESCRIPTOR" \
+          "$WORKSPACE/test-inputs/resolved-descriptor.json"
+
+    - name: Run clear signing tests
+      id: run-tests
+      shell: bash
+      working-directory: dmk
+      env:
+        FORCE_COLOR: "0"
+        GATING_TOKEN: ${{ inputs.gating-token }}
+        COIN_APPS_PATH: ${{ github.workspace }}/coin-apps
+        WORKSPACE: ${{ github.workspace }}
+        TEST_TYPE: ${{ inputs.test-type }}
+        DEVICE: ${{ inputs.device }}
+      run: |
+        CMD=$([[ "$TEST_TYPE" == "calldata" ]] && echo "raw-file" || echo "typed-data-file")
+
+        set +e
+        pnpm cs-tester cli $CMD "$WORKSPACE/test-inputs/tests.json" \
+          --device "$DEVICE" \
+          --erc7730-files "$WORKSPACE/test-inputs/resolved-descriptor.json" \
+          --screenshot-folder-path "$WORKSPACE/screenshots" \
+          --log-level info 2>&1 | tee test-output.log
+        EXIT_CODE=${PIPESTATUS[0]}
+        echo "exit_code=$EXIT_CODE" >> $GITHUB_OUTPUT
+
+    - name: Write job summary
+      if: always()
+      shell: bash
+      working-directory: dmk
+      env:
+        ENTITY: ${{ inputs.entity }}
+        DESCRIPTOR_NAME: ${{ inputs.descriptor-name }}
+        DEVICE: ${{ inputs.device }}
+        EXIT_CODE: ${{ steps.run-tests.outputs.exit_code }}
+      run: |
+        echo "## $ENTITY / $DESCRIPTOR_NAME ($DEVICE)" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+
+        if [ "$EXIT_CODE" == "0" ]; then
+          echo "✅ **All tests passed (clear signed)**" >> $GITHUB_STEP_SUMMARY
+        else
+          echo "⚠️ **Exit code: $EXIT_CODE**" >> $GITHUB_STEP_SUMMARY
+        fi
+
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo "### Test Output (last 30 lines)" >> $GITHUB_STEP_SUMMARY
+        echo '```' >> $GITHUB_STEP_SUMMARY
+        tail -30 test-output.log >> $GITHUB_STEP_SUMMARY 2>/dev/null || echo "No output available"
+        echo '```' >> $GITHUB_STEP_SUMMARY
+
+    - name: Save exit code for PR comment
+      if: always()
+      shell: bash
+      env:
+        EXIT_CODE: ${{ steps.run-tests.outputs.exit_code }}
+      run: |
+        echo "$EXIT_CODE" > screenshots/exit_code.txt
+
+    - name: Upload screenshots
+      if: always()
+      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+      with:
+        name: screenshots-${{ inputs.device }}-${{ inputs.entity }}-${{ inputs.descriptor-name }}
+        path: screenshots/
+        retention-days: 30
+
+    - name: Upload logs
+      if: always()
+      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+      with:
+        name: logs-${{ inputs.device }}-${{ inputs.entity }}-${{ inputs.descriptor-name }}
+        path: dmk/test-output.log
+        retention-days: 7

.github/actions/run-rust-tests/action.yml

@@ -0,0 +1,50 @@
+name: Run Rust (llbartekll/clear-signing) clear-signing tests
+description: Install llbartekll/clear-signing's cs-test binary via cargo, run it against a .tests.json fixture, and produce a results.json conforming to the format in .github/test-runner-docs/.
+
+inputs:
+  test-file:
+    description: Absolute path to the .tests.json fixture.
+    required: true
+  output:
+    description: Absolute path where results.json should be written.
+    required: true
+
+outputs:
+  results-path:
+    description: Path to the produced results.json.
+    value: ${{ inputs.output }}
+
+runs:
+  using: composite
+  steps:
+    - name: Checkout llbartekll/clear-signing
+      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      with:
+        repository: llbartekll/clear-signing
+        ref: 0809971c81182ea93271759c62c04d1935415eff # main @ 2026-06-19
+        path: cs-runner
+        persist-credentials: false
+
+    - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
+
+    - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
+      with:
+        workspaces: cs-runner
+
+    - name: Install cs-test
+      shell: bash
+      working-directory: cs-runner
+      run: cargo install --path crates/cs-test --locked
+
+    - name: Run tests
+      shell: bash
+      env:
+        TEST_FILE: ${{ inputs.test-file }}
+        OUTPUT: ${{ inputs.output }}
+      run: |
+        # Registry root = two levels up from the test file's `testsv2` dir
+        # (.../registry/<entity>/testsv2/<name>.tests.json). This lets the runner
+        # resolve nested-call descriptors (e.g. Safe inner call, kiln inner
+        # deposit) from anywhere in the tree, not just beside the test file.
+        REGISTRY_ROOT="$(cd "$(dirname "$TEST_FILE")/../.." && pwd)"
+        cs-test run "$TEST_FILE" --registry "$REGISTRY_ROOT" --output "$OUTPUT"

.github/actions/run-sourcify-tests/action.yml

@@ -0,0 +1,47 @@
+name: Run Sourcify clear-signing tests
+description: Clone and build @ethereum-sourcify/clear-signing-test-runner, then run it against a .tests.json fixture and produce a results.json conforming to the format in .github/test-runner-docs/.
+
+inputs:
+  test-file:
+    description: Absolute path to the .tests.json fixture.
+    required: true
+  output:
+    description: Absolute path where results.json should be written.
+    required: true
+outputs:
+  results-path:
+    description: Path to the produced results.json.
+    value: ${{ inputs.output }}
+
+runs:
+  using: composite
+  steps:
+    - name: Checkout Sourcify test runner
+      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      with:
+        repository: sourcifyeth/clear-signing-test-runner
+        ref: 3006d68e79c34278008000dff95a439d2b7c7665 # main @ 2026-06-08
+        path: sourcify-test-runner
+        persist-credentials: false
+
+    - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
+      with:
+        node-version: '22'
+
+    - name: Install dependencies
+      shell: bash
+      working-directory: sourcify-test-runner
+      run: npm ci
+
+    - name: Build
+      shell: bash
+      working-directory: sourcify-test-runner
+      run: npm run build
+
+    - name: Run tests
+      shell: bash
+      working-directory: sourcify-test-runner
+      env:
+        TEST_FILE: ${{ inputs.test-file }}
+        OUTPUT: ${{ inputs.output }}
+      run: node dist/cli.js "$TEST_FILE" --output "$OUTPUT" --verbose

.github/actions/upload-test-results/action.yml

@@ -0,0 +1,70 @@
+name: Upload clear-signing test results
+description: Validate the basic structure of a results.json against the spec in .github/test-runner-docs/README.md, then upload it as an artifact with a consistent name. Shared by all runners that emit the unified results format.
+
+inputs:
+  results-path:
+    description: Absolute path to the results.json file.
+    required: true
+  implementation-slug:
+    description: Short slug identifying the implementation (e.g. sourcify-ts-clear-signing). Used in the artifact name; the full package@version identifier lives inside the JSON.
+    required: true
+  entity:
+    description: Entity name (e.g. aave).
+    required: true
+  descriptor-name:
+    description: Descriptor name (e.g. calldata-lpv2).
+    required: true
+  retention-days:
+    description: Artifact retention in days.
+    required: false
+    default: '30'
+
+runs:
+  using: composite
+  steps:
+    - name: Validate results.json structure
+      shell: bash
+      env:
+        RESULTS_PATH: ${{ inputs.results-path }}
+      run: |
+        if [ ! -f "$RESULTS_PATH" ]; then
+          echo "::error::results.json not found at $RESULTS_PATH"
+          exit 1
+        fi
+
+        if ! jq -e '
+          has("runner") and
+          has("implementation") and
+          has("cases") and
+          (.cases | type == "array") and
+          (.cases | all(
+            has("description") and
+            has("status") and
+            (.status | IN("pass","fail","error","skipped"))
+          ))
+        ' "$RESULTS_PATH" > /dev/null; then
+          echo "::error file=$RESULTS_PATH::results.json failed structural validation against the spec in .github/test-runner-docs/README.md"
+          exit 1
+        fi
+
+    - name: Stage with unique filename
+      shell: bash
+      env:
+        RESULTS_PATH: ${{ inputs.results-path }}
+        SLUG: ${{ inputs.implementation-slug }}
+        ENTITY: ${{ inputs.entity }}
+        DESCRIPTOR: ${{ inputs.descriptor-name }}
+        WORKSPACE: ${{ github.workspace }}
+      run: |
+        # Rename so multi-artifact downloads (with merge-multiple: true) don't
+        # collide on a shared `results.json` filename. The slug+entity+descriptor
+        # combination is guaranteed unique per matrix entry.
+        mkdir -p "$WORKSPACE/results-staging"
+        cp "$RESULTS_PATH" "$WORKSPACE/results-staging/${SLUG}__${ENTITY}__${DESCRIPTOR}.json"
+
+    - name: Upload results
+      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+      with:
+        name: results__${{ inputs.implementation-slug }}__${{ inputs.entity }}__${{ inputs.descriptor-name }}
+        path: ${{ github.workspace }}/results-staging/${{ inputs.implementation-slug }}__${{ inputs.entity }}__${{ inputs.descriptor-name }}.json
+        retention-days: ${{ inputs.retention-days }}