diff --git a/.github/workflows/spark-ci.yml b/.github/workflows/spark-ci.yml index 5b13ae9bc463..15991785ca7f 100644 --- a/.github/workflows/spark-ci.yml +++ b/.github/workflows/spark-ci.yml @@ -75,12 +75,18 @@ concurrency: jobs: spark-tests: runs-on: ubuntu-24.04 + timeout-minutes: 90 strategy: - max-parallel: 15 + # 20 is the Apache infra policy ceiling (https://infra.apache.org/github-actions-policy.html). + # Keep matrix <= 20 jobs; exceeding it queues a second wave and slows CI. + max-parallel: 20 matrix: jvm: [17, 21] spark: ['3.4', '3.5', '4.0', '4.1'] scala: ['2.12', '2.13'] + # Split iceberg-spark (core) from iceberg-spark-extensions/-runtime + # so they run as concurrent jobs rather than serially in one job. + tests: [core, extensions] exclude: # Spark 3.5 is the first version not failing on Java 21 (https://issues.apache.org/jira/browse/SPARK-42369) # Full Java 21 support is coming in Spark 4 (https://issues.apache.org/jira/browse/SPARK-43831) @@ -104,19 +110,20 @@ jobs: with: # Read-only: java-ci's build-checks (17) is the global canonical writer. cache-read-only: true - - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - with: - tool-cache: false - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts - - run: | + - name: Run tests + run: | + if [[ "${{ matrix.tests }}" == "core" ]]; then + projects=":iceberg-spark:iceberg-spark-${{ matrix.spark }}_${{ matrix.scala }}:check" + else + projects=":iceberg-spark:iceberg-spark-extensions-${{ matrix.spark }}_${{ matrix.scala }}:check :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark }}_${{ matrix.scala }}:check" + fi ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersion=${{ matrix.scala }} -DflinkVersions= -DkafkaVersions= \ - :iceberg-spark:iceberg-spark-${{ matrix.spark }}_${{ matrix.scala }}:check \ - :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark }}_${{ matrix.scala }}:check \ - :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark }}_${{ matrix.scala }}:check \ - -Pquick=true -x javadoc + $projects -Pquick=true -x javadoc -DtestParallelism=auto - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: failure() with: - name: test logs + name: test logs (${{ matrix.jvm }}, ${{ matrix.spark }}, ${{ matrix.scala }}, ${{ matrix.tests }}) + retention-days: 7 path: | **/build/testlogs