#483: Spark 4+Java17 compatibility (Agent)

.github/workflows/build.yml

@@ -24,8 +24,27 @@ on:
 
 jobs:
   test-agent-reader-and-model:
-    name: Test Agent, Reader and Model
-    runs-on: ubuntu-latest
+    name: ${{ matrix.name }}
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: Test Agent, Reader and Model (Spark 3, Java 8)
+            java-version: '8'
+            test-command: testAgentReaderModelSpark3
+            doc-command: docAgentReaderModelSpark3
+
+          - name: Test Agent, Reader and Model (Spark 3, Java 11)
+            java-version: '11'
+            test-command: testAgentReaderModelSpark3
+            doc-command: docAgentReaderModelSpark3
+
+          - name: Test Agent, Reader and Model (Spark 4, Java 17)
+            java-version: '17'
+            test-command: testAgentReaderModelSpark4
+            doc-command: docAgentReaderModelSpark4
+
     steps:
       - name: Checkout code
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
@@ -34,20 +53,21 @@ jobs:
           persist-credentials: false
       - uses: coursier/cache-action@95e5b1029b6b86e7bac033ee44a0697d8a527d2d
 
-      - name: Setup Scala
-        uses: olafurpg/setup-scala@32ffa16635ff8f19cc21ea253a987f0fdf29844c
+      - name: Setup JDK ${{ matrix.java-version }}
+        uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1
         with:
-          java-version: "adopt@1.8"
+          distribution: temurin
+          java-version: ${{ matrix.java-version }}
 
       - name: Build and run tests
-        run: sbt testAllStandard
+        run: sbt ${{ matrix.test-command }}
 
-      - name: Generate documenation
-        run: sbt doc
+      - name: Generate documentation
+        run: sbt ${{ matrix.doc-command }}
 
   test-database-and-server:
-    name: Test Database and Server
-    runs-on: ubuntu-latest
+    name: Test Database and Server (Java 11)
+    runs-on: ubuntu-22.04
     services:
       postgres:
         image: postgres:15
@@ -69,10 +89,11 @@ jobs:
           persist-credentials: false
       - uses: coursier/cache-action@95e5b1029b6b86e7bac033ee44a0697d8a527d2d
 
-      - name: Setup Scala
-        uses: olafurpg/setup-scala@32ffa16635ff8f19cc21ea253a987f0fdf29844c
+      - name: Setup JDK 11
+        uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1
         with:
-          java-version: "adopt@1.11.0-11"
+          distribution: temurin
+          java-version: '11'
 
       - name: Build and run unit tests
         run: sbt "project database" test "project server" test
@@ -88,7 +109,7 @@ jobs:
 
   sbom-and-aquasec-manifest:
     name: SBOM and Aquasec Manifest
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout code
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1

.github/workflows/format_check.yml

@@ -24,7 +24,7 @@ on:
 jobs:
   scala-format-check:
     name: Scala Format Check
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout code
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
@@ -33,10 +33,11 @@ jobs:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.ref }}
 
-      - name: Setup Scala
-        uses: olafurpg/setup-scala@32ffa16635ff8f19cc21ea253a987f0fdf29844c
+      - name: Setup JDK 8
+        uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1
         with:
-          java-version: "adopt@1.8"
+          distribution: temurin
+          java-version: '8'
 
       - name: Run scalafmt And Print Diff
         continue-on-error: true

.github/workflows/jacoco_report.yml

@@ -51,7 +51,7 @@ env:
 jobs:
   detect:
     name: Detect Changed Files
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     timeout-minutes: 2
     outputs:
       scala_changed: ${{ steps.filter.outputs.scala }}
@@ -74,7 +74,7 @@ jobs:
     name: JaCoCo Report
     needs: detect
     if: needs.detect.outputs.scala_changed == 'true'
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     permissions:
       contents: read
       issues: write
@@ -101,10 +101,11 @@ jobs:
           persist-credentials: false
           fetch-depth: 0
 
-      - name: Setup Scala
-        uses: olafurpg/setup-scala@32ffa16635ff8f19cc21ea253a987f0fdf29844c
+      - name: Setup JDK 11
+        uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1
         with:
-          java-version: "adopt@1.11.0-11"
+          distribution: temurin
+          java-version: '11'
 
       - name: Prepare testing database
         run: sbt flywayMigrate
@@ -132,6 +133,6 @@ jobs:
     name: No Operation
     needs: detect
     if: needs.detect.outputs.scala_changed != 'true'
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - run: echo "No changes in Scala related files — passing."

.github/workflows/license_check.yml

@@ -24,14 +24,15 @@ on:
 jobs:
   license-check:
     name: License Check
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout code
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
         with:
           persist-credentials: false
-      - name: Setup Scala
-        uses: olafurpg/setup-scala@32ffa16635ff8f19cc21ea253a987f0fdf29844c
+      - name: Setup JDK 8
+        uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1
         with:
-          java-version: "adopt@1.8"
+          distribution: temurin
+          java-version: '8'
       - run: sbt headerCheck

.github/workflows/release_publish.yml

@@ -22,7 +22,7 @@ on:
 jobs:
   publish-to-sonatype:
     name: Publish to Sonatype
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout code
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
@@ -31,10 +31,14 @@ jobs:
           fetch-depth: 0
       - uses: coursier/cache-action@95e5b1029b6b86e7bac033ee44a0697d8a527d2d
 
-      - name: Setup Scala
-        uses: olafurpg/setup-scala@32ffa16635ff8f19cc21ea253a987f0fdf29844c
+      - name: Setup JDK 17
+        # The Spark 4 rows of the agent are only added to the project matrix on Java 17+, so publishing has to
+        # run on a JDK 17 or `atum-agent-spark4_2.13` would silently never be released. The Spark 3 / model /
+        # reader rows still compile with `-release 8`, so their bytecode stays Java 8 compatible either way.
+        uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1
         with:
-          java-version: "adopt@1.8"
+          distribution: temurin
+          java-version: '17'
 
       - name: Run sbt ci-release
         run: sbt ci-release
@@ -47,7 +51,7 @@ jobs:
 
   publish-server-jar:
     name: Publish server jar
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout code
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
@@ -56,10 +60,12 @@ jobs:
           fetch-depth: 0
       - uses: coursier/cache-action@95e5b1029b6b86e7bac033ee44a0697d8a527d2d
 
-      - name: Setup Scala
-        uses: olafurpg/setup-scala@32ffa16635ff8f19cc21ea253a987f0fdf29844c
+      - name: Setup JDK 11
+        uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1
         with:
-          java-version: "adopt@1.11.0-11"
+          distribution: temurin
+          java-version: '11'
+
       - name: Run sbt assembly (produces jar)
         run: sbt assembly
 

.sbtrc

@@ -14,26 +14,43 @@
 # limitations under the License.
 #
 
-# Aliases in this file expected usage of test file naming conventions:
-#   - "UnitTests" suffix for test files and Suites which define unit tests
-#   - "IntegrationTests" suffix for test files and Suites which define integration tests
-#   - "CompatibilityTests" suffix for test files and Suites which define Agent <-> Server compatibility tests
+# Naming convention for aliases in this file:
+#   - Type-based tests; `test<Type>` acts on whichever project is currently selected; no scope of their own:
+#       `test`                       - unit tests only            (files/suites named `*UnitTests`)
+#       `testIT`                     - integration tests only     (files/suites named `*IntegrationTests`)
+#       `testStandard`               - both of the above combined ("standard", as opposed to the special kinds below)
+#       `testCompatibility`          - Agent <-> Server compatibility tests (files/suites named `*CompatibilityTests`)
+#
+#   - Scoped-based tests; `test<Scope>` aliases always mean "run `testStandard` for exactly this scope":
+#       `testAgentSpark3`            - both Spark 3 agent rows (Scala 2.12 + 2.13), recommended Java 11 runtime
+#       `testAgentSpark4`            - the Spark 4 agent row (Scala 2.13), requires Java 17+
+#       `testAgentReaderModelSpark3` - as `testAgentSpark3` but also with reader + model
+#       `testAgentReaderModelSpark4` - as `testAgentSpark4` but also with reader + model
+#       `testDB`                     - the `database` module
+#
+#   - `doc<Scope>` aliases mirror the `test<Scope>` scopes above but generate docs instead of running tests.
 
-# CPS QA types aliases
-# Run unit tests
+# Type-based tests
 alias test=; testOnly *UnitTests
-
-# Run integration tests
 alias testIT=; testOnly *IntegrationTests
+alias testStandard=; test; testIT
+alias testCompatibility=; testOnly *CompatibilityTests
 
-# Run all standard tests (all except the special ones below)
-alias testAllStandard=; test; testIT
+# Scope: agent, per Spark build row
+alias testAgentSpark3=; project agent_spark3; testStandard; project agent_spark32_12; testStandard
+alias testAgentSpark4=; project agent_spark4; testStandard
+alias docAgentSpark3=; project agent_spark3; doc; project agent_spark32_12; doc
+alias docAgentSpark4=; project agent_spark4; doc
 
-# Run specific: database tests using Balta
-alias testDB=; project database; testOnly *
+# Scope: reader + model + a specific agent Spark build row
+alias testAgentReaderModelSpark3=; project reader; testStandard; project model; testStandard; testAgentSpark3
+alias docAgentReaderModelSpark3=; project reader; doc; project model; doc; docAgentSpark3
 
-# Run specific: agent-server compatibility tests
-alias testCompatibility=; testOnly *CompatibilityTests
+alias testAgentReaderModelSpark4=; project reader; testStandard; project model; testStandard; testAgentSpark4
+alias docAgentReaderModelSpark4=; project reader; doc; project model; doc; docAgentSpark4
+
+# Scope: database
+alias testDB=; project database; testStandard
 
 # Jacoco Aliases
 alias jacoco=; jacocoOn; jacocoAgent; jacocoReader; jacocoModel; jacocoServer; jacocoOff