ci: various tweaks

.github/workflows/main.yml

@@ -39,10 +39,21 @@ env:
   # TODO(yy): remove this value and use TRANCHES.
   SMALL_TRANCHES: 8
 
+  # SPLIT_ITEST_PARALLELISM defines the number of tranches each runner job
+  # of a tranche-split itest configuration runs in parallel. Split configs
+  # run SMALL_TRANCHES tranches in total, spread over
+  # SMALL_TRANCHES / SPLIT_ITEST_PARALLELISM runner jobs.
+  SPLIT_ITEST_PARALLELISM: 2
+
   # If you change this please also update GO_VERSION in Makefile (then run
   # `make lint` to see where else it needs to be updated as well).
   GO_VERSION: 1.26.4
 
+  # NUM_UNIT_RACE_TRANCHES defines the number of tranches the race detector
+  # unit tests are split into, each tranche running on its own runner. Must
+  # match the tranche list in the unit-race matrix.
+  NUM_UNIT_RACE_TRANCHES: 4
+
 jobs:
   static-checks:
     name: Static Checks
@@ -212,9 +223,6 @@ jobs:
           - unit tags="kvdb_sqlite"
           - unit tags="test_db_sqlite"
           - unit tags="test_db_postgres"
-          - unit-race
-          - unit-race tags="test_db_sqlite"
-          - unit-race tags="test_db_postgres"
           - unit-module
 
     steps:
@@ -266,6 +274,60 @@ jobs:
           parallel: true
 
 
+  ########################
+  # run unit tests in race detector mode
+  ########################
+  unit-race:
+    name: Unit race (${{ matrix.flavor.name }}, ${{ matrix.tranche }})
+    runs-on: ubuntu-latest
+    strategy:
+      # Allow other tests in the matrix to continue if one fails.
+      fail-fast: false
+      matrix:
+        # This list must contain NUM_UNIT_RACE_TRANCHES entries.
+        tranche: [0, 1, 2, 3]
+        flavor:
+          - name: unit-race
+            args: ''
+          - name: unit-race-sqlite
+            args: tags="test_db_sqlite"
+          - name: unit-race-postgres
+            args: tags="test_db_postgres"
+
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+
+      - name: Clean up runner space
+        uses: ./.github/actions/cleanup-space
+
+      - name: Fetch and rebase on ${{ github.base_ref }}
+        if: github.event_name == 'pull_request'
+        uses: ./.github/actions/rebase
+
+      - name: Git checkout fuzzing seeds
+        uses: actions/checkout@v5
+        with:
+          repository: lightninglabs/lnd-fuzz
+          path: lnd-fuzz
+
+      - name: Rsync fuzzing seeds
+        run: rsync -a --ignore-existing lnd-fuzz/ ./
+
+      - name: Setup go ${{ env.GO_VERSION }}
+        uses: ./.github/actions/setup-go
+        with:
+          go-version: '${{ env.GO_VERSION }}'
+          key-prefix: unit-test
+
+      - name: Install bitcoind
+        run: ./scripts/install_bitcoind.sh $BITCOIN_VERSION
+
+      - name: Run ${{ matrix.flavor.name }} tranche ${{ matrix.tranche }}
+        run: make unit-race-parallel tranche=${{ matrix.tranche }} tranches=${{ env.NUM_UNIT_RACE_TRANCHES }} ${{ matrix.flavor.args }}
+
   ########################
   # run integration tests with TRANCHES
   ########################
@@ -371,12 +433,6 @@ jobs:
             args: backend=bitcoind dbbackend=sqlite nativesql=true
           - name: bitcoind-sqlite=nativesql-experiment
             args: backend=bitcoind dbbackend=sqlite nativesql=true tags=test_native_sql
-          - name: bitcoind-postgres
-            args: backend=bitcoind dbbackend=postgres
-          - name: bitcoind-postgres-nativesql
-            args: backend=bitcoind dbbackend=postgres nativesql=true
-          - name: bitcoind-postgres-nativesql-experiment
-            args: backend=bitcoind dbbackend=postgres nativesql=true tags=test_native_sql
     steps:
       - name: Git checkout
         uses: actions/checkout@v5
@@ -441,6 +497,139 @@ jobs:
           retention-days: 5
 
 
+  ########################
+  # build binaries for tranche-split integration tests
+  ########################
+  build-split-itest:
+    name: Build split itest binaries
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: bitcoind-postgres
+            args: backend=bitcoind dbbackend=postgres
+          - name: bitcoind-postgres-experiment
+            args: backend=bitcoind dbbackend=postgres tags=test_native_sql
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+
+      - name: Check for no-itest label
+        id: check-label
+        uses: ./.github/actions/check-label
+        with:
+          label: 'no-itest'
+          skip-message: "Tests auto-passed due to 'no-itest' label"
+
+      - name: Clean up runner space
+        if: steps.check-label.outputs.skip != 'true'
+        uses: ./.github/actions/cleanup-space
+
+      - name: Fetch and rebase on ${{ github.base_ref }}
+        if: github.event_name == 'pull_request' && steps.check-label.outputs.skip != 'true'
+        uses: ./.github/actions/rebase
+
+      - name: Setup go ${{ env.GO_VERSION }}
+        if: steps.check-label.outputs.skip != 'true'
+        uses: ./.github/actions/setup-go
+        with:
+          go-version: '${{ env.GO_VERSION }}'
+          key-prefix: integration-test
+
+      - name: Build itest binaries for ${{ matrix.name }}
+        if: steps.check-label.outputs.skip != 'true'
+        run: make build-itest ${{ matrix.args }}
+
+      - name: Upload itest binaries
+        if: steps.check-label.outputs.skip != 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: itest-binaries-${{ matrix.name }}
+          path: |
+            itest/itest.test
+            itest/lnd-itest
+            itest/btcd-itest
+          retention-days: 1
+
+  ########################
+  # run tranche-split integration tests
+  ########################
+  split-integration-test:
+    name: Split itests (${{ matrix.config.name }}, ${{ matrix.tranche_group }})
+    runs-on: ubuntu-latest
+    needs: build-split-itest
+    strategy:
+      # Allow other tests in the matrix to continue if one fails.
+      fail-fast: false
+      matrix:
+        # Each tranche group runs SPLIT_ITEST_PARALLELISM tranches, so
+        # this list must contain SMALL_TRANCHES / SPLIT_ITEST_PARALLELISM
+        # entries.
+        tranche_group: [0, 1, 2, 3]
+        config:
+          - name: bitcoind-postgres
+            binaries: bitcoind-postgres
+            seed: 1
+            args: backend=bitcoind dbbackend=postgres
+          - name: bitcoind-postgres-nativesql
+            binaries: bitcoind-postgres
+            seed: 2
+            args: backend=bitcoind dbbackend=postgres nativesql=true
+          - name: bitcoind-postgres-nativesql-experiment
+            binaries: bitcoind-postgres-experiment
+            seed: 3
+            args: backend=bitcoind dbbackend=postgres nativesql=true tags=test_native_sql
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v5
+
+      - name: Check for no-itest label
+        id: check-label
+        uses: ./.github/actions/check-label
+        with:
+          label: 'no-itest'
+          skip-message: "Tests auto-passed due to 'no-itest' label"
+
+      - name: Download itest binaries
+        if: steps.check-label.outputs.skip != 'true'
+        uses: actions/download-artifact@v4
+        with:
+          name: itest-binaries-${{ matrix.config.binaries }}
+          path: itest
+
+      - name: Make itest binaries executable
+        if: steps.check-label.outputs.skip != 'true'
+        run: chmod +x itest/itest.test itest/lnd-itest itest/btcd-itest
+
+      - name: Install bitcoind
+        if: steps.check-label.outputs.skip != 'true'
+        run: ./scripts/install_bitcoind.sh $BITCOIN_VERSION
+
+      - name: Run ${{ matrix.config.name }} tranche group ${{ matrix.tranche_group }}
+        if: steps.check-label.outputs.skip != 'true'
+        run: |
+          OFFSET=$((${{ matrix.tranche_group }} * ${{ env.SPLIT_ITEST_PARALLELISM }}))
+          make itest-only-parallel tranches=${{ env.SMALL_TRANCHES }} \
+            parallel=${{ env.SPLIT_ITEST_PARALLELISM }} trancheoffset=$OFFSET \
+            ${{ matrix.config.args }} \
+            shuffleseed=${{ github.run_id }}${{ matrix.config.seed }}
+
+      - name: Zip log files on failure
+        if: ${{ failure() && steps.check-label.outputs.skip != 'true' }}
+        timeout-minutes: 5 # timeout after 5 minute
+        run: 7z a logs-itest-${{ matrix.config.name }}-${{ matrix.tranche_group }}.zip itest/**/*.log
+
+      - name: Upload log files on failure
+        uses: actions/upload-artifact@v4
+        if: ${{ failure() && steps.check-label.outputs.skip != 'true' }}
+        with:
+          name: logs-itest-${{ matrix.config.name }}-${{ matrix.tranche_group }}
+          path: logs-itest-${{ matrix.config.name }}-${{ matrix.tranche_group }}.zip
+          retention-days: 5
+
   ########################
   # run windows integration test
   ########################

Makefile

@@ -263,13 +263,16 @@ itest: build-itest itest-only
 #? itest-race: Build and run integration tests in race detector mode
 itest-race: build-itest-race itest-only
 
-#? itest-parallel: Build and run integration tests in parallel mode, running up to ITEST_PARALLELISM test tranches in parallel (default 4)
-itest-parallel: clean-itest-logs build-itest db-instance
+#? itest-only-parallel: Run integration tests in parallel mode without re-building binaries, running up to ITEST_PARALLELISM test tranches in parallel starting at the tranche given by trancheoffset (default 0)
+itest-only-parallel: clean-itest-logs db-instance
 	@$(call print, "Running tests")
 	date
-	EXEC_SUFFIX=$(EXEC_SUFFIX) scripts/itest_parallel.sh $(ITEST_PARALLELISM) $(NUM_ITEST_TRANCHES) $(SHUFFLE_SEED) $(TEST_FLAGS) $(ITEST_FLAGS)
+	EXEC_SUFFIX=$(EXEC_SUFFIX) ITEST_TRANCHE_OFFSET=$(ITEST_TRANCHE_OFFSET) scripts/itest_parallel.sh $(ITEST_PARALLELISM) $(NUM_ITEST_TRANCHES) $(SHUFFLE_SEED) $(TEST_FLAGS) $(ITEST_FLAGS)
 	$(COLLECT_ITEST_COVERAGE)
 
+#? itest-parallel: Build and run integration tests in parallel mode, running up to ITEST_PARALLELISM test tranches in parallel (default 4)
+itest-parallel: build-itest itest-only-parallel
+
 #? itest-clean: Kill all running itest processes
 itest-clean:
 	@$(call print, "Cleaning old itest processes")
@@ -300,6 +303,13 @@ unit-race: $(BTCD_BIN)
 	@$(call print, "Running unit race tests.")
 	env CGO_ENABLED=1 GORACE="history_size=7 halt_on_errors=1" $(UNIT_RACE)
 
+#? unit-race-parallel: Run one tranche of the unit tests in race detector mode (tranche=<index> tranches=<total>)
+unit-race-parallel: $(BTCD_BIN)
+	@$(call print, "Running unit race tests tranche ${tranche} of ${tranches}.")
+	PKG="$(PKG)" DEV_TAGS="$(DEV_TAGS)" \
+		scripts/unit_race_part.sh $(tranche) $(tranches) \
+		-tags="$(DEV_TAGS) $(RPC_TAGS) $(LOG_TAGS)" $(TEST_FLAGS)
+
 #? unit-bench: Run benchmark tests
 unit-bench: $(BTCD_BIN)
 	@$(call print, "Running benchmark tests.")

make/testing_flags.mk

@@ -10,6 +10,7 @@ COVER_PKG = $$($(GOCC) list -deps -tags="$(DEV_TAGS)" ./... | grep '$(PKG)')
 COVER_FLAGS = -coverprofile=coverage.txt -covermode=atomic -coverpkg=$(PKG)/...
 NUM_ITEST_TRANCHES = 4
 ITEST_PARALLELISM = $(NUM_ITEST_TRANCHES)
+ITEST_TRANCHE_OFFSET = 0
 POSTGRES_START_DELAY = 5
 SHUFFLE_SEED = 0
 
@@ -29,6 +30,12 @@ ifneq ($(parallel),)
 ITEST_PARALLELISM = $(parallel)
 endif
 
+# Run only a subset of the total tranches, starting at this offset. This
+# allows a single itest run to be spread over multiple machines.
+ifneq ($(trancheoffset),)
+ITEST_TRANCHE_OFFSET = $(trancheoffset)
+endif
+
 # Set the seed for shuffling the test cases.
 ifneq ($(shuffleseed),)
 SHUFFLE_SEED = $(shuffleseed)

scripts/itest_parallel.sh

@@ -8,12 +8,16 @@ SHUFFLE_SEED=$3
 # Here we also shift 3 times and get the rest of our flags to pass on in $@.
 shift 3
 
+# The tranche offset allows running only a subset of the total tranches,
+# spreading a single itest run over multiple machines.
+OFFSET=${ITEST_TRANCHE_OFFSET:-0}
+
 # Create a variable to hold the final exit code.
 exit_code=0
 
 # Run commands using xargs in parallel and capture their PIDs
 pids=()
-for ((i=0; i<PROCESSES; i++)); do 
+for ((i=OFFSET; i<OFFSET+PROCESSES; i++)); do
     scripts/itest_part.sh $i $TRANCHES $SHUFFLE_SEED $@ &
     pids+=($!)
 done