diff --git a/.github/workflows/lntest.yml b/.github/workflows/lntest.yml new file mode 100644 index 00000000000..00547639050 --- /dev/null +++ b/.github/workflows/lntest.yml @@ -0,0 +1,79 @@ +name: lntest + +# The bitcoind-miner itest configuration exercises the lntest +# harness's bitcoind miner support rather than any production lnd +# code, so it only runs when the harness itself (or this workflow) +# changes. +on: + push: + branches: + - "master" + paths: + - "lntest/**" + pull_request: + branches: + - "*" + paths: + - "lntest/**" + - ".github/workflows/lntest.yml" + +permissions: + contents: read + +concurrency: + group: lntest-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + # Accepts either a major image tag like "30" or a patch tag like "29.1". + BITCOIN_VERSION: "31" + + # TRANCHES defines the number of tranches used in the itests. + TRANCHES: 16 + + # If you change this please also update GO_VERSION in Makefile (then run + # `make lint` to see where else it needs to be updated as well). + GO_VERSION: 1.26.4 + +jobs: + bitcoind-miner-itest: + name: Run bitcoind-miner itests + runs-on: ubuntu-latest + steps: + - name: Git checkout + uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Fetch and rebase on ${{ github.base_ref }} + if: github.event_name == 'pull_request' + uses: ./.github/actions/rebase + + - name: Setup go ${{ env.GO_VERSION }} + uses: ./.github/actions/setup-go + with: + go-version: '${{ env.GO_VERSION }}' + key-prefix: integration-test + + - name: Install bitcoind + run: ./scripts/install_bitcoind.sh $BITCOIN_VERSION + + - name: Run bitcoind-miner itests + run: make itest-parallel tranches=${{ env.TRANCHES }} backend=bitcoind minerbackend=bitcoind shuffleseed=${{ github.run_id }} + + - name: Zip log files on failure + if: ${{ failure() }} + timeout-minutes: 5 # timeout after 5 minute + run: 7z a logs-itest-bitcoind-miner.zip itest/**/*.log + + - name: Upload log files on failure + uses: actions/upload-artifact@v4 + if: ${{ failure() }} + with: + name: logs-itest-bitcoind-miner + path: logs-itest-bitcoind-miner.zip + retention-days: 5 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 85553fca2a6..45e448955df 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,8 +12,6 @@ on: - "master" permissions: - # Required to manage and delete caches. - actions: write # Default permission for checking out code. contents: read @@ -39,10 +37,32 @@ env: # TODO(yy): remove this value and use TRANCHES. SMALL_TRANCHES: 8 + # SPLIT_TRANCHES defines the total number of tranches used by the + # tranche-split itest configurations, spread over + # SPLIT_TRANCHES / SPLIT_ITEST_PARALLELISM runner jobs. + SPLIT_TRANCHES: 16 + + # SPLIT_ITEST_PARALLELISM defines the number of tranches each runner job + # of a tranche-split itest configuration runs in parallel. + SPLIT_ITEST_PARALLELISM: 4 + # If you change this please also update GO_VERSION in Makefile (then run # `make lint` to see where else it needs to be updated as well). GO_VERSION: 1.26.4 + # DB_UNIT_PKGS is the list of packages whose unit tests exercise the + # configurable database backends, i.e. the packages whose tests behave + # differently under the kvdb_* and test_db_* build tags (they construct + # test stores via kvdb.GetTestBackend, channeldb.MakeTestDB, the sqldb + # test constructors or their wrappers). The DB-variant unit jobs run + # only these packages; all other packages run identically in the + # plain unit jobs. + DB_UNIT_PKGS: >- + autopilot batch chainparams channeldb cluster contractcourt graph + graph/db graph/db/migration1 invoices kvdb kvdb/etcd kvdb/postgres + kvdb/sqlbase kvdb/sqlite payments/db payments/db/migration1 routing + sqldb sweep + jobs: static-checks: name: Static Checks @@ -54,9 +74,6 @@ jobs: # Needed for some checks. fetch-depth: 0 - - name: Clean up runner space - uses: ./.github/actions/cleanup-space - - name: Setup Go ${{ env.GO_VERSION }} uses: ./.github/actions/setup-go with: @@ -107,6 +124,35 @@ jobs: - name: Check mobile specific code run: go build --tags="mobile" ./mobile + ######################## + # check pinned dependencies + ######################## + - name: Check pinned dependencies + run: | + while read -r dep; do + if ! grep -q "$dep" go.mod; then + echo "dependency $dep should not be altered" + exit 1 + fi + done < coverage-norpc.txt @@ -442,11 +500,24 @@ jobs: ######################## - # run windows integration test + # build binaries for itest jobs that run prebuilt binaries ######################## - windows-integration-test: - name: Run windows itest - runs-on: windows-latest + build-split-itest: + name: Build itest binaries + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - name: bitcoind-postgres-experiment + args: backend=bitcoind dbbackend=postgres tags=test_native_sql + # The windows binaries are cross-compiled here since building + # them on the (much slower) windows runner dominated that + # job's running time. All itest binaries are built with + # CGO_ENABLED=0, so cross-compilation is safe. + - name: windows + args: windows=1 + goos: windows steps: - name: Git checkout uses: actions/checkout@v5 @@ -471,10 +542,132 @@ jobs: go-version: '${{ env.GO_VERSION }}' key-prefix: integration-test - - name: Run itest + - name: Build itest binaries for ${{ matrix.name }} if: steps.check-label.outputs.skip != 'true' - run: make itest-parallel tranches=${{ env.SMALL_TRANCHES }} windows=1 shuffleseed=${{ github.run_id }} - + env: + GOOS: ${{ matrix.goos }} + run: make build-itest ${{ matrix.args }} + + - name: Upload itest binaries + if: steps.check-label.outputs.skip != 'true' + uses: actions/upload-artifact@v4 + with: + name: itest-binaries-${{ matrix.name }} + path: | + itest/itest.test* + itest/lnd-itest* + itest/btcd-itest* + retention-days: 1 + + ######################## + # run tranche-split integration tests + ######################## + split-integration-test: + name: Split itests (${{ matrix.config.name }}, ${{ matrix.tranche_group }}) + runs-on: ubuntu-latest + needs: build-split-itest + strategy: + # Allow other tests in the matrix to continue if one fails. + fail-fast: false + matrix: + # Each tranche group runs SPLIT_ITEST_PARALLELISM tranches, so + # this list must contain SPLIT_TRANCHES / SPLIT_ITEST_PARALLELISM + # entries. + tranche_group: [0, 1, 2, 3] + config: + - name: bitcoind-postgres-nativesql-experiment + binaries: bitcoind-postgres-experiment + seed: 3 + args: backend=bitcoind dbbackend=postgres nativesql=true tags=test_native_sql + steps: + - name: Git checkout + uses: actions/checkout@v5 + + - name: Check for no-itest label + id: check-label + uses: ./.github/actions/check-label + with: + label: 'no-itest' + skip-message: "Tests auto-passed due to 'no-itest' label" + + - name: Download itest binaries + if: steps.check-label.outputs.skip != 'true' + uses: actions/download-artifact@v4 + with: + name: itest-binaries-${{ matrix.config.binaries }} + path: itest + + - name: Make itest binaries executable + if: steps.check-label.outputs.skip != 'true' + run: chmod +x itest/itest.test itest/lnd-itest itest/btcd-itest + + - name: Install bitcoind + if: steps.check-label.outputs.skip != 'true' + run: ./scripts/install_bitcoind.sh $BITCOIN_VERSION + + - name: Run ${{ matrix.config.name }} tranche group ${{ matrix.tranche_group }} + if: steps.check-label.outputs.skip != 'true' + run: | + OFFSET=$((${{ matrix.tranche_group }} * ${{ env.SPLIT_ITEST_PARALLELISM }})) + make itest-only-parallel tranches=${{ env.SPLIT_TRANCHES }} \ + parallel=${{ env.SPLIT_ITEST_PARALLELISM }} trancheoffset=$OFFSET \ + ${{ matrix.config.args }} \ + shuffleseed=${{ github.run_id }}${{ matrix.config.seed }} + + - name: Zip log files on failure + if: ${{ failure() && steps.check-label.outputs.skip != 'true' }} + timeout-minutes: 5 # timeout after 5 minute + run: 7z a logs-itest-${{ matrix.config.name }}-${{ matrix.tranche_group }}.zip itest/**/*.log + + - name: Upload log files on failure + uses: actions/upload-artifact@v4 + if: ${{ failure() && steps.check-label.outputs.skip != 'true' }} + with: + name: logs-itest-${{ matrix.config.name }}-${{ matrix.tranche_group }} + path: logs-itest-${{ matrix.config.name }}-${{ matrix.tranche_group }}.zip + retention-days: 5 + + ######################## + # run windows integration test + ######################## + windows-integration-test: + name: Run windows itest (${{ matrix.tranche_group }}) + runs-on: windows-latest + needs: build-split-itest + strategy: + # Allow the other group to continue if one fails. + fail-fast: false + matrix: + # The windows runner is machine-bound rather than wait-bound, + # so the suite is spread over two runners with half of the + # TRANCHES (16) tranches each. + tranche_group: [0, 1] + steps: + - name: Git checkout + uses: actions/checkout@v5 + + - name: Check for no-itest label + id: check-label + uses: ./.github/actions/check-label + with: + label: 'no-itest' + skip-message: "Tests auto-passed due to 'no-itest' label" + + - name: Download itest binaries + if: steps.check-label.outputs.skip != 'true' + uses: actions/download-artifact@v4 + with: + name: itest-binaries-windows + path: itest + + - name: Run itest tranche group ${{ matrix.tranche_group }} + if: steps.check-label.outputs.skip != 'true' + run: | + OFFSET=$((${{ matrix.tranche_group }} * 8)) + make itest-only-parallel tranches=${{ env.TRANCHES }} \ + parallel=8 trancheoffset=$OFFSET windows=1 \ + shuffleseed=${{ github.run_id }} + - name: Kill any remaining lnd processes if: ${{ failure() && steps.check-label.outputs.skip != 'true' }} shell: powershell @@ -483,14 +676,14 @@ jobs: - name: Zip log files on failure if: ${{ failure() && steps.check-label.outputs.skip != 'true' }} timeout-minutes: 5 # timeout after 5 minute - run: 7z a logs-itest-windows.zip itest/**/*.log + run: 7z a logs-itest-windows-${{ matrix.tranche_group }}.zip itest/**/*.log - name: Upload log files on failure uses: actions/upload-artifact@v4 if: ${{ failure() && steps.check-label.outputs.skip != 'true' }} with: - name: logs-itest-windows - path: logs-itest-windows.zip + name: logs-itest-windows-${{ matrix.tranche_group }} + path: logs-itest-windows-${{ matrix.tranche_group }}.zip retention-days: 5 ######################## @@ -540,55 +733,6 @@ jobs: path: logs-itest-macos.zip retention-days: 5 - ######################## - # check pinned dependencies - ######################## - dep-pin: - name: Check pinned dependencies - runs-on: ubuntu-latest - strategy: - # Allow other tests in the matrix to continue if one fails. - fail-fast: false - matrix: - pinned_dep: - - google.golang.org/grpc v1.79.3 - - github.com/golang/protobuf v1.5.4 - - steps: - - name: Git checkout - uses: actions/checkout@v5 - - - name: Clean up runner space - uses: ./.github/actions/cleanup-space - - - name: Ensure dependencies at correct version - run: if ! grep -q "${{ matrix.pinned_dep }}" go.mod; then echo dependency ${{ matrix.pinned_dep }} should not be altered ; exit 1 ; fi - - ######################## - # check PR updates release notes - ######################## - milestone-check: - name: Check release notes updated - runs-on: ubuntu-latest - steps: - - name: Git checkout - uses: actions/checkout@v5 - - - name: Check for no-changelog label - id: check-label - uses: ./.github/actions/check-label - with: - label: 'no-changelog' - skip-message: "Changelog check auto-passed due to 'no-changelog' label" - - - name: Clean up runner space - if: steps.check-label.outputs.skip != 'true' - uses: ./.github/actions/cleanup-space - - - name: Release notes check - if: steps.check-label.outputs.skip != 'true' - run: scripts/check-release-notes.sh - ######################## # Backwards Compatibility Test ######################## @@ -613,46 +757,6 @@ jobs: path: scripts/bw-compatibility-test/logs/ retention-days: 7 - ######################################### - # Auto Cache Cleanup on Pull Requests - ######################################### - auto-cleanup-cache: - name: Cache Cleanup - runs-on: ubuntu-latest - - # This condition checks for pull requests from authors with write access. - if: >- - contains('OWNER, MEMBER, COLLABORATOR', github.event.pull_request.author_association) - - steps: - - name: Checkout repository - uses: actions/checkout@v5 - - - name: Delete caches older than 12 hours - continue-on-error: true - env: - # GITHUB_TOKEN is required for the gh CLI. - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - echo "Finding caches not used in the last 12 hours..." - - # Get the current time and the cutoff time (12 hours ago) in Unix - # timestamp format. - cutoff_timestamp=$(date -d "12 hours ago" +%s) - - # Use gh and jq to parse caches. Delete any cache last accessed - # before the cutoff time. - gh cache list --json key,lastAccessedAt | jq -r '.[] | - select(.lastAccessedAt != null) | "\(.lastAccessedAt) \(.key)"' | - while read -r last_accessed_at key; do - last_accessed_timestamp=$(date -d "$last_accessed_at" +%s) - - if (( last_accessed_timestamp < cutoff_timestamp )); then - echo "Deleting old cache. Key: $key, Last Used: $last_accessed_at" - gh cache delete "$key" - fi - done - # Notify about the completion of all coverage collecting jobs. finish: name: Send coverage report diff --git a/Makefile b/Makefile index 91213847e84..fe1c55c7edd 100644 --- a/Makefile +++ b/Makefile @@ -263,13 +263,16 @@ itest: build-itest itest-only #? itest-race: Build and run integration tests in race detector mode itest-race: build-itest-race itest-only -#? itest-parallel: Build and run integration tests in parallel mode, running up to ITEST_PARALLELISM test tranches in parallel (default 4) -itest-parallel: clean-itest-logs build-itest db-instance +#? itest-only-parallel: Run integration tests in parallel mode without re-building binaries, running up to ITEST_PARALLELISM test tranches in parallel starting at the tranche given by trancheoffset (default 0) +itest-only-parallel: clean-itest-logs db-instance @$(call print, "Running tests") date - EXEC_SUFFIX=$(EXEC_SUFFIX) scripts/itest_parallel.sh $(ITEST_PARALLELISM) $(NUM_ITEST_TRANCHES) $(SHUFFLE_SEED) $(TEST_FLAGS) $(ITEST_FLAGS) + EXEC_SUFFIX=$(EXEC_SUFFIX) ITEST_TRANCHE_OFFSET=$(ITEST_TRANCHE_OFFSET) scripts/itest_parallel.sh $(ITEST_PARALLELISM) $(NUM_ITEST_TRANCHES) $(SHUFFLE_SEED) $(TEST_FLAGS) $(ITEST_FLAGS) $(COLLECT_ITEST_COVERAGE) +#? itest-parallel: Build and run integration tests in parallel mode, running up to ITEST_PARALLELISM test tranches in parallel (default 4) +itest-parallel: build-itest itest-only-parallel + #? itest-clean: Kill all running itest processes itest-clean: @$(call print, "Cleaning old itest processes") @@ -300,6 +303,13 @@ unit-race: $(BTCD_BIN) @$(call print, "Running unit race tests.") env CGO_ENABLED=1 GORACE="history_size=7 halt_on_errors=1" $(UNIT_RACE) +#? unit-race-parallel: Run one tranche of the unit tests in race detector mode (tranche= tranches=) +unit-race-parallel: $(BTCD_BIN) + @$(call print, "Running unit race tests tranche ${tranche} of ${tranches}.") + PKG="$(PKG)" DEV_TAGS="$(DEV_TAGS)" UNIT_RACE_PKGS="$(pkg)" \ + scripts/unit_race_part.sh $(tranche) $(tranches) \ + -tags="$(DEV_TAGS) $(RPC_TAGS) $(LOG_TAGS)" $(TEST_FLAGS) + #? unit-bench: Run benchmark tests unit-bench: $(BTCD_BIN) @$(call print, "Running benchmark tests.") diff --git a/chainntnfs/test/bitcoind/bitcoind_test.go b/chainntnfs/test/bitcoind/bitcoind_test.go index b5a2e6626c7..6c27e1a6566 100644 --- a/chainntnfs/test/bitcoind/bitcoind_test.go +++ b/chainntnfs/test/bitcoind/bitcoind_test.go @@ -21,8 +21,20 @@ func TestInterfaces(t *testing.T) { return } - t.Run("bitcoind rpc polling", func(st *testing.T) { + success = t.Run("bitcoind rpc polling", func(st *testing.T) { st.Parallel() chainntnfstest.TestInterfaces(st, "bitcoind-rpc-polling") }) + + if !success { + return + } + + // Run the suite against a bitcoind backend without a transaction + // index, which forces the notifier through its manual historical + // confirmation and spend lookup fallbacks. + t.Run("bitcoind no txindex", func(st *testing.T) { + st.Parallel() + chainntnfstest.TestInterfaces(st, "bitcoind-no-txindex") + }) } diff --git a/chainntnfs/test/test_interface.go b/chainntnfs/test/test_interface.go index 7536d24c503..3dfd5b06801 100644 --- a/chainntnfs/test/test_interface.go +++ b/chainntnfs/test/test_interface.go @@ -1902,9 +1902,20 @@ func TestInterfaces(t *testing.T, targetBackEnd string) { log.Printf("Running %v ChainNotifier interface tests", 2*len(txNtfnTests)+len(blockNtfnTests)+len(blockCatchupTests)) + // The bitcoind-no-txindex variant runs against the same registered + // bitcoind driver as the plain bitcoind variant, but with the + // backend's transaction index disabled. This exercises the manual + // historical confirmation/spend lookup fallbacks in the notifier. + driverType := targetBackEnd + txindex := true + if targetBackEnd == "bitcoind-no-txindex" { + driverType = "bitcoind" + txindex = false + } + for _, notifierDriver := range chainntnfs.RegisteredNotifiers() { notifierType := notifierDriver.NotifierType - if notifierType != targetBackEnd { + if notifierType != driverType { continue } @@ -1928,11 +1939,11 @@ func TestInterfaces(t *testing.T, targetBackEnd string) { newNotifier func() (chainntnfs.TestChainNotifier, error) ) - switch notifierType { - case "bitcoind": + switch targetBackEnd { + case "bitcoind", "bitcoind-no-txindex": var bitcoindConn *chain.BitcoindConn bitcoindConn = unittest.NewBitcoindBackend( - t, unittest.NetParams, miner, true, false, + t, unittest.NetParams, miner, txindex, false, ) newNotifier = func() (chainntnfs.TestChainNotifier, error) { return bitcoindnotify.New( @@ -1976,21 +1987,21 @@ func TestInterfaces(t *testing.T, targetBackEnd string) { } log.Printf("Running ChainNotifier interface tests for: %v", - notifierType) + targetBackEnd) notifier, err := newNotifier() if err != nil { t.Fatalf("unable to create %v notifier: %v", - notifierType, err) + targetBackEnd, err) } if err := notifier.Start(); err != nil { t.Fatalf("unable to start notifier %v: %v", - notifierType, err) + targetBackEnd, err) } for _, txNtfnTest := range txNtfnTests { for _, scriptDispatch := range []bool{false, true} { - testName := fmt.Sprintf("%v %v", notifierType, + testName := fmt.Sprintf("%v %v", targetBackEnd, txNtfnTest.name) if scriptDispatch { testName += " with script dispatch" @@ -2008,7 +2019,7 @@ func TestInterfaces(t *testing.T, targetBackEnd string) { } for _, blockNtfnTest := range blockNtfnTests { - testName := fmt.Sprintf("%v %v", notifierType, + testName := fmt.Sprintf("%v %v", targetBackEnd, blockNtfnTest.name) success := t.Run(testName, func(t *testing.T) { blockNtfnTest.test(miner, notifier, t) @@ -2026,10 +2037,10 @@ func TestInterfaces(t *testing.T, targetBackEnd string) { notifier, err = newNotifier() if err != nil { t.Fatalf("unable to create %v notifier: %v", - notifierType, err) + targetBackEnd, err) } - testName := fmt.Sprintf("%v %v", notifierType, + testName := fmt.Sprintf("%v %v", targetBackEnd, blockCatchupTest.name) success := t.Run(testName, func(t *testing.T) { diff --git a/make/testing_flags.mk b/make/testing_flags.mk index 4f7bd42d568..86691a91561 100644 --- a/make/testing_flags.mk +++ b/make/testing_flags.mk @@ -10,6 +10,7 @@ COVER_PKG = $$($(GOCC) list -deps -tags="$(DEV_TAGS)" ./... | grep '$(PKG)') COVER_FLAGS = -coverprofile=coverage.txt -covermode=atomic -coverpkg=$(PKG)/... NUM_ITEST_TRANCHES = 4 ITEST_PARALLELISM = $(NUM_ITEST_TRANCHES) +ITEST_TRANCHE_OFFSET = 0 POSTGRES_START_DELAY = 5 SHUFFLE_SEED = 0 @@ -29,6 +30,12 @@ ifneq ($(parallel),) ITEST_PARALLELISM = $(parallel) endif +# Run only a subset of the total tranches, starting at this offset. This +# allows a single itest run to be spread over multiple machines. +ifneq ($(trancheoffset),) +ITEST_TRANCHE_OFFSET = $(trancheoffset) +endif + # Set the seed for shuffling the test cases. ifneq ($(shuffleseed),) SHUFFLE_SEED = $(shuffleseed) @@ -45,12 +52,21 @@ ifneq ($(windows),) EXEC_SUFFIX = .exe endif -# If specific package is being unit tested, construct the full name of the -# subpackage. +# If specific packages are being unit tested, construct the full names of +# the subpackages. ifneq ($(pkg),) -UNITPKG := $(PKG)/$(pkg) +UNITPKG := $(addprefix $(PKG)/,$(pkg)) UNIT_TARGETED = yes -COVER_PKG = $(PKG)/$(pkg) +COVER_PKG = $(addprefix $(PKG)/,$(pkg)) + +# When more than one package is targeted, run their test binaries +# serially. The untargeted path serializes packages via xargs -L 1, and +# some test fixtures bind fixed ports (e.g. the kvdb postgres fixture's +# embedded postgres on port 9876), so package binaries collide when go +# test runs them in parallel. +ifneq ($(word 2,$(pkg)),) +UNIT_PKG_FLAGS = -p 1 +endif endif # If a specific unit test case is being target, construct test.run filter. @@ -141,9 +157,9 @@ UNIT_TARGETED ?= no # If a specific package/test case was requested, run the unit test for the # targeted case. Otherwise, default to running all tests. ifeq ($(UNIT_TARGETED), yes) -UNIT := $(GOTEST) -tags="$(DEV_TAGS) $(RPC_TAGS) $(LOG_TAGS)" $(TEST_FLAGS) $(UNITPKG) +UNIT := $(GOTEST) $(UNIT_PKG_FLAGS) -tags="$(DEV_TAGS) $(RPC_TAGS) $(LOG_TAGS)" $(TEST_FLAGS) $(UNITPKG) UNIT_DEBUG := $(GOTEST) -v -tags="$(DEV_TAGS) $(RPC_TAGS) $(LOG_TAGS)" $(TEST_FLAGS) $(UNITPKG) -UNIT_RACE := $(GOTEST) -tags="$(DEV_TAGS) $(RPC_TAGS) $(LOG_TAGS) lowscrypt" $(TEST_FLAGS) -race $(UNITPKG) +UNIT_RACE := $(GOTEST) $(UNIT_PKG_FLAGS) -tags="$(DEV_TAGS) $(RPC_TAGS) $(LOG_TAGS) lowscrypt" $(TEST_FLAGS) -race $(UNITPKG) # NONE is a special value which selects no other tests but only executes the benchmark tests here. UNIT_BENCH := $(GOTEST) -tags="$(DEV_TAGS) $(LOG_TAGS)" -test.bench=. -test.run=NONE $(UNITPKG) endif diff --git a/scripts/itest_parallel.sh b/scripts/itest_parallel.sh index ab1b3efa282..e430c745cf5 100755 --- a/scripts/itest_parallel.sh +++ b/scripts/itest_parallel.sh @@ -8,12 +8,16 @@ SHUFFLE_SEED=$3 # Here we also shift 3 times and get the rest of our flags to pass on in $@. shift 3 +# The tranche offset allows running only a subset of the total tranches, +# spreading a single itest run over multiple machines. +OFFSET=${ITEST_TRANCHE_OFFSET:-0} + # Create a variable to hold the final exit code. exit_code=0 # Run commands using xargs in parallel and capture their PIDs pids=() -for ((i=0; i [go test flags...]" >&2 + exit 1 +fi + +if ! [[ "${TRANCHE}" =~ ^[0-9]+$ && "${NUM_TRANCHES}" =~ ^[0-9]+$ ]]; then + echo "tranche and num_tranches must be non-negative integers" >&2 + exit 1 +fi + +if (( NUM_TRANCHES <= 0 )); then + echo "num_tranches must be greater than 0" >&2 + exit 1 +fi + +if (( TRANCHE < 0 || TRANCHE >= NUM_TRANCHES )); then + echo "tranche must be in range [0, num_tranches)" >&2 + exit 1 +fi + +shift 2 + +PKG_PREFIX=${PKG:-github.com/lightningnetwork/lnd} +DEV_TAGS=${DEV_TAGS:-dev} + +# pkg_weight echoes the approximate race-mode test duration of a +# package in seconds, measured from CI logs (the per-package "ok" +# lines of the unit race jobs). Unlisted packages get a small default +# weight. The exact values only matter relative to each other; refresh +# them occasionally if the tranches drift out of balance. +pkg_weight() { + case "${1#"${PKG_PREFIX}"/}" in + invoices) echo 691 ;; + lnwire) echo 328 ;; + internal/musig2v040) echo 246 ;; + lnwallet/test/neutrino) echo 171 ;; + lnwallet/test/bitcoind) echo 170 ;; + lnwallet/btcwallet) echo 103 ;; + htlcswitch) echo 84 ;; + lnwallet/test/btcd) echo 82 ;; + routing/chainview) echo 66 ;; + watchtower/wtclient) echo 54 ;; + chainntnfs/test/bitcoind) echo 46 ;; + lnwallet) echo 45 ;; + chainntnfs/test/btcd) echo 45 ;; + chainntnfs/test/neutrino) echo 41 ;; + contractcourt) echo 38 ;; + channeldb/migration30) echo 26 ;; + sqldb) echo 24 ;; + chainntnfs) echo 21 ;; + discovery) echo 21 ;; + funding) echo 17 ;; + *) echo 3 ;; + esac +} + +all_pkgs=() +if [[ -n "${UNIT_RACE_PKGS:-}" ]]; then + # An explicit package list (relative to PKG) restricts the run, e.g. + # to the database-touching packages. + for pkg in ${UNIT_RACE_PKGS}; do + all_pkgs+=("${PKG_PREFIX}/${pkg}") + done +else + # Capture the package list via command substitution so a go list + # failure aborts the script (set -e -o pipefail) instead of silently + # yielding an empty list and a vacuously green test run. + pkg_list=$(go list -tags="${DEV_TAGS}" -deps "${PKG_PREFIX}/..." | \ + grep "${PKG_PREFIX}" | grep -v "/vendor/") + + while IFS= read -r pkg; do + all_pkgs+=("${pkg}") + done <<< "${pkg_list}" + + if (( ${#all_pkgs[@]} == 0 )); then + echo "go list produced no packages" >&2 + exit 1 + fi +fi + +# Sort packages by descending weight (ties broken by name for +# determinism across tranche jobs) and greedily assign each to the +# currently lightest tranche. Every tranche job computes the same +# assignment and runs only its own bucket. +weighted=$(for pkg in "${all_pkgs[@]}"; do + echo "$(pkg_weight "${pkg}") ${pkg}" +done | LC_ALL=C sort -k1,1rn -k2,2) + +sums=() +for ((i=0; i&2 + exit 0 +fi + +echo "Tranche ${TRANCHE} of ${NUM_TRANCHES}:" \ + "${#selected[@]} packages, estimated weight ${sums[TRANCHE]}s" + +exit_code=0 +for pkg in "${selected[@]}"; do + echo "Running race unit tests for ${pkg}" + if ! env CGO_ENABLED=1 GORACE="history_size=7 halt_on_errors=1" \ + go test -race "$@" "${pkg}"; then + exit_code=1 + fi +done + +if (( exit_code != 0 )); then + echo "One or more packages failed in tranche ${TRANCHE} of" \ + "${NUM_TRANCHES}" >&2 +fi + +exit ${exit_code}