feat: [OCISDEV-793] reduce ci duration, parallelize jobs inside single runner

Author: mklos-kw

.github/workflows/acceptance-tests.yml

@@ -105,48 +105,30 @@ jobs:
       fail-fast: false
       matrix:
         suite:
-          # contract & locks
-          - apiContract
-          - apiLocks
-          # settings & notifications (needs email)
-          - apiSettings
-          - apiNotification
-          - apiCors
-          # graph
-          - apiGraphUser
-          - apiGraph
-          - apiGraphGroup
-          # spaces & dav
-          - apiSpaces
-          - apiSpacesShares
-          - apiSpacesDavOperation
-          - apiDownloads
-          - apiAsyncUpload
-          - apiDepthInfinity
-          - apiArchiver
-          - apiActivities
-          # search
-          - apiSearch1
-          - apiSearch2
-          - apiSearchContent    # needs Tika
-          # sharing
-          - apiSharingNgShares
-          - apiReshare
-          - apiSharingNgPermissions
-          - apiSharingNgAdditionalShareRole
-          - apiSharingNgDriveInvitation
-          - apiSharingNgItemInvitation
-          - apiSharingNgDriveLinkShare
-          - apiSharingNgItemLinkShare
-          - apiSharingNgLinkShareManagement
-          # auth
-          - apiAuthApp
-          # antivirus (needs ClamAV)
-          - apiAntivirus
-          # federation (needs email + federation ocis)
-          - apiOcm
-          # collaboration (needs WOPI)
-          - apiCollaboration
+          # contract & locks — 2 suites in parallel (contractAndLock group)
+          - "apiContract,apiLocks"
+          # settings, notifications & cors — 3 suites in parallel (needs email)
+          - "apiSettings,apiNotification,apiCors"
+          # graph + search-no-tika — 4 suites in parallel (graphUser + groupAndSearch1 groups)
+          - "apiGraphUser,apiSearch1,apiGraph,apiGraphGroup"
+          # spaces — 2 suites in parallel (spaces + spacesShares groups)
+          - "apiSpaces,apiSpacesShares"
+          # dav operations — 6 suites in parallel (davOperations group)
+          - "apiSpacesDavOperation,apiDownloads,apiAsyncUpload,apiDepthInfinity,apiArchiver,apiActivities"
+          # search with Tika — 2 suites in parallel (search2 group)
+          - "apiSearch2,apiSearchContent"
+          # sharing ng part 1 — 4 suites in parallel (sharingNg1 + sharingNgAdditionalShareRole groups)
+          - "apiSharingNgShares,apiReshare,apiSharingNgPermissions,apiSharingNgAdditionalShareRole"
+          # sharing ng part 2 — 5 suites in parallel (sharingNgShareInvitation + sharingNgLinkShare groups)
+          - "apiSharingNgDriveInvitation,apiSharingNgItemInvitation,apiSharingNgDriveLinkShare,apiSharingNgItemLinkShare,apiSharingNgLinkShareManagement"
+          # auth app — kept isolated (enables auth-app service + PROXY_ENABLE_APP_AUTH)
+          - "apiAuthApp"
+          # antivirus — kept isolated (needs ClamAV)
+          - "apiAntivirus"
+          # federation — kept isolated (needs email + federation ocis)
+          - "apiOcm"
+          # collaboration — kept isolated (needs WOPI)
+          - "apiCollaboration"
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
@@ -216,7 +198,16 @@ jobs:
           composer --version
 
       - name: Run ${{ matrix.suite }}
-        run: BEHAT_SUITES=${{ matrix.suite }} python3 tests/acceptance/run-github.py
+        run: |
+          vmstat 2 > /tmp/vmstat.log & MONITOR_PID=$!
+          BEHAT_SUITES="${{ matrix.suite }}" python3 tests/acceptance/run-github.py
+          EXIT=$?
+          kill $MONITOR_PID 2>/dev/null; wait $MONITOR_PID 2>/dev/null || true
+          awk '/^[ ]*[0-9]/ { busy=100-$15; sum_b+=busy; if(busy>pk_b)pk_b=busy;
+                               sum_r+=$1;   if($1>pk_r) pk_r=$1; sum_wa+=$16; n++ }
+               END { printf "=== load (2 vCPU): avg busy %d%% peak %d%% | avg runq %.0f peak %d | avg wa %d%%\n",
+                     sum_b/n, pk_b, sum_r/n, pk_r, sum_wa/n }' /tmp/vmstat.log
+          exit $EXIT
 
   cli-tests:
     needs: [build-and-test]
@@ -252,7 +243,16 @@ jobs:
           composer --version
 
       - name: Run ${{ matrix.suite }}
-        run: BEHAT_SUITES="${{ matrix.suite }}" python3 tests/acceptance/run-github.py
+        run: |
+          vmstat 2 > /tmp/vmstat.log & MONITOR_PID=$!
+          BEHAT_SUITES="${{ matrix.suite }}" python3 tests/acceptance/run-github.py
+          EXIT=$?
+          kill $MONITOR_PID 2>/dev/null; wait $MONITOR_PID 2>/dev/null || true
+          awk '/^[ ]*[0-9]/ { busy=100-$15; sum_b+=busy; if(busy>pk_b)pk_b=busy;
+                               sum_r+=$1;   if($1>pk_r) pk_r=$1; sum_wa+=$16; n++ }
+               END { printf "=== load (2 vCPU): avg busy %d%% peak %d%% | avg runq %.0f peak %d | avg wa %d%%\n",
+                     sum_b/n, pk_b, sum_r/n, pk_r, sum_wa/n }' /tmp/vmstat.log
+          exit $EXIT
 
   core-api-tests:
     name: ${{ matrix.suite }}
@@ -338,11 +338,17 @@ jobs:
           curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin --filename=composer
 
       - name: Run ${{ matrix.suite }}
-        run: >
-          BEHAT_SUITES="${{ matrix.suite }}"
-          ACCEPTANCE_TEST_TYPE=core-api
-          WITH_REMOTE_PHP=true
-          python3 tests/acceptance/run-github.py
+        run: |
+          vmstat 2 > /tmp/vmstat.log & MONITOR_PID=$!
+          BEHAT_SUITES="${{ matrix.suite }}" ACCEPTANCE_TEST_TYPE=core-api WITH_REMOTE_PHP=true \
+            python3 tests/acceptance/run-github.py
+          EXIT=$?
+          kill $MONITOR_PID 2>/dev/null; wait $MONITOR_PID 2>/dev/null || true
+          awk '/^[ ]*[0-9]/ { busy=100-$15; sum_b+=busy; if(busy>pk_b)pk_b=busy;
+                               sum_r+=$1;   if($1>pk_r) pk_r=$1; sum_wa+=$16; n++ }
+               END { printf "=== load (2 vCPU): avg busy %d%% peak %d%% | avg runq %.0f peak %d | avg wa %d%%\n",
+                     sum_b/n, pk_b, sum_r/n, pk_r, sum_wa/n }' /tmp/vmstat.log
+          exit $EXIT
 
   e2e-tests:
     name: e2e-${{ matrix.suite }}
@@ -449,7 +455,16 @@ jobs:
           echo "keycloak ready."
 
       - name: Run e2e-${{ matrix.suite }}
-        run: E2E_ARGS="${{ matrix.args }}" python3 tests/acceptance/run-e2e.py
+        run: |
+          vmstat 2 > /tmp/vmstat.log & MONITOR_PID=$!
+          E2E_ARGS="${{ matrix.args }}" python3 tests/acceptance/run-e2e.py
+          EXIT=$?
+          kill $MONITOR_PID 2>/dev/null; wait $MONITOR_PID 2>/dev/null || true
+          awk '/^[ ]*[0-9]/ { busy=100-$15; sum_b+=busy; if(busy>pk_b)pk_b=busy;
+                               sum_r+=$1;   if($1>pk_r) pk_r=$1; sum_wa+=$16; n++ }
+               END { printf "=== load (2 vCPU): avg busy %d%% peak %d%% | avg runq %.0f peak %d | avg wa %d%%\n",
+                     sum_b/n, pk_b, sum_r/n, pk_r, sum_wa/n }' /tmp/vmstat.log
+          exit $EXIT
         env:
           TIKA_NEEDED: ${{ matrix.tika == true && 'true' || 'false' }}
           KEYCLOAK_NEEDED: ${{ matrix.keycloak == true && 'true' || 'false' }}
@@ -465,7 +480,16 @@ jobs:
           go-version-file: go.mod
           cache: true
       - name: Run litmus
-        run: python3 tests/acceptance/run-litmus.py
+        run: |
+          vmstat 2 > /tmp/vmstat.log & MONITOR_PID=$!
+          python3 tests/acceptance/run-litmus.py
+          EXIT=$?
+          kill $MONITOR_PID 2>/dev/null; wait $MONITOR_PID 2>/dev/null || true
+          awk '/^[ ]*[0-9]/ { busy=100-$15; sum_b+=busy; if(busy>pk_b)pk_b=busy;
+                               sum_r+=$1;   if($1>pk_r) pk_r=$1; sum_wa+=$16; n++ }
+               END { printf "=== load (2 vCPU): avg busy %d%% peak %d%% | avg runq %.0f peak %d | avg wa %d%%\n",
+                     sum_b/n, pk_b, sum_r/n, pk_r, sum_wa/n }' /tmp/vmstat.log
+          exit $EXIT
 
   cs3api:
     name: cs3api
@@ -478,7 +502,16 @@ jobs:
           go-version-file: go.mod
           cache: true
       - name: Run cs3api validator
-        run: python3 tests/acceptance/run-cs3api.py
+        run: |
+          vmstat 2 > /tmp/vmstat.log & MONITOR_PID=$!
+          python3 tests/acceptance/run-cs3api.py
+          EXIT=$?
+          kill $MONITOR_PID 2>/dev/null; wait $MONITOR_PID 2>/dev/null || true
+          awk '/^[ ]*[0-9]/ { busy=100-$15; sum_b+=busy; if(busy>pk_b)pk_b=busy;
+                               sum_r+=$1;   if($1>pk_r) pk_r=$1; sum_wa+=$16; n++ }
+               END { printf "=== load (2 vCPU): avg busy %d%% peak %d%% | avg runq %.0f peak %d | avg wa %d%%\n",
+                     sum_b/n, pk_b, sum_r/n, pk_r, sum_wa/n }' /tmp/vmstat.log
+          exit $EXIT
 
   wopi-builtin:
     name: wopi-builtin
@@ -491,7 +524,16 @@ jobs:
           go-version-file: go.mod
           cache: true
       - name: Run WOPI validator (builtin)
-        run: python3 tests/acceptance/run-wopi.py --type builtin
+        run: |
+          vmstat 2 > /tmp/vmstat.log & MONITOR_PID=$!
+          python3 tests/acceptance/run-wopi.py --type builtin
+          EXIT=$?
+          kill $MONITOR_PID 2>/dev/null; wait $MONITOR_PID 2>/dev/null || true
+          awk '/^[ ]*[0-9]/ { busy=100-$15; sum_b+=busy; if(busy>pk_b)pk_b=busy;
+                               sum_r+=$1;   if($1>pk_r) pk_r=$1; sum_wa+=$16; n++ }
+               END { printf "=== load (2 vCPU): avg busy %d%% peak %d%% | avg runq %.0f peak %d | avg wa %d%%\n",
+                     sum_b/n, pk_b, sum_r/n, pk_r, sum_wa/n }' /tmp/vmstat.log
+          exit $EXIT
 
   wopi-cs3:
     name: wopi-cs3
@@ -504,7 +546,16 @@ jobs:
           go-version-file: go.mod
           cache: true
       - name: Run WOPI validator (cs3)
-        run: python3 tests/acceptance/run-wopi.py --type cs3
+        run: |
+          vmstat 2 > /tmp/vmstat.log & MONITOR_PID=$!
+          python3 tests/acceptance/run-wopi.py --type cs3
+          EXIT=$?
+          kill $MONITOR_PID 2>/dev/null; wait $MONITOR_PID 2>/dev/null || true
+          awk '/^[ ]*[0-9]/ { busy=100-$15; sum_b+=busy; if(busy>pk_b)pk_b=busy;
+                               sum_r+=$1;   if($1>pk_r) pk_r=$1; sum_wa+=$16; n++ }
+               END { printf "=== load (2 vCPU): avg busy %d%% peak %d%% | avg runq %.0f peak %d | avg wa %d%%\n",
+                     sum_b/n, pk_b, sum_r/n, pk_r, sum_wa/n }' /tmp/vmstat.log
+          exit $EXIT
 
   all-acceptance-tests:
     needs: [local-api-tests, cli-tests, core-api-tests, litmus, cs3api, wopi-builtin, wopi-cs3, e2e-tests]

tests/acceptance/run-github.py

@@ -694,12 +694,42 @@ def cleanup(*_):
         }
         behat_env.update(cfg["extraEnvironment"])
 
-        print(f"Running suites: {behat_suites_raw} (type: {acceptance_test_type})")
-        result = subprocess.run(
-            ["make", "-C", str(repo_root), "test-acceptance-api"],
-            env=behat_env,
-        )
-        return result.returncode
+        print(f"Running suites: {suites} (type: {acceptance_test_type})")
+
+        if len(suites) == 1:
+            # single suite: stream output directly
+            result = subprocess.run(
+                ["make", "-C", str(repo_root), "test-acceptance-api"],
+                env=behat_env,
+            )
+            return result.returncode
+
+        # multiple suites: run in parallel against the shared OCIS instance,
+        # one behat process per suite, output captured and printed serially on completion.
+        import concurrent.futures
+
+        def run_suite(suite: str) -> tuple:
+            env = {**behat_env, "BEHAT_SUITES": suite}
+            proc = subprocess.run(
+                ["make", "-C", str(repo_root), "test-acceptance-api"],
+                env=env, capture_output=True, text=True,
+            )
+            return suite, proc.returncode, proc.stdout + proc.stderr
+
+        results = []
+        with concurrent.futures.ThreadPoolExecutor(max_workers=len(suites)) as pool:
+            fut_to_suite = {pool.submit(run_suite, s): s for s in suites}
+            for fut in concurrent.futures.as_completed(fut_to_suite):
+                suite, rc, output = fut.result()
+                print(f"\n{'=' * 60}\n=== suite: {suite} (exit {rc}) ===\n{'=' * 60}", flush=True)
+                print(output, flush=True)
+                results.append((suite, rc))
+
+        failed = [s for s, rc in results if rc != 0]
+        if failed:
+            print(f"FAILED suites: {failed}")
+            return 1
+        return 0
 
     finally:
         cleanup()

tests/acceptance/run-litmus.py

@@ -144,8 +144,7 @@ def setup_for_litmus(ocis_url: str) -> tuple:
     return space_id, public_token
 
 
-def run_litmus(name: str, endpoint: str) -> int:
-    print(f"\nTesting endpoint [{name}]: {endpoint}", flush=True)
+def run_litmus(name: str, endpoint: str) -> tuple:
     result = subprocess.run(
         ["docker", "run", "--rm",
          "-e", f"LITMUS_URL={endpoint}",
@@ -156,8 +155,9 @@ def run_litmus(name: str, endpoint: str) -> int:
          # No extra CMD — ENTRYPOINT is already litmus-wrapper; passing it again
          # would make the wrapper use the path as LITMUS_URL, overriding the env var.
          ],
+        capture_output=True, text=True,
     )
-    return result.returncode
+    return result.returncode, result.stdout + result.stderr
 
 
 def main() -> int:
@@ -224,11 +224,21 @@ def cleanup(*_):
             ("spaces-endpoint", f"{litmus_base}/remote.php/dav/spaces/{space_id}"),
         ]
 
-        failed = []
-        for name, endpoint in endpoints:
-            rc = run_litmus(name, endpoint)
-            if rc != 0:
-                failed.append(name)
+        import concurrent.futures
+
+        # all 5 endpoints are independent docker containers — run in parallel
+        print(f"Running {len(endpoints)} litmus endpoints in parallel", flush=True)
+        results = []
+        with concurrent.futures.ThreadPoolExecutor(max_workers=len(endpoints)) as pool:
+            fut_to_name = {pool.submit(run_litmus, n, e): n for n, e in endpoints}
+            for fut in concurrent.futures.as_completed(fut_to_name):
+                name = fut_to_name[fut]
+                rc, output = fut.result()
+                print(f"\n{'=' * 60}\n=== endpoint: {name} (exit {rc}) ===\n{'=' * 60}", flush=True)
+                print(output, flush=True)
+                results.append((name, rc))
+
+        failed = [name for name, rc in results if rc != 0]
 
         if failed:
             print(f"\nFailed endpoints: {', '.join(failed)}", file=sys.stderr)

tests/acceptance/run-wopi.py

@@ -218,8 +218,7 @@ def prepare_test_file(bridge_ip: str) -> tuple:
 
 
 def run_validator(group: str, token: str, wopi_src: str, ttl: str,
-                  secure: bool = False) -> int:
-    print(f"\nRunning testgroup [{group}] secure={secure}", flush=True)
+                  secure: bool = False) -> tuple:
     cmd = [
         "docker", "run", "--rm",
         "--workdir", "/app",
@@ -229,7 +228,8 @@ def run_validator(group: str, token: str, wopi_src: str, ttl: str,
     if secure:
         cmd.append("-s")
     cmd += ["-t", token, "-w", wopi_src, "-l", ttl, "--testgroup", group]
-    return subprocess.run(cmd).returncode
+    result = subprocess.run(cmd, capture_output=True, text=True)
+    return result.returncode, result.stdout + result.stderr
 
 
 def main() -> int:
@@ -348,18 +348,28 @@ def cleanup(*_):
         # --- prepare-test-file: upload file, get WOPI credentials ---
         access_token, ttl, wopi_src = prepare_test_file(bridge_ip)
 
-        # --- Run validator for each testgroup ---
-        failed = []
-        for group in SHARED_TESTGROUPS:
-            rc = run_validator(group, access_token, wopi_src, ttl, secure=False)
-            if rc != 0:
-                failed.append(group)
+        # --- Run all validator testgroups in parallel ---
+        import concurrent.futures
 
+        all_groups = [(g, False) for g in SHARED_TESTGROUPS]
         if wopi_type == "builtin":
-            for group in BUILTIN_ONLY_TESTGROUPS:
-                rc = run_validator(group, access_token, wopi_src, ttl, secure=True)
-                if rc != 0:
-                    failed.append(group)
+            all_groups += [(g, True) for g in BUILTIN_ONLY_TESTGROUPS]
+
+        print(f"Running {len(all_groups)} WOPI testgroups in parallel", flush=True)
+        results = []
+        with concurrent.futures.ThreadPoolExecutor(max_workers=len(all_groups)) as pool:
+            fut_to_group = {
+                pool.submit(run_validator, g, access_token, wopi_src, ttl, s): g
+                for g, s in all_groups
+            }
+            for fut in concurrent.futures.as_completed(fut_to_group):
+                group = fut_to_group[fut]
+                rc, output = fut.result()
+                print(f"\n{'=' * 60}\n=== testgroup: {group} (exit {rc}) ===\n{'=' * 60}", flush=True)
+                print(output, flush=True)
+                results.append((group, rc))
+
+        failed = [g for g, rc in results if rc != 0]
 
         if failed:
             print(f"\nFailed testgroups: {', '.join(failed)}", file=sys.stderr)