Eb/poc smoke test

.github/workflows/run-smoke-test.yaml

@@ -0,0 +1,43 @@
+name: Smoke Test
+
+on:
+  pull_request:
+    paths:
+      - "reporting-pipeline-service/**"
+      - "testing-tools/smoke-testing/**"
+      - "docker-compose.yaml"
+      - ".github/workflows/**"
+
+jobs:
+  smoke-test:
+    name: Run smoke test
+    runs-on: ubuntu-latest
+
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Install sqlcmd
+        run: |
+          curl -fsSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | sudo tee /usr/share/keyrings/microsoft-prod.gpg > /dev/null
+          curl -fsSL "https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list" | sudo tee /etc/apt/sources.list.d/msprod.list
+          sudo apt-get update
+          sudo apt-get install -y mssql-tools18 unixodbc-dev
+          echo "/opt/mssql-tools18/bin" >> "$GITHUB_PATH"
+
+      - name: Run smoke test
+        run: ./testing-tools/smoke-testing/run.sh
+
+      - name: Capture container logs on failure
+        if: failure()
+        run: docker compose logs --no-color > /tmp/container-logs.txt 2>&1 || true
+
+      - name: Upload container logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: container-logs
+          path: /tmp/container-logs.txt

containers/db/initialize/0001-reset-bad-investigation-tables.sql

@@ -0,0 +1,16 @@
+---------------------------------------------------
+-- This is temporary, some RDB tables in our
+-- Golden Backups is in a bad state so we are going
+-- to drop those tables early so RTR can rebuild.
+----------------------------------------------------
+USE RDB;
+
+DROP TABLE IF EXISTS D_INVESTIGATION_REPEAT;
+DROP TABLE IF EXISTS L_INVESTIGATION_REPEAT;
+DROP TABLE IF EXISTS S_INVESTIGATION_REPEAT;
+
+DROP TABLE IF EXISTS D_INVESTIGATION_REPEAT_INC;
+DROP TABLE IF EXISTS L_INVESTIGATION_REPEAT_INC;
+DROP TABLE IF EXISTS S_INVESTIGATION_REPEAT_INC;
+
+GO

containers/db/initialize/005-rdb-modern-functional-test-prep.sql

@@ -0,0 +1,31 @@
+------------------------------------------------
+-- Set file sizes
+------------------------------------------------
+DECLARE @CurrentSize int;
+
+SELECT @CurrentSize = size
+FROM
+    sys.master_files
+WHERE
+    name = 'RDB_log'
+    AND database_id = (
+        SELECT database_id FROM sys.databases
+        WHERE name = 'RDB'
+    );
+
+IF @CurrentSize < 524288
+    BEGIN
+        ALTER DATABASE rdb MODIFY FILE (NAME = 'RDB_log', SIZE = 4096MB);
+        ALTER DATABASE rdb MODIFY FILE (
+            NAME = 'RDB_log', MAXSIZE = UNLIMITED, FILEGROWTH = 256MB
+        );
+    END
+GO
+
+------------------------------------------------
+-- Enable CDC
+------------------------------------------------
+
+USE rdb
+EXEC sys.sp_cdc_enable_db;
+GO

reporting-pipeline-service/src/test/resources/testData/unit/covidCaseDatamart/setup.sql

@@ -24,7 +24,6 @@ DELETE FROM [RDB_MODERN].[dbo].[F_PAGE_CASE]
 DELETE FROM [RDB_MODERN].[dbo].[L_INV_SYMPTOM]
 DELETE FROM [RDB_MODERN].[dbo].[L_INV_RISK_FACTOR]
 DELETE FROM [RDB_MODERN].[dbo].[L_INV_MEDICAL_HISTORY]
-DELETE FROM [RDB_MODERN].[dbo].[L_INVESTIGATION_REPEAT]
 DELETE FROM [RDB_MODERN].[dbo].[L_INV_LAB_FINDING]
 DELETE FROM [RDB_MODERN].[dbo].[L_INV_PATIENT_OBS]
 DELETE FROM [RDB_MODERN].[dbo].[nrt_investigation_key]
@@ -317,10 +316,6 @@ INSERT INTO [dbo].[L_INV_MEDICAL_HISTORY] ([PAGE_CASE_UID], [D_INV_MEDICAL_HISTO
 -- step: 1
 INSERT INTO [dbo].[LOOKUP_TABLE_N_REPT] ([PAGE_CASE_UID]) VALUES (10009289);
 
--- dbo.L_INVESTIGATION_REPEAT
--- step: 1
-INSERT INTO [dbo].[L_INVESTIGATION_REPEAT] ([D_INVESTIGATION_REPEAT_KEY], [PAGE_CASE_UID]) VALUES (1.0, 10009289);
-
 -- dbo.L_INV_RISK_FACTOR
 -- step: 1
 INSERT INTO [dbo].[L_INV_RISK_FACTOR] ([PAGE_CASE_UID], [D_INV_RISK_FACTOR_KEY]) VALUES (10009289.0, 1.0);

testing-tools/smoke-testing/run.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+# Smoke test for the RTR reporting pipeline.
+#
+# Starts the full stack, loads a Hepatitis A patient + investigation into NBS_ODSE,
+# then polls rdb_modern.dbo.job_flow_log until 67 SP_COMPLETE rows appear after the
+# run start time and no errors are recorded. Exits 0 on success, 1 on timeout or failure.
+# The stack is torn down on exit via the trap.
+
+set -euo pipefail
+
+cd "$(dirname "$0")/../.." || exit
+
+UP_TIMEOUT=180
+UP_ELAPSED=0
+UP_WAIT=10
+VAL_TIMEOUT=120
+VAL_ELAPSED=0
+VAL_WAIT=10
+EXPECTED_SP_COMPLETE=66
+
+# Source .env if present; otherwise fall back to defaults defined in docker-compose.yaml
+[ -f .env ] && source .env
+DATABASE_PORT=${DATABASE_PORT:-3433}
+DATABASE_PASSWORD=${DATABASE_PASSWORD:-$(docker compose config --format json | jq -r '.services."nbs-mssql".environment.DATABASE_PASSWORD')}
+
+run_query() {
+    sqlcmd -S "localhost,${DATABASE_PORT}" -U sa -P "$DATABASE_PASSWORD" -C \
+        -d RDB_MODERN -h -1 -W -Q "SET NOCOUNT ON; $1" |
+        tr -d '\r' | grep -v '^[[:space:]]*$' || true
+}
+
+#trap 'docker compose down' EXIT
+
+docker compose down -v
+docker compose up --build reporting-pipeline-service -d
+
+# Wait for the reporting pipeline container to become healthy (5 minute timeout)
+echo "Waiting for containers to be healthy..."
+while ! docker compose ps --format json | jq -r 'select(.Health == "healthy") | .Name' | grep -q 'reporting-pipeline-service'; do
+    if [ "$UP_ELAPSED" -ge "$UP_TIMEOUT" ]; then
+        echo "Timed out waiting for reporting-pipeline-service to become healthy after ${UP_TIMEOUT}s"
+        exit 1
+    fi
+    echo "Still waiting for reporting-pipeline-service... (${UP_ELAPSED}s elapsed)"
+    sleep 5
+    UP_ELAPSED=$((UP_ELAPSED + UP_WAIT))
+done
+echo "Reporting pipeline is healthy!"
+
+START_TIME=$(run_query "SELECT CONVERT(varchar(30), GETDATE(), 120)")
+
+sqlcmd -S "localhost,${DATABASE_PORT}" -U sa -P "$DATABASE_PASSWORD" -C -i testing-tools/smoke-testing/setup.sql
+
+while true; do
+    echo "Validating job_flow_log... (${VAL_ELAPSED}s elapsed)"
+    PASS=true
+
+    # Check a: N SP_COMPLETE rows have appeared since the run started
+    ACTUAL_COUNT=$(run_query "SELECT COUNT(*) FROM [dbo].[job_flow_log] WHERE [Step_Name] = N'SP_COMPLETE' AND create_dttm > '${START_TIME}'")
+    if [ "$ACTUAL_COUNT" != "$EXPECTED_SP_COMPLETE" ]; then
+        echo "  [FAIL] Expected ${EXPECTED_SP_COMPLETE} SP_COMPLETE rows since ${START_TIME}, got ${ACTUAL_COUNT}"
+        PASS=false
+    fi
+
+    # Check b: no rows with a non-null Error_Description since the run started
+    ERROR_ROWS=$(run_query "SELECT Error_Description FROM [dbo].[job_flow_log] WHERE Error_Description IS NOT NULL AND create_dttm > '${START_TIME}'")
+    if [ -n "$ERROR_ROWS" ]; then
+        echo "  [FAIL] Errors found in job_flow_log:"
+        echo "$ERROR_ROWS" | sed 's/^/    /'
+        PASS=false
+    fi
+
+    if [ "$PASS" = true ]; then
+        echo "Validation passed."
+        exit 0
+    fi
+
+    if [ "$VAL_ELAPSED" -ge "$VAL_TIMEOUT" ]; then
+        echo "Validation timed out after ${VAL_TIMEOUT}s."
+        exit 1
+    fi
+
+    sleep "$VAL_WAIT"
+    VAL_ELAPSED=$((VAL_ELAPSED + VAL_WAIT))
+done