Release 1.0.3 preparation

Makefile

@@ -31,7 +31,7 @@ stop:
 REGISTRY ?= localhost:5000
 WORKERS  ?= 2
 ENV_FILE ?= .env
-SWARM_VARS = REGISTRY CERTBOT_DOMAIN CERTBOT_EMAIL NFS_SERVER_IP WORKER_CPU_LIMIT WORKER_MEMORY_LIMIT WORKER_CPU_RESERVATION WORKER_MEMORY_RESERVATION
+SWARM_VARS = REGISTRY CERTBOT_DOMAIN CERTBOT_EMAIL NFS_SERVER_IP NFS_BASE_PATH WORKER_CPU_LIMIT WORKER_MEMORY_LIMIT WORKER_CPU_RESERVATION WORKER_MEMORY_RESERVATION
 SWARM_ENV = ENV_FILE="$(ENV_FILE)" $(foreach v,$(SWARM_VARS),$(v)="$(shell grep '^$(v)=' $(ENV_FILE) | head -1 | cut -d= -f2-)")
 
 start-swarm:

backend/core/settings.py

@@ -251,7 +251,7 @@
         "OPTIONS": {
             "pool": {
                 "min_size": 2,
-                "max_size": 10,
+                "max_size": 4,  # per-process cap; swarm total = max_size × (gunicorn + celery + beat). Lowered 10→4 to stay under Postgres max_connections.
                 "max_lifetime": 600,  # recycle connections before overlay network kills them (~13 min)
             },
         },

docker-compose.swarm.nodb.yml

@@ -57,7 +57,7 @@ services:
             interval: 30s
             timeout: 10s
             retries: 3
-            start_period: 60s
+            start_period: 300s
 
     worker:
         image: ${REGISTRY}/validationsvc-backend
@@ -264,13 +264,13 @@ volumes:
         driver_opts:
             type: nfs
             o: "addr=${NFS_SERVER_IP},nfsvers=4.1,rw,hard,timeo=600,retrans=2"
-            device: ":/srv/nfs/files_data"
+            device: ":${NFS_BASE_PATH}/files_data"
     gherkin_rules_log_data:
         driver: local
         driver_opts:
             type: nfs
             o: "addr=${NFS_SERVER_IP},nfsvers=4.1,rw,hard,timeo=600,retrans=2"
-            device: ":/srv/nfs/gherkin_logs"
+            device: ":${NFS_BASE_PATH}/gherkin_logs"
     clamav_data:
     prometheus_data:
     grafana_data:

docker/grafana/dashboards/vs-platform-usage.json

@@ -270,7 +270,7 @@
   "refresh": "5m",
   "tags": [
     "observability",
-    "ivs-681"
+    "validation-service"
   ],
   "time": {
     "from": "now-30d",

docker/grafana/dashboards/vs-system-health.json

@@ -5,7 +5,7 @@
       "id": 8,
       "type": "stat",
       "title": "Alert candidate: antivirus queue",
-      "description": "E1 threshold: backlog on the antivirus queue > 0. Every validation passes through this queue; the av_worker zombie of 27 July sat dead for 28 hours without anyone noticing.",
+      "description": "Alert threshold: backlog on the antivirus queue > 0. Every validation passes through this queue; the av_worker zombie of 27 July sat dead for 28 hours without anyone noticing.",
       "datasource": {
         "type": "prometheus",
         "uid": "prometheus"
@@ -62,7 +62,7 @@
       "id": 9,
       "type": "stat",
       "title": "Alert candidate: memory (min across nodes)",
-      "description": "E1 threshold: MemAvailable < 2 GB (red; orange below 4 GB). Lowest value across all nodes. Without swap, memory pressure is immediately fatal — that is what made the freeze so abrupt.",
+      "description": "Alert threshold: MemAvailable < 2 GB (red; orange below 4 GB). Lowest value across all nodes. Without swap, memory pressure is immediately fatal — that is what made the freeze so abrupt.",
       "datasource": {
         "type": "prometheus",
         "uid": "prometheus"
@@ -123,7 +123,7 @@
       "id": 10,
       "type": "stat",
       "title": "Alert candidate: disk (max across nodes)",
-      "description": "E1 threshold: disk > 85% (orange), > 95% red. The fullest node counts. Currently red as expected: dev-vm-worker-1 sits around 96% (IVS-828, backlog item A4).",
+      "description": "Alert threshold: disk > 85% (orange), > 95% red. The fullest node counts. The small 28 GB disk of dev-vm-worker-1 historically fills first (Docker images); at 100% it broke a deploy on 3 Aug.",
       "datasource": {
         "type": "prometheus",
         "uid": "prometheus"
@@ -280,7 +280,7 @@
           "refId": "A"
         }
       ],
-      "title": "Disk used % (per node) — the IVS-828 chart",
+      "title": "Disk used % (per node)",
       "type": "timeseries",
       "description": "Percentage of disk in use, per Swarm node. Above ~90% things get risky."
     },
@@ -310,7 +310,8 @@
         }
       ],
       "title": "Memory available (per node)",
-      "type": "timeseries"
+      "type": "timeseries",
+      "description": "Memory the kernel can still hand out, per node. These VMs have no swap, so hitting zero freezes the node outright (as happened 28 Jul). Bands match the alert tile: orange below 4 GB, red below 2 GB."
     },
     {
       "datasource": {
@@ -339,7 +340,8 @@
         }
       ],
       "title": "CPU usage % (per node)",
-      "type": "timeseries"
+      "type": "timeseries",
+      "description": "CPU load per node (100% = all cores busy). In every incident so far the bottleneck was memory or disk, never CPU — kept for completeness."
     },
     {
       "datasource": {
@@ -368,7 +370,8 @@
         }
       ],
       "title": "Celery: active workers per queue",
-      "type": "timeseries"
+      "type": "timeseries",
+      "description": "Number of Celery worker instances consuming each queue. Zero on a queue means nobody is listening — the zombie-worker signature of 27 Jul, when the queue filled while Docker still reported the container as Running."
     },
     {
       "datasource": {
@@ -397,7 +400,8 @@
         }
       ],
       "title": "Celery: queue length (backlog)",
-      "type": "timeseries"
+      "type": "timeseries",
+      "description": "Tasks waiting in each Redis queue. A growing line while workers appear active means processing is stuck. The antivirus queue matters most: it has a single consumer and every validation passes through it first."
     },
     {
       "datasource": {
@@ -579,7 +583,7 @@
   "refresh": "30s",
   "tags": [
     "observability",
-    "ivs-681"
+    "validation-service"
   ],
   "time": {
     "from": "now-3h",

docker/grafana/dashboards/vs-validation-perf.json

@@ -473,7 +473,7 @@
       "id": 7,
       "type": "barchart",
       "title": "Most expensive gherkin rules (total CPU time, cumulative)",
-      "description": "From the gherkin logs on the NFS share (201k measurements since Dec 2025), ingested nightly by perf-metrics/gherkin_rule_timings.sh. NOTE: this is CPU time (time.process_time), not wall-clock time, so I/O wait is not included. Logged in PRODUCTION mode only.",
+      "description": "TEMPORARILY EMPTY after the 3 Aug release: the nightly log-ingest (textfile exporter) is not part of the release stack yet — it returns via a small follow-up PR. The underlying data (201k measurements) is safe in the gherkin logs. From the gherkin logs on the NFS share (201k measurements since Dec 2025), ingested nightly by perf-metrics/gherkin_rule_timings.sh. NOTE: this is CPU time (time.process_time), not wall-clock time, so I/O wait is not included. Logged in PRODUCTION mode only.",
       "gridPos": {
         "h": 9,
         "w": 12,
@@ -507,7 +507,7 @@
       "id": 8,
       "type": "table",
       "title": "Rule cost: total, average and longest run",
-      "description": "Average versus longest run shows the skew: a few large models dominate. A rule with a low average but an extreme maximum is a tail risk.",
+      "description": "TEMPORARILY EMPTY after the 3 Aug release: the nightly log-ingest (textfile exporter) is not part of the release stack yet — it returns via a small follow-up PR. The underlying data (201k measurements) is safe in the gherkin logs. Average versus longest run shows the skew: a few large models dominate. A rule with a low average but an extreme maximum is a tail risk.",
       "gridPos": {
         "h": 9,
         "w": 12,
@@ -562,7 +562,7 @@
       "id": 9,
       "type": "table",
       "title": "Outcomes per rule (top 15, whole DB)",
-      "description": "Which rules produce the most outcomes (severity WARNING/ERROR)? Feeds backlog item F4: SWE001 and IFC105 together account for ~1.7M of the ~5M rows.",
+      "description": "Which rules produce the most outcomes (severity WARNING/ERROR)? SWE001 and IFC105 together account for ~1.7M of the ~5M rows (candidates for retention pruning).",
       "datasource": {
         "type": "grafana-postgresql-datasource",
         "uid": "devpg"