diff --git a/Makefile b/Makefile index b572209f..b817a7d9 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ stop: REGISTRY ?= localhost:5000 WORKERS ?= 2 ENV_FILE ?= .env -SWARM_VARS = REGISTRY CERTBOT_DOMAIN CERTBOT_EMAIL NFS_SERVER_IP WORKER_CPU_LIMIT WORKER_MEMORY_LIMIT WORKER_CPU_RESERVATION WORKER_MEMORY_RESERVATION +SWARM_VARS = REGISTRY CERTBOT_DOMAIN CERTBOT_EMAIL NFS_SERVER_IP NFS_BASE_PATH WORKER_CPU_LIMIT WORKER_MEMORY_LIMIT WORKER_CPU_RESERVATION WORKER_MEMORY_RESERVATION SWARM_ENV = ENV_FILE="$(ENV_FILE)" $(foreach v,$(SWARM_VARS),$(v)="$(shell grep '^$(v)=' $(ENV_FILE) | head -1 | cut -d= -f2-)") start-swarm: diff --git a/backend/core/settings.py b/backend/core/settings.py index e1c6be20..b6e34ea0 100644 --- a/backend/core/settings.py +++ b/backend/core/settings.py @@ -251,7 +251,7 @@ "OPTIONS": { "pool": { "min_size": 2, - "max_size": 10, + "max_size": 4, # per-process cap; swarm total = max_size × (gunicorn + celery + beat). Lowered 10→4 to stay under Postgres max_connections. "max_lifetime": 600, # recycle connections before overlay network kills them (~13 min) }, }, diff --git a/docker-compose.swarm.nodb.yml b/docker-compose.swarm.nodb.yml index e6403ecc..637275a1 100644 --- a/docker-compose.swarm.nodb.yml +++ b/docker-compose.swarm.nodb.yml @@ -57,7 +57,7 @@ services: interval: 30s timeout: 10s retries: 3 - start_period: 60s + start_period: 300s worker: image: ${REGISTRY}/validationsvc-backend @@ -264,13 +264,13 @@ volumes: driver_opts: type: nfs o: "addr=${NFS_SERVER_IP},nfsvers=4.1,rw,hard,timeo=600,retrans=2" - device: ":/srv/nfs/files_data" + device: ":${NFS_BASE_PATH}/files_data" gherkin_rules_log_data: driver: local driver_opts: type: nfs o: "addr=${NFS_SERVER_IP},nfsvers=4.1,rw,hard,timeo=600,retrans=2" - device: ":/srv/nfs/gherkin_logs" + device: ":${NFS_BASE_PATH}/gherkin_logs" clamav_data: prometheus_data: grafana_data: \ No newline at end of file diff --git a/docker/grafana/dashboards/vs-platform-usage.json b/docker/grafana/dashboards/vs-platform-usage.json index 22567760..79712838 100644 --- a/docker/grafana/dashboards/vs-platform-usage.json +++ b/docker/grafana/dashboards/vs-platform-usage.json @@ -270,7 +270,7 @@ "refresh": "5m", "tags": [ "observability", - "ivs-681" + "validation-service" ], "time": { "from": "now-30d", diff --git a/docker/grafana/dashboards/vs-system-health.json b/docker/grafana/dashboards/vs-system-health.json index 0de18c4b..c67d5abb 100644 --- a/docker/grafana/dashboards/vs-system-health.json +++ b/docker/grafana/dashboards/vs-system-health.json @@ -5,7 +5,7 @@ "id": 8, "type": "stat", "title": "Alert candidate: antivirus queue", - "description": "E1 threshold: backlog on the antivirus queue > 0. Every validation passes through this queue; the av_worker zombie of 27 July sat dead for 28 hours without anyone noticing.", + "description": "Alert threshold: backlog on the antivirus queue > 0. Every validation passes through this queue; the av_worker zombie of 27 July sat dead for 28 hours without anyone noticing.", "datasource": { "type": "prometheus", "uid": "prometheus" @@ -62,7 +62,7 @@ "id": 9, "type": "stat", "title": "Alert candidate: memory (min across nodes)", - "description": "E1 threshold: MemAvailable < 2 GB (red; orange below 4 GB). Lowest value across all nodes. Without swap, memory pressure is immediately fatal — that is what made the freeze so abrupt.", + "description": "Alert threshold: MemAvailable < 2 GB (red; orange below 4 GB). Lowest value across all nodes. Without swap, memory pressure is immediately fatal — that is what made the freeze so abrupt.", "datasource": { "type": "prometheus", "uid": "prometheus" @@ -123,7 +123,7 @@ "id": 10, "type": "stat", "title": "Alert candidate: disk (max across nodes)", - "description": "E1 threshold: disk > 85% (orange), > 95% red. The fullest node counts. Currently red as expected: dev-vm-worker-1 sits around 96% (IVS-828, backlog item A4).", + "description": "Alert threshold: disk > 85% (orange), > 95% red. The fullest node counts. The small 28 GB disk of dev-vm-worker-1 historically fills first (Docker images); at 100% it broke a deploy on 3 Aug.", "datasource": { "type": "prometheus", "uid": "prometheus" @@ -280,7 +280,7 @@ "refId": "A" } ], - "title": "Disk used % (per node) — the IVS-828 chart", + "title": "Disk used % (per node)", "type": "timeseries", "description": "Percentage of disk in use, per Swarm node. Above ~90% things get risky." }, @@ -310,7 +310,8 @@ } ], "title": "Memory available (per node)", - "type": "timeseries" + "type": "timeseries", + "description": "Memory the kernel can still hand out, per node. These VMs have no swap, so hitting zero freezes the node outright (as happened 28 Jul). Bands match the alert tile: orange below 4 GB, red below 2 GB." }, { "datasource": { @@ -339,7 +340,8 @@ } ], "title": "CPU usage % (per node)", - "type": "timeseries" + "type": "timeseries", + "description": "CPU load per node (100% = all cores busy). In every incident so far the bottleneck was memory or disk, never CPU — kept for completeness." }, { "datasource": { @@ -368,7 +370,8 @@ } ], "title": "Celery: active workers per queue", - "type": "timeseries" + "type": "timeseries", + "description": "Number of Celery worker instances consuming each queue. Zero on a queue means nobody is listening — the zombie-worker signature of 27 Jul, when the queue filled while Docker still reported the container as Running." }, { "datasource": { @@ -397,7 +400,8 @@ } ], "title": "Celery: queue length (backlog)", - "type": "timeseries" + "type": "timeseries", + "description": "Tasks waiting in each Redis queue. A growing line while workers appear active means processing is stuck. The antivirus queue matters most: it has a single consumer and every validation passes through it first." }, { "datasource": { @@ -579,7 +583,7 @@ "refresh": "30s", "tags": [ "observability", - "ivs-681" + "validation-service" ], "time": { "from": "now-3h", diff --git a/docker/grafana/dashboards/vs-validation-perf.json b/docker/grafana/dashboards/vs-validation-perf.json index 4dd5891e..db9a2107 100644 --- a/docker/grafana/dashboards/vs-validation-perf.json +++ b/docker/grafana/dashboards/vs-validation-perf.json @@ -473,7 +473,7 @@ "id": 7, "type": "barchart", "title": "Most expensive gherkin rules (total CPU time, cumulative)", - "description": "From the gherkin logs on the NFS share (201k measurements since Dec 2025), ingested nightly by perf-metrics/gherkin_rule_timings.sh. NOTE: this is CPU time (time.process_time), not wall-clock time, so I/O wait is not included. Logged in PRODUCTION mode only.", + "description": "TEMPORARILY EMPTY after the 3 Aug release: the nightly log-ingest (textfile exporter) is not part of the release stack yet — it returns via a small follow-up PR. The underlying data (201k measurements) is safe in the gherkin logs. From the gherkin logs on the NFS share (201k measurements since Dec 2025), ingested nightly by perf-metrics/gherkin_rule_timings.sh. NOTE: this is CPU time (time.process_time), not wall-clock time, so I/O wait is not included. Logged in PRODUCTION mode only.", "gridPos": { "h": 9, "w": 12, @@ -507,7 +507,7 @@ "id": 8, "type": "table", "title": "Rule cost: total, average and longest run", - "description": "Average versus longest run shows the skew: a few large models dominate. A rule with a low average but an extreme maximum is a tail risk.", + "description": "TEMPORARILY EMPTY after the 3 Aug release: the nightly log-ingest (textfile exporter) is not part of the release stack yet — it returns via a small follow-up PR. The underlying data (201k measurements) is safe in the gherkin logs. Average versus longest run shows the skew: a few large models dominate. A rule with a low average but an extreme maximum is a tail risk.", "gridPos": { "h": 9, "w": 12, @@ -562,7 +562,7 @@ "id": 9, "type": "table", "title": "Outcomes per rule (top 15, whole DB)", - "description": "Which rules produce the most outcomes (severity WARNING/ERROR)? Feeds backlog item F4: SWE001 and IFC105 together account for ~1.7M of the ~5M rows.", + "description": "Which rules produce the most outcomes (severity WARNING/ERROR)? SWE001 and IFC105 together account for ~1.7M of the ~5M rows (candidates for retention pruning).", "datasource": { "type": "grafana-postgresql-datasource", "uid": "devpg"