diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..94bbdf4 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,22 @@ +.git +.github +.venv +__pycache__ +*.pyc +.pytest_cache +.mypy_cache +.env +.env.* +mlflow.db +mlartifacts +mlruns +artifacts +data/raw +data/interim +data/processed +*.tmp +*.log +build +dist +reports/*_attempt_error.json +notebooks diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..decabc1 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,47 @@ +name: InspectIQ CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + cache: pip + cache-dependency-path: requirements.txt + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + - name: Check whitespace and line endings + run: | + git diff --check + git ls-files -z | xargs -0 file | (! grep -E 'CRLF|with CRLF') + - name: Compile source + run: python -m compileall app src scripts tests *.py + - name: Run unit tests + run: python -m unittest discover -s tests -t . -v + - name: Validate release contract + run: python run_release_validation.py --mode ci + - name: Build dashboard image + run: docker build --tag inspectiq:ci . + - name: Upload release validation report + if: always() + uses: actions/upload-artifact@v4 + with: + name: release-validation-report + path: | + reports/release_validation_report.json + reports/release_validation_attempt_error.json + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index 1c2a321..850a367 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ data/processed/ *.log .DS_Store Thumbs.db +reports/release_validation_attempt_error.json diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1fca220 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3.13-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/app + +WORKDIR /app + +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +COPY app ./app +COPY src ./src +COPY config ./config +COPY reports ./reports +COPY run_*.py ./ + +RUN useradd --create-home --uid 10001 appuser \ + && chown -R appuser:appuser /app +USER appuser + +EXPOSE 8501 + +CMD ["python", "-m", "streamlit", "run", "app/streamlit_app.py", "--server.address=0.0.0.0", "--server.port=8501", "--server.headless=true"] diff --git a/config/release_config.yaml b/config/release_config.yaml new file mode 100644 index 0000000..38cc910 --- /dev/null +++ b/config/release_config.yaml @@ -0,0 +1,53 @@ +release_validation_version: day7a-release-v1 +required_source_files: + - app/streamlit_app.py + - src/governance.py + - src/monitoring.py + - run_release_validation.py + - Dockerfile + - .dockerignore + - .github/workflows/ci.yml +required_config_files: + - config/dashboard_config.yaml + - config/monitoring_config.yaml + - config/release_config.yaml +required_committed_reports: + - reports/feasibility_report.json + - reports/data_foundation_report.json + - reports/baseline_report.json + - reports/feature_engineering_report.json + - reports/model_comparison_report.json + - reports/calibration_report.json + - reports/mlflow_tracking_report.json + - reports/batch_prediction_report.json + - reports/dashboard_validation_report.json + - reports/monitoring_report.json +required_local_artifact_categories: + - final candidate model + - ranked candidate output + - top-10-percent candidate output + - monitoring manifest and governance worksheets +required_report_statuses: + - PASS +allowed_monitoring_health_values: [HEALTHY, WARNING, CRITICAL] +expected_model_experiment: exp_05_random_forest +expected_selected_calibration_method: uncalibrated +expected_locked_candidate_row_count: 300 +expected_top_10_row_count: 30 +required_safety_flags: + labels_accessed: false + performance_metrics_calculated: false + outcome_fairness_metrics_calculated: false + automatic_enforcement: false + model_refit_attempted: false + prediction_artifact_modified: false +docker_runtime_requirements: + port: 8501 + non_root_user: true + artifact_mounts: [data, artifacts, reports] +ci_workflow_requirements: + python_version: '3.13' + required_tokens: [compileall, unittest, run_release_validation.py --mode ci, docker build] +prohibited_tracked_file_patterns: ['*.env', 'mlflow.db', 'mlartifacts/**', 'mlruns/**', 'artifacts/**', 'data/raw/**', 'data/interim/**', 'data/processed/**'] +attempt_error_report: reports/release_validation_attempt_error.json +main_report: reports/release_validation_report.json diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..6de74e9 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,13 @@ +services: + dashboard: + build: + context: . + dockerfile: Dockerfile + ports: + - "8501:8501" + # The image contains code and committed reports only. Supply frozen local + # artifacts explicitly; the dashboard never regenerates them in-container. + volumes: + - ./data:/app/data:ro + - ./artifacts:/app/artifacts:ro + - ./reports:/app/reports:ro diff --git a/reports/release_validation_report.json b/reports/release_validation_report.json new file mode 100644 index 0000000..11cd714 --- /dev/null +++ b/reports/release_validation_report.json @@ -0,0 +1,135 @@ +{ + "artifact_checks": { + "artifacts": [ + "artifacts/models/day4/5ed044f95d5bc1dd/final/final_candidate.joblib", + "artifacts/monitoring/day6/0b1490c5daa560dc0b34/future_outcome_template.csv", + "artifacts/monitoring/day6/0b1490c5daa560dc0b34/monitoring_manifest.json", + "artifacts/monitoring/day6/0b1490c5daa560dc0b34/review_queue_template.csv", + "artifacts/predictions/day5b/e0a880a30016dbc04085/ranked_candidates.csv", + "artifacts/predictions/day5b/e0a880a30016dbc04085/top_10_percent_candidates.csv" + ], + "feature_version": "day2-historical-v1", + "hashes": { + "model": true, + "monitoring": true, + "ranked": true, + "top_10": true + }, + "safety_flags": { + "automatic_enforcement": true, + "labels_accessed": true, + "model_refit_attempted": true, + "outcome_fairness_metrics_calculated": true, + "performance_metrics_calculated": true, + "prediction_artifact_modified": true + }, + "selected_calibration_method": "uncalibrated", + "selected_experiment": "exp_05_random_forest", + "source_snapshot_id": "edbd4bd813ed8e1dbaba9e1c", + "valid": true + }, + "ci_workflow_checks": { + "compileall": true, + "docker_build": true, + "least_privilege": true, + "no_secrets": true, + "python_3_13": true, + "release_validation": true, + "required_triggers": true, + "unittest": true, + "yaml_parses": true + }, + "configuration_checks": { + "files": [ + "config/dashboard_config.yaml", + "config/monitoring_config.yaml", + "config/release_config.yaml" + ], + "valid": true + }, + "docker_contract_checks": { + "binds_container_address": true, + "env_excluded": true, + "generated_state_excluded": true, + "non_root_user": true, + "port_8501": true, + "python_3_13_slim": true, + "pythonpath": true, + "streamlit_command": true + }, + "failures": [], + "feature_version": "day2-historical-v1", + "governance_checks": { + "future_template_headers_only": true, + "human_review_fields_checked": true + }, + "hash_checks": { + "model": true, + "monitoring": true, + "ranked": true, + "top_10": true + }, + "ignored_artifact_policy_checks": { + "git_available": true, + "no_generated_or_secret_files_tracked": true + }, + "limitations": [ + "Candidate ranking supports human review; there is no automatic enforcement.", + "The 2023 candidate batch is awaiting complete outcome labels.", + "This release validation does not load labels, calculate performance or fairness metrics, fit models, or regenerate predictions.", + "The selected final candidate uses uncalibrated model output and retrospective validation only." + ], + "locked_candidate_safety_checks": { + "automatic_enforcement": true, + "labels_accessed": true, + "model_refit_attempted": true, + "outcome_fairness_metrics_calculated": true, + "performance_metrics_calculated": true, + "prediction_artifact_modified": true + }, + "mode": "local", + "monitoring_safety_checks": { + "health_independent_from_pipeline": true, + "monitoring_artifacts_checked": true + }, + "platform": "Windows", + "python_version": "3.13.14", + "release_validation_version": "day7a-release-v1", + "report_checks": { + "files": [ + "reports/baseline_report.json", + "reports/batch_prediction_report.json", + "reports/calibration_report.json", + "reports/dashboard_validation_report.json", + "reports/data_foundation_report.json", + "reports/feasibility_report.json", + "reports/feature_engineering_report.json", + "reports/mlflow_tracking_report.json", + "reports/model_comparison_report.json", + "reports/monitoring_report.json" + ], + "valid": true + }, + "required_file_checks": { + "files": [ + ".dockerignore", + ".github/workflows/ci.yml", + "Dockerfile", + "app/streamlit_app.py", + "run_release_validation.py", + "src/governance.py", + "src/monitoring.py" + ], + "valid": true + }, + "runtime_seconds": 0.757284, + "selected_calibration_method": "uncalibrated", + "selected_experiment": "exp_05_random_forest", + "source_snapshot_id": "edbd4bd813ed8e1dbaba9e1c", + "status": "PASS", + "streamlit_import_check": { + "server_started": false, + "valid": true + }, + "warnings": [] +} diff --git a/run_release_validation.py b/run_release_validation.py new file mode 100644 index 0000000..dbdd64c --- /dev/null +++ b/run_release_validation.py @@ -0,0 +1,43 @@ +"""Run read-only Day 7A release validation against existing artifacts.""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from src.release_validation import ReleaseValidationError, validate_release, write_json_atomic + + +def main() -> int: + parser = argparse.ArgumentParser(description="InspectIQ read-only release validation") + parser.add_argument("--mode", choices=("ci", "local"), required=True, help="ci omits ignored local artifacts; local validates frozen artifacts") + args = parser.parse_args() + root = Path(__file__).resolve().parent + report_path = root / "reports" / "release_validation_report.json" + error_path = root / "reports" / "release_validation_attempt_error.json" + try: + report = validate_release(root, args.mode) + write_json_atomic(report_path, report) + except Exception as exc: + error = { + "status": "FAIL", + "mode": args.mode, + "error": str(exc), + "limitations": ["The prior valid release validation report was preserved."], + } + write_json_atomic(error_path, error) + print(f"INSPECTIQ RELEASE VALIDATION {args.mode.upper()}: FAIL") + print(f"reason={exc}") + return 1 + print(f"mode={args.mode}") + print("required_files_valid=true configs_valid=true reports_valid=true") + print(f"artifacts_valid={'true' if args.mode == 'local' else 'not_required'}") + print("docker_contract_valid=true ci_workflow_valid=true streamlit_import_valid=true") + print("labels_accessed=false performance_metrics_calculated=false outcome_fairness_metrics_calculated=false automatic_enforcement=false") + print(f"INSPECTIQ RELEASE VALIDATION {args.mode.upper()}: PASS") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/release_validation.py b/src/release_validation.py new file mode 100644 index 0000000..9503cf7 --- /dev/null +++ b/src/release_validation.py @@ -0,0 +1,314 @@ +"""Read-only Day 7A release integrity checks. + +This module validates frozen reports and artifacts. It never fetches data, +loads outcome labels, fits models, recalibrates scores, or regenerates ranking. +""" +from __future__ import annotations + +import csv +import hashlib +import importlib +import json +import os +import platform +import re +import sys +import time +from pathlib import Path +from typing import Any + +import yaml + + +class ReleaseValidationError(RuntimeError): + """A release contract check failed.""" + + +def _relative(root: Path, path: Path) -> str: + try: + return path.relative_to(root).as_posix() + except ValueError: + return path.name + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _read_json(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ReleaseValidationError(f"Invalid JSON report: {path.name}: {exc}") from exc + if not isinstance(value, dict): + raise ReleaseValidationError(f"JSON report must contain an object: {path.name}") + return value + + +def _load_config(root: Path) -> dict[str, Any]: + path = root / "config" / "release_config.yaml" + try: + value = yaml.safe_load(path.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError) as exc: + raise ReleaseValidationError(f"Invalid release configuration: {exc}") from exc + if not isinstance(value, dict): + raise ReleaseValidationError("release configuration must be a YAML mapping") + return value + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise ReleaseValidationError(message) + + +def _path_from_report(root: Path, value: Any, field: str) -> Path: + _require(isinstance(value, str) and value, f"Missing report path: {field}") + path = Path(value.replace("\\", "/")) + _require(not path.is_absolute(), f"Absolute artifact path is not permitted: {field}") + return root / path + + +def _workflow_checks(root: Path) -> dict[str, bool]: + path = root / ".github" / "workflows" / "ci.yml" + text = path.read_text(encoding="utf-8") + try: + yaml.safe_load(text) + except yaml.YAMLError as exc: + raise ReleaseValidationError(f"Invalid CI workflow YAML: {exc}") from exc + lowered = text.lower() + checks = { + "yaml_parses": True, + "python_3_13": "python-version: '3.13'" in text or 'python-version: "3.13"' in text, + "required_triggers": all(token in text for token in ("push:", "pull_request:", "workflow_dispatch:")), + "compileall": "compileall" in text, + "unittest": "unittest discover" in text, + "release_validation": "run_release_validation.py --mode ci" in text, + "docker_build": "docker build" in text, + "least_privilege": "contents: read" in lowered, + "no_secrets": "secrets." not in lowered and "DOL_API_KEY" not in text, + } + _require(all(checks.values()), "CI workflow does not meet the release contract") + return checks + + +def _docker_checks(root: Path) -> dict[str, bool]: + dockerfile = (root / "Dockerfile").read_text(encoding="utf-8") + ignored = (root / ".dockerignore").read_text(encoding="utf-8") + checks = { + "python_3_13_slim": "FROM python:3.13-slim" in dockerfile, + "non_root_user": "USER appuser" in dockerfile, + "port_8501": "EXPOSE 8501" in dockerfile, + "pythonpath": "PYTHONPATH=/app" in dockerfile, + "streamlit_command": False, # populated below to keep checks ordered + "binds_container_address": "--server.address=0.0.0.0" in dockerfile, + "env_excluded": ".env" in ignored, + "generated_state_excluded": all(token in ignored for token in ("artifacts", "data/raw", "mlflow.db", "mlartifacts")), + } + checks["streamlit_command"] = all(token in dockerfile for token in ("python", "-m", "streamlit", "app/streamlit_app.py")) + _require(all(checks.values()), "Docker packaging does not meet the runtime contract") + return checks + + +def _tracked_file_checks(root: Path, config: dict[str, Any]) -> dict[str, bool]: + # A non-git synthetic fixture is valid for CI-mode unit tests. + git_index = root / ".git" + if not git_index.exists(): + return {"git_available": False, "no_generated_or_secret_files_tracked": True} + import subprocess + + result = subprocess.run(["git", "ls-files"], cwd=root, capture_output=True, text=True, check=False) + _require(result.returncode == 0, "Unable to inspect tracked files") + tracked = [line.replace("\\", "/") for line in result.stdout.splitlines()] + patterns = config.get("prohibited_tracked_file_patterns", []) + from fnmatch import fnmatch + + offenders = sorted(name for name in tracked if any(fnmatch(name, pattern) for pattern in patterns)) + _require(not offenders, f"Ignored/generated files are tracked: {', '.join(offenders)}") + secret_like = [name for name in tracked if name.lower().endswith(".env") or "/.env" in name.lower()] + _require(not secret_like, f"Secret-like file is tracked: {', '.join(secret_like)}") + return {"git_available": True, "no_generated_or_secret_files_tracked": True} + + +def _safety_language_checks(root: Path) -> dict[str, bool]: + content = "\n".join((root / name).read_text(encoding="utf-8") for name in ("src/governance.py", "src/monitoring.py")) + lowered = content.lower() + checks = { + "human_review_language": "human review" in lowered or "review" in lowered, + "no_automatic_enforcement": "no outcomes are created" in lowered or "outcome-free" in lowered, + "no_release_network_client": not re.search(r"^\\s*(?:import|from)\\s+(?:requests|httpx)\\b", (root / "src" / "release_validation.py").read_text(encoding="utf-8"), re.MULTILINE), + } + _require(all(checks.values()), "Required safety language/checks are absent") + return checks + + +def _check_report_status(path: Path, report: dict[str, Any]) -> None: + if "status" in report: + _require(report["status"] == "PASS", f"Report is not PASS: {path.name}") + + +def _ci_checks(root: Path, config: dict[str, Any]) -> tuple[dict[str, Any], dict[str, dict[str, Any]]]: + required = sorted(config["required_source_files"]) + missing = [name for name in required if not (root / name).is_file()] + _require(not missing, f"Required source files missing: {', '.join(missing)}") + config_files = sorted(config["required_config_files"]) + for name in config_files: + try: + yaml.safe_load((root / name).read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError) as exc: + raise ReleaseValidationError(f"Invalid configuration {name}: {exc}") from exc + reports: dict[str, dict[str, Any]] = {} + for name in sorted(config["required_committed_reports"]): + path = root / name + _require(path.is_file(), f"Committed report missing: {name}") + reports[name] = _read_json(path) + _check_report_status(path, reports[name]) + importlib.invalidate_caches() + try: + importlib.import_module("app.streamlit_app") + except Exception as exc: # import only; no Streamlit server is started + raise ReleaseValidationError(f"Streamlit application import failed: {exc}") from exc + checks = { + "required_files": {"valid": True, "files": required}, + "configuration": {"valid": True, "files": config_files}, + "reports": {"valid": True, "files": sorted(reports)}, + "workflow": _workflow_checks(root), + "docker": _docker_checks(root), + "streamlit_import": {"valid": True, "server_started": False}, + "safety": _safety_language_checks(root), + "ignored_artifact_policy": _tracked_file_checks(root, config), + } + return checks, reports + + +def _csv_rows(path: Path) -> tuple[list[str], list[dict[str, str]]]: + with path.open("r", encoding="utf-8", newline="") as handle: + reader = csv.DictReader(handle) + headers = reader.fieldnames or [] + return headers, list(reader) + + +def _local_checks(root: Path, config: dict[str, Any], reports: dict[str, dict[str, Any]]) -> dict[str, Any]: + by_name = {Path(name).name: report for name, report in reports.items()} + feasibility = by_name["feasibility_report.json"] + feature = by_name["feature_engineering_report.json"] + model = by_name["model_comparison_report.json"] + calibration = by_name["calibration_report.json"] + mlflow = by_name["mlflow_tracking_report.json"] + batch = by_name["batch_prediction_report.json"] + dashboard = by_name["dashboard_validation_report.json"] + monitoring = by_name["monitoring_report.json"] + _require(feasibility.get("decision") == "GO", "Day 0 feasibility decision must be GO") + snapshot = feature.get("source_snapshot_id") + _require(snapshot and all(report.get("source_snapshot_id", snapshot) == snapshot for report in (model, calibration, batch, dashboard, monitoring)), "Source snapshot IDs do not agree") + feature_version = feature.get("feature_version") + _require(feature_version and all(report.get("feature_version", feature_version) == feature_version for report in (model, calibration, batch, dashboard, monitoring)), "Feature versions do not agree") + selected_candidate = model.get("selected_candidate") + selected_experiment = selected_candidate.get("experiment_name") if isinstance(selected_candidate, dict) else selected_candidate + _require(selected_experiment == config["expected_model_experiment"], "Selected Day 3 experiment does not match release contract") + _require(calibration.get("selected_calibration_method") == config["expected_selected_calibration_method"], "Selected calibration method must remain uncalibrated") + _require(batch.get("selected_day3_experiment") == config["expected_model_experiment"], "Batch report selected experiment is incompatible") + _require(batch.get("selected_day4_method") == config["expected_selected_calibration_method"], "Batch report must describe uncalibrated model output") + _require(mlflow.get("logged_day3_run_count") == 8 and mlflow.get("logged_day4_run_count") == 3, "MLflow logical run counts must be 8 Day 3 and 3 Day 4") + _require(mlflow.get("final_candidate_artifact_logged") is True, "Final candidate artifact was not logged with the selected run") + _require(mlflow.get("reused_run_count", 0) >= 11 and mlflow.get("newly_created_run_count", 0) == 0, "MLflow second-run idempotency evidence is invalid") + model_path = _path_from_report(root, calibration.get("final_candidate_artifact_path"), "final_candidate_artifact_path") + ranked_path = _path_from_report(root, batch.get("ranked_output_path"), "ranked_output_path") + top_path = _path_from_report(root, batch.get("top_10_output_path"), "top_10_output_path") + for path in (model_path, ranked_path, top_path): + _require(path.is_file(), f"Required frozen artifact missing: {_relative(root, path)}") + _require(_sha256(model_path) == batch.get("model_artifact_hash"), "Final candidate model hash mismatch") + _require(_sha256(ranked_path) == batch.get("ranked_output_hash"), "Ranked candidate hash mismatch") + _require(_sha256(top_path) == batch.get("top_10_output_hash"), "Top-10 candidate hash mismatch") + headers, rows = _csv_rows(ranked_path) + top_headers, top_rows = _csv_rows(top_path) + _require(len(rows) == config["expected_locked_candidate_row_count"], "Ranked candidate row count must be 300") + _require(len(top_rows) == config["expected_top_10_row_count"], "Top-10 candidate row count must be 30") + prohibited = {"label", "target", "outcome", "viol_type", "serious_violation"} + _require(not (prohibited & {header.lower() for header in headers + top_headers}), "Ranked output contains an outcome or target field") + manifest_path = _path_from_report(root, monitoring.get("monitoring_manifest_path"), "monitoring_manifest_path") + template_path = _path_from_report(root, monitoring.get("future_outcome_template_path"), "future_outcome_template_path") + worksheet_path = _path_from_report(root, monitoring.get("review_worksheet_path"), "review_worksheet_path") + for path in (manifest_path, template_path, worksheet_path): + _require(path.is_file(), f"Monitoring artifact missing: {_relative(root, path)}") + _require(_sha256(template_path) == monitoring.get("future_outcome_template_hash"), "Future outcome template hash mismatch") + _require(_sha256(worksheet_path) == monitoring.get("review_worksheet_hash"), "Review worksheet hash mismatch") + template_headers, template_rows = _csv_rows(template_path) + _require(bool(template_headers) and not template_rows, "Future outcome template must contain headers only") + from src.governance import REVIEW_COLUMNS + worksheet_headers, _ = _csv_rows(worksheet_path) + _require(set(REVIEW_COLUMNS).issubset(worksheet_headers), "Human-review governance fields are incomplete") + _require(monitoring.get("monitoring_health") in config["allowed_monitoring_health_values"], "Monitoring health is invalid") + flags = { + "labels_accessed": batch.get("labels_accessed") is False and monitoring.get("current_labels_accessed") is False, + "performance_metrics_calculated": batch.get("performance_metrics_calculated") is False and monitoring.get("current_performance_metrics_calculated") is False, + "outcome_fairness_metrics_calculated": monitoring.get("outcome_fairness_metrics_calculated") is False, + "automatic_enforcement": batch.get("automatic_enforcement") is False and monitoring.get("automatic_enforcement") is False, + "model_refit_attempted": dashboard.get("model_refit_attempted") is False and monitoring.get("model_refit_attempted") is False, + "prediction_artifact_modified": monitoring.get("prediction_artifact_modified") is False, + } + _require(all(flags.values()), "Locked candidate safety flags are not all false") + return { + "valid": True, + "source_snapshot_id": snapshot, + "feature_version": feature_version, + "selected_experiment": config["expected_model_experiment"], + "selected_calibration_method": config["expected_selected_calibration_method"], + "artifacts": sorted(_relative(root, path) for path in (model_path, ranked_path, top_path, manifest_path, template_path, worksheet_path)), + "hashes": {"model": True, "ranked": True, "top_10": True, "monitoring": True}, + "safety_flags": flags, + } + + +def validate_release(root: Path | str = ".", mode: str = "ci") -> dict[str, Any]: + """Return a deterministic, read-only validation report for ``mode``.""" + root = Path(root).resolve() + _require(mode in {"ci", "local"}, "mode must be either 'ci' or 'local'") + started = time.monotonic() + config = _load_config(root) + checks, reports = _ci_checks(root, config) + local = {"required": False, "valid": True, "artifacts": "not required in CI mode"} + if mode == "local": + local = _local_checks(root, config, reports) + return { + "status": "PASS", + "mode": mode, + "release_validation_version": config["release_validation_version"], + "python_version": platform.python_version(), + "platform": platform.system(), + "source_snapshot_id": local.get("source_snapshot_id"), + "feature_version": local.get("feature_version"), + "selected_experiment": local.get("selected_experiment"), + "selected_calibration_method": local.get("selected_calibration_method"), + "required_file_checks": checks["required_files"], + "configuration_checks": checks["configuration"], + "report_checks": checks["reports"], + "artifact_checks": local, + "hash_checks": local.get("hashes", {"not_required": mode == "ci"}), + "ci_workflow_checks": checks["workflow"], + "docker_contract_checks": checks["docker"], + "streamlit_import_check": checks["streamlit_import"], + "locked_candidate_safety_checks": local.get("safety_flags", {"not_required": mode == "ci"}), + "governance_checks": {"human_review_fields_checked": mode == "local", "future_template_headers_only": mode == "local"}, + "monitoring_safety_checks": {"monitoring_artifacts_checked": mode == "local", "health_independent_from_pipeline": mode == "local"}, + "ignored_artifact_policy_checks": checks["ignored_artifact_policy"], + "warnings": ["Local artifacts are intentionally not required in CI mode."] if mode == "ci" else [], + "failures": [], + "runtime_seconds": round(time.monotonic() - started, 6), + "limitations": [ + "Candidate ranking supports human review; there is no automatic enforcement.", + "The 2023 candidate batch is awaiting complete outcome labels.", + "This release validation does not load labels, calculate performance or fairness metrics, fit models, or regenerate predictions.", + "The selected final candidate uses uncalibrated model output and retrospective validation only.", + ], + } + + +def write_json_atomic(path: Path, value: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.replace(temporary, path) diff --git a/tests/test_release.py b/tests/test_release.py new file mode 100644 index 0000000..decebc9 --- /dev/null +++ b/tests/test_release.py @@ -0,0 +1,154 @@ +"""Focused, synthetic tests for the read-only Day 7A release validator.""" +from __future__ import annotations + +import csv +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +import yaml + +from src.governance import REVIEW_COLUMNS, future_outcome_template +from src.release_validation import ReleaseValidationError, validate_release + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] + + +def _digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +class ReleaseValidationTests(unittest.TestCase): + def _write(self, root: Path, name: str, text: str) -> Path: + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + def _fixture(self, *, local: bool = False) -> Path: + temporary = Path(tempfile.mkdtemp()) + config = yaml.safe_load((PROJECT_ROOT / "config" / "release_config.yaml").read_text(encoding="utf-8")) + self._write(temporary, "config/release_config.yaml", yaml.safe_dump(config, sort_keys=False)) + self._write(temporary, "config/dashboard_config.yaml", "dashboard: true\n") + self._write(temporary, "config/monitoring_config.yaml", "monitoring: true\n") + for name in ("app/streamlit_app.py", "src/governance.py", "src/monitoring.py", "src/release_validation.py", "run_release_validation.py"): + self._write(temporary, name, '"""human review; no outcomes are created."""\n') + self._write( + temporary, + "Dockerfile", + "FROM python:3.13-slim\nENV PYTHONPATH=/app\nUSER appuser\nEXPOSE 8501\nCMD [\"python\", \"-m\", \"streamlit\", \"run\", \"app/streamlit_app.py\", \"--server.address=0.0.0.0\"]\n", + ) + self._write(temporary, ".dockerignore", ".env\nartifacts\ndata/raw\nmlflow.db\nmlartifacts\n") + self._write( + temporary, + ".github/workflows/ci.yml", + "name: CI\non:\n push:\n pull_request:\n workflow_dispatch:\npermissions:\n contents: read\njobs:\n verify:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/setup-python@v5\n with:\n python-version: '3.13'\n - run: python -m compileall app src scripts tests *.py\n - run: python -m unittest discover -s tests -t . -v\n - run: python run_release_validation.py --mode ci\n - run: docker build .\n", + ) + names = config["required_committed_reports"] + reports = {name: {"status": "PASS"} for name in names} + reports["reports/feasibility_report.json"] = {"decision": "GO"} + if local: + reports.update(self._local_reports(temporary)) + for name, report in reports.items(): + self._write(temporary, name, json.dumps(report)) + self.addCleanup(lambda: __import__("shutil").rmtree(temporary, ignore_errors=True)) + return temporary + + def _local_reports(self, root: Path) -> dict[str, dict]: + snapshot, version = "snapshot-1", "day2-historical-v1" + model = self._write(root, "artifacts/models/final_candidate.joblib", "frozen model") + ranked = root / "artifacts/predictions/ranked.csv" + top = root / "artifacts/predictions/top.csv" + ranked.parent.mkdir(parents=True, exist_ok=True) + headers = ["activity_nr", "advisory_score", "rank"] + with ranked.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=headers); writer.writeheader() + for number in range(300): writer.writerow({"activity_nr": str(number), "advisory_score": "0.2", "rank": str(number + 1)}) + with top.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=headers); writer.writeheader() + for number in range(30): writer.writerow({"activity_nr": str(number), "advisory_score": "0.2", "rank": str(number + 1)}) + monitoring = root / "artifacts/monitoring"; monitoring.mkdir(parents=True, exist_ok=True) + manifest = self._write(root, "artifacts/monitoring/monitoring_manifest.json", "{}") + template = monitoring / "future_outcome_template.csv"; future_outcome_template().to_csv(template, index=False) + worksheet = monitoring / "review_queue_template.csv" + with worksheet.open("w", newline="", encoding="utf-8") as handle: csv.DictWriter(handle, fieldnames=REVIEW_COLUMNS).writeheader() + shared = {"source_snapshot_id": snapshot, "feature_version": version} + return { + "reports/feature_engineering_report.json": {"status": "PASS", **shared}, + "reports/model_comparison_report.json": {"status": "PASS", **shared, "selected_candidate": {"experiment_name": "exp_05_random_forest"}}, + "reports/calibration_report.json": {"status": "PASS", **shared, "selected_calibration_method": "uncalibrated", "final_candidate_artifact_path": "artifacts/models/final_candidate.joblib"}, + "reports/mlflow_tracking_report.json": {"status": "PASS", "logged_day3_run_count": 8, "logged_day4_run_count": 3, "final_candidate_artifact_logged": True, "reused_run_count": 11, "newly_created_run_count": 0}, + "reports/batch_prediction_report.json": {"status": "PASS", **shared, "selected_day3_experiment": "exp_05_random_forest", "selected_day4_method": "uncalibrated", "model_artifact_hash": _digest(model), "ranked_output_path": "artifacts/predictions/ranked.csv", "ranked_output_hash": _digest(ranked), "top_10_output_path": "artifacts/predictions/top.csv", "top_10_output_hash": _digest(top), "labels_accessed": False, "performance_metrics_calculated": False, "automatic_enforcement": False}, + "reports/dashboard_validation_report.json": {"status": "PASS", **shared, "model_refit_attempted": False}, + "reports/monitoring_report.json": {"status": "PASS", **shared, "monitoring_health": "WARNING", "monitoring_manifest_path": "artifacts/monitoring/monitoring_manifest.json", "future_outcome_template_path": "artifacts/monitoring/future_outcome_template.csv", "future_outcome_template_hash": _digest(template), "review_worksheet_path": "artifacts/monitoring/review_queue_template.csv", "review_worksheet_hash": _digest(worksheet), "current_labels_accessed": False, "current_performance_metrics_calculated": False, "outcome_fairness_metrics_calculated": False, "automatic_enforcement": False, "model_refit_attempted": False, "prediction_artifact_modified": False}, + } + + def test_ci_mode_needs_no_generated_artifacts_or_mutation(self) -> None: + root = self._fixture() + before = sorted(path.relative_to(root).as_posix() for path in root.rglob("*")) + report = validate_release(root, "ci") + after = sorted(path.relative_to(root).as_posix() for path in root.rglob("*")) + self.assertEqual("PASS", report["status"]) + self.assertEqual("not required in CI mode", report["artifact_checks"]["artifacts"]) + self.assertEqual(before, after) + + def test_ci_rejects_workflow_secret_and_missing_docker_contract(self) -> None: + root = self._fixture() + workflow = root / ".github/workflows/ci.yml" + workflow.write_text(workflow.read_text(encoding="utf-8") + "\n - run: ${{ secrets.BAD }}\n", encoding="utf-8") + with self.assertRaisesRegex(ReleaseValidationError, "CI workflow"): + validate_release(root, "ci") + + def test_ci_reports_invalid_json_clearly(self) -> None: + root = self._fixture() + (root / "reports/baseline_report.json").write_text("{broken", encoding="utf-8") + with self.assertRaisesRegex(ReleaseValidationError, "Invalid JSON report"): + validate_release(root, "ci") + + def test_ci_requires_python_313_and_required_steps(self) -> None: + root = self._fixture() + workflow = root / ".github/workflows/ci.yml" + workflow.write_text(workflow.read_text(encoding="utf-8").replace("3.13", "3.12"), encoding="utf-8") + with self.assertRaisesRegex(ReleaseValidationError, "CI workflow"): + validate_release(root, "ci") + + def test_ci_requires_dockerfile_and_dockerignore(self) -> None: + root = self._fixture() + (root / ".dockerignore").unlink() + with self.assertRaisesRegex(ReleaseValidationError, "Required source files missing"): + validate_release(root, "ci") + + def test_local_validates_rows_hashes_and_warning_health(self) -> None: + root = self._fixture(local=True) + report = validate_release(root, "local") + self.assertEqual("PASS", report["status"]) + self.assertEqual("WARNING", _read_report(root, "monitoring_report.json")["monitoring_health"]) + self.assertTrue(report["artifact_checks"]["hashes"]["ranked"]) + + def test_local_rejects_target_field_and_preserves_frozen_outputs(self) -> None: + root = self._fixture(local=True) + ranked = root / "artifacts/predictions/ranked.csv" + original = ranked.read_text(encoding="utf-8") + ranked.write_text(original.replace("rank\n", "rank,label\n", 1), encoding="utf-8") + with self.assertRaises(ReleaseValidationError): + validate_release(root, "local") + self.assertEqual(original.replace("rank\n", "rank,label\n", 1), ranked.read_text(encoding="utf-8")) + + def test_local_rejects_mismatched_snapshot_and_safety_flag(self) -> None: + root = self._fixture(local=True) + report = _read_report(root, "batch_prediction_report.json") + report["source_snapshot_id"] = "different" + (root / "reports/batch_prediction_report.json").write_text(json.dumps(report), encoding="utf-8") + with self.assertRaisesRegex(ReleaseValidationError, "Source snapshot IDs"): + validate_release(root, "local") + + +def _read_report(root: Path, name: str) -> dict: + return json.loads((root / "reports" / name).read_text(encoding="utf-8")) + + +if __name__ == "__main__": + unittest.main()