From b19e7becbe34bb208e6e802b872d66154a2f9874 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 24 Feb 2026 12:24:29 +0100 Subject: [PATCH 01/53] feat(fusion-doctor): Add Python report generator and HTML template Signed-off-by: Alberto Miranda --- assets/templates/fusion_report_template.html | 437 +++++++++++++++++++ bin/generate_fusion_report.py | 225 ++++++++++ 2 files changed, 662 insertions(+) create mode 100644 assets/templates/fusion_report_template.html create mode 100755 bin/generate_fusion_report.py diff --git a/assets/templates/fusion_report_template.html b/assets/templates/fusion_report_template.html new file mode 100644 index 0000000..6752bcc --- /dev/null +++ b/assets/templates/fusion_report_template.html @@ -0,0 +1,437 @@ + + + + + + Fusion Diagnostic Report + + + +
+
+

Fusion Diagnostic Report

+
+
+ Generated + {{ timestamp }} +
+
+ Status + {{ overall_status }} +
+
+
+ + + {% if doctor_report %} +
+
+

System & Validation

+ ▼ +
+
+ {% if doctor_report.summary %} +
+
+
Overall Status
+
+ {{ doctor_report.summary.status }} +
+
+ {% if doctor_report.version %} +
+
Fusion Version
+
{{ doctor_report.version }}
+
+ {% endif %} +
+ {% endif %} + + {% if doctor_report.checks %} +

Checks

+ {% for check_name, check_data in doctor_report.checks.items() %} +
+
+ + {% if check_data.status == 'pass' %}✓{% elif check_data.status == 'warn' %}⚠{% else %}✗{% endif %} + + {{ check_name|title }} +
+
{{ check_data.message }}
+ {% if check_data.details %} +
+ Details: + 
{{ check_data.details|tojson(indent=2) }}
+
+ {% endif %} + {% if check_data.remediation %} +
+ Remediation: {{ check_data.remediation }} +
+ {% endif %} +
+ {% endfor %} + {% endif %} +
+
+ {% endif %} + + + {% if bench_report and bench_report.keys()|length > 0 and 'error' not in bench_report %} +
+
+

Filesystem Benchmark

+ ▼ +
+
+ {% if bench_report.summary %} +
+
+
Status
+
+ {{ bench_report.summary.status }} +
+
+
+ {% endif %} + + {% if bench_report.results %} +
+ 
{{ bench_report.results|tojson(indent=2) }}
+
+ {% elif bench_report.error %} +
+
Error: {{ bench_report.error }}
+
+ {% else %} +

No benchmark results available

+ {% endif %} +
+
+ {% endif %} + + + {% if objbench_report and objbench_report.keys()|length > 0 and 'error' not in objbench_report %} +
+
+

Object Storage Benchmark

+ ▼ +
+
+ {% if objbench_report.summary %} +
+
+
Status
+
+ {{ objbench_report.summary.status }} +
+
+
+ {% endif %} + + {% if objbench_report.results %} +
+ 
{{ objbench_report.results|tojson(indent=2) }}
+
+ {% elif objbench_report.error %} +
+
Error: {{ objbench_report.error }}
+
+ {% else %} +

No benchmark results available

+ {% endif %} +
+
+ {% endif %} + +
+ Fusion Diagnostic Report • Generated on {{ timestamp }} +
+
+ + + + diff --git a/bin/generate_fusion_report.py b/bin/generate_fusion_report.py new file mode 100755 index 0000000..b746fbe --- /dev/null +++ b/bin/generate_fusion_report.py @@ -0,0 +1,225 @@ +#!/usr/bin/env -S uv run --script + +# /// script +# requires-python = ">=3.12" +# dependencies = ["jinja2"] +# /// + +""" +Generate a consolidated Fusion diagnostic report from doctor/bench/objbench outputs. +Produces both JSON and self-contained HTML reports. +""" + +import json +import sys +from pathlib import Path +from datetime import datetime, timezone +from typing import Dict, Any, Optional + + +def load_json_report(path: Optional[str]) -> Dict[str, Any]: + """Load a JSON report file, return empty dict if path is None. + + Args: + path: Path to JSON report file, or None + + Returns: + Parsed JSON dictionary, or dict with "error" key if loading fails + """ + if not path: + return {} + + try: + with open(path, 'r') as f: + return json.load(f) + except FileNotFoundError: + return {"error": f"Report file not found: {path}"} + except json.JSONDecodeError as e: + return {"error": f"Malformed JSON in {path}: {str(e)}"} + except IOError as e: + return {"error": f"Cannot read {path}: {str(e)}"} + + +def merge_reports( + doctor_report: Optional[str] = None, + bench_report: Optional[str] = None, + objbench_report: Optional[str] = None, +) -> Dict[str, Any]: + """ + Merge individual diagnostic reports into a single combined report. + + Args: + doctor_report: Path to fusion doctor JSON output + bench_report: Path to fusion bench JSON output + objbench_report: Path to fusion objbench JSON output + + Returns: + Merged report dictionary + """ + combined = { + "timestamp": datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z'), + "reports": { + "doctor": load_json_report(doctor_report), + "bench": load_json_report(bench_report), + "objbench": load_json_report(objbench_report), + }, + } + + # Compute overall status (fail > warn > pass) + statuses = [] + for report in combined["reports"].values(): + if report and "error" not in report: + if "summary" in report: + statuses.append(report["summary"].get("status", "unknown")) + # Note: Reports without "summary" are silently ignored in status aggregation + + if "fail" in statuses: + combined["overall_status"] = "fail" + elif "warn" in statuses: + combined["overall_status"] = "warn" + else: + combined["overall_status"] = "pass" + + return combined + + +def render_html(combined_report: Dict[str, Any], template_str: Optional[str] = None) -> str: + """ + Render self-contained HTML report from combined data. + + Args: + combined_report: Merged diagnostic report dictionary + template_str: The HTML template string (pre-loaded for robustness). + If None, will be loaded at call time (slower, for backward compatibility). + + Returns: + HTML string with inline CSS/JS + """ + from jinja2 import Template + + # Load template if not provided (for backward compatibility with tests) + if template_str is None: + template_str = load_template() + + template = Template(template_str) + + html = template.render( + timestamp=combined_report.get("timestamp", ""), + overall_status=combined_report.get("overall_status", "unknown"), + doctor_report=combined_report.get("reports", {}).get("doctor", {}), + bench_report=combined_report.get("reports", {}).get("bench", {}), + objbench_report=combined_report.get("reports", {}).get("objbench", {}), + ) + + return html + + +def load_template() -> str: + """Load the HTML template from assets/templates directory. + + Returns: + HTML template string + + Raises: + FileNotFoundError: If template file is not found + IOError: If template file cannot be read + """ + # Go up from bin/ to project root, then into assets/templates + template_path = Path(__file__).parent.parent / "assets" / "templates" / "fusion_report_template.html" + try: + with open(template_path, 'r') as f: + return f.read() + except FileNotFoundError: + raise FileNotFoundError( + f"HTML template not found at {template_path}. " + "Ensure fusion_report_template.html is in 'assets/templates/' directory." + ) + except IOError as e: + raise IOError(f"Cannot read template file {template_path}: {str(e)}") + + +def main(): + """Main entry point for report generation.""" + import argparse + + parser = argparse.ArgumentParser( + description="Generate consolidated Fusion diagnostic HTML report" + ) + parser.add_argument( + "--doctor", type=str, required=False, + help="Path to fusion doctor JSON report" + ) + parser.add_argument( + "--bench", type=str, required=False, + help="Path to fusion bench JSON report" + ) + parser.add_argument( + "--objbench", type=str, required=False, + help="Path to fusion objbench JSON report" + ) + parser.add_argument( + "--output-html", type=str, default="fusion_report.html", + help="Output path for HTML report (default: fusion_report.html)" + ) + parser.add_argument( + "--output-json", type=str, default="fusion_report.json", + help="Output path for combined JSON report (default: fusion_report.json)" + ) + + args = parser.parse_args() + + # Validate that at least one report was provided + if not any([args.doctor, args.bench, args.objbench]): + print( + "ERROR: At least one report must be provided. Use '--doctor', '--bench', or '--objbench'.", + file=sys.stderr + ) + sys.exit(1) + + # Load template at startup for early failure detection + try: + template_str = load_template() + except (FileNotFoundError, IOError) as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + sys.exit(1) + + # Merge all reports + combined = merge_reports( + doctor_report=args.doctor, + bench_report=args.bench, + objbench_report=args.objbench, + ) + + # Write combined JSON + try: + with open(args.output_json, 'w') as f: + json.dump(combined, f, indent=2) + except IOError as e: + print(f"ERROR: Failed to write JSON report to {args.output_json}: {str(e)}", file=sys.stderr) + sys.exit(1) + + # Render and write HTML + try: + html = render_html(combined, template_str) + with open(args.output_html, 'w') as f: + f.write(html) + except IOError as e: + print(f"ERROR: Failed to write HTML report to {args.output_html}: {str(e)}", file=sys.stderr) + sys.exit(1) + except Exception as e: + print(f"ERROR: Failed to render HTML report: {str(e)}", file=sys.stderr) + sys.exit(1) + + print(f"Reports generated:") + print(f" HTML: {args.output_html}") + print(f" JSON: {args.output_json}") + overall_status = combined.get("overall_status", "unknown") + print(f" Status: {overall_status.upper()}") + + # Exit with appropriate code based on status + exit_codes = {"pass": 0, "warn": 0, "fail": 1, "unknown": 1} + sys.exit(exit_codes.get(overall_status, 1)) + + +if __name__ == "__main__": + main() From 05b3ff745a7ace7027097a2ab0f1794e70ef5e4b Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 24 Feb 2026 15:33:39 +0100 Subject: [PATCH 02/53] test(fusion-doctor): Add unit tests for report generation Signed-off-by: Alberto Miranda --- tests/test_generate_fusion_report.py | 439 +++++++++++++++++++++++++++ 1 file changed, 439 insertions(+) create mode 100755 tests/test_generate_fusion_report.py diff --git a/tests/test_generate_fusion_report.py b/tests/test_generate_fusion_report.py new file mode 100755 index 0000000..585fe94 --- /dev/null +++ b/tests/test_generate_fusion_report.py @@ -0,0 +1,439 @@ +#!/usr/bin/env -S uv run --script + +# /// script +# requires-python = ">=3.12" +# dependencies = ["pytest", "jinja2"] +# /// + +""" +Unit tests for generate_fusion_report.py +Tests load_json_report, merge_reports, render_html, and status aggregation logic. +""" + +import json +import tempfile +from pathlib import Path +import sys +import os + +# Add parent directory to path to import generate_fusion_report +sys.path.insert(0, str(Path(__file__).parent.parent / "bin")) + +import pytest +from generate_fusion_report import ( + load_json_report, + merge_reports, + render_html, + load_template, +) + + +class TestLoadJsonReport: + """Test cases for load_json_report function.""" + + def test_load_valid_json(self): + """Test loading a valid JSON file.""" + test_data = {"status": "pass", "message": "All checks passed"} + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f: + json.dump(test_data, f) + temp_path = f.name + + try: + result = load_json_report(temp_path) + assert result == test_data + finally: + os.unlink(temp_path) + + def test_load_none_path(self): + """Test with None path returns empty dict.""" + result = load_json_report(None) + assert result == {} + + def test_load_empty_string_path(self): + """Test with empty string path returns empty dict.""" + result = load_json_report("") + assert result == {} + + def test_load_missing_file(self): + """Test loading non-existent file.""" + result = load_json_report("/nonexistent/path/file.json") + assert "error" in result + assert "not found" in result["error"] + + def test_load_invalid_json(self): + """Test loading malformed JSON.""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f: + f.write("{invalid json content") + temp_path = f.name + + try: + result = load_json_report(temp_path) + assert "error" in result + assert "Malformed JSON" in result["error"] + finally: + os.unlink(temp_path) + + def test_load_empty_json_file(self): + """Test loading empty JSON file.""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f: + f.write("") + temp_path = f.name + + try: + result = load_json_report(temp_path) + assert "error" in result + finally: + os.unlink(temp_path) + + def test_load_complex_json(self): + """Test loading complex nested JSON structure.""" + test_data = { + "summary": {"status": "warn"}, + "checks": { + "check1": {"status": "pass", "message": "OK"}, + "check2": {"status": "fail", "message": "Failed"}, + }, + "version": "v1.5.0", + } + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f: + json.dump(test_data, f) + temp_path = f.name + + try: + result = load_json_report(temp_path) + assert result == test_data + assert result["summary"]["status"] == "warn" + finally: + os.unlink(temp_path) + + +class TestMergeReports: + """Test cases for merge_reports function.""" + + def test_merge_all_pass(self): + """Test merging reports with all pass status.""" + doctor = {"summary": {"status": "pass"}} + bench = {"summary": {"status": "pass"}} + objbench = {"summary": {"status": "pass"}} + + with tempfile.TemporaryDirectory() as tmpdir: + doctor_path = Path(tmpdir) / "doctor.json" + bench_path = Path(tmpdir) / "bench.json" + objbench_path = Path(tmpdir) / "objbench.json" + + doctor_path.write_text(json.dumps(doctor)) + bench_path.write_text(json.dumps(bench)) + objbench_path.write_text(json.dumps(objbench)) + + result = merge_reports(str(doctor_path), str(bench_path), str(objbench_path)) + assert result["overall_status"] == "pass" + + def test_merge_with_warning(self): + """Test that warn status takes precedence over pass.""" + doctor = {"summary": {"status": "pass"}} + bench = {"summary": {"status": "warn"}} + objbench = {"summary": {"status": "pass"}} + + with tempfile.TemporaryDirectory() as tmpdir: + doctor_path = Path(tmpdir) / "doctor.json" + bench_path = Path(tmpdir) / "bench.json" + objbench_path = Path(tmpdir) / "objbench.json" + + doctor_path.write_text(json.dumps(doctor)) + bench_path.write_text(json.dumps(bench)) + objbench_path.write_text(json.dumps(objbench)) + + result = merge_reports(str(doctor_path), str(bench_path), str(objbench_path)) + assert result["overall_status"] == "warn" + + def test_merge_with_failure(self): + """Test that fail status takes precedence over warn and pass.""" + doctor = {"summary": {"status": "fail"}} + bench = {"summary": {"status": "warn"}} + objbench = {"summary": {"status": "pass"}} + + with tempfile.TemporaryDirectory() as tmpdir: + doctor_path = Path(tmpdir) / "doctor.json" + bench_path = Path(tmpdir) / "bench.json" + objbench_path = Path(tmpdir) / "objbench.json" + + doctor_path.write_text(json.dumps(doctor)) + bench_path.write_text(json.dumps(bench)) + objbench_path.write_text(json.dumps(objbench)) + + result = merge_reports(str(doctor_path), str(bench_path), str(objbench_path)) + assert result["overall_status"] == "fail" + + def test_merge_with_no_reports(self): + """Test merging with no reports provided.""" + result = merge_reports(None, None, None) + assert result["overall_status"] == "pass" + assert result["reports"]["doctor"] == {} + assert result["reports"]["bench"] == {} + assert result["reports"]["objbench"] == {} + + def test_merge_partial_reports(self): + """Test merging with only some reports provided.""" + doctor = {"summary": {"status": "pass"}} + + with tempfile.TemporaryDirectory() as tmpdir: + doctor_path = Path(tmpdir) / "doctor.json" + doctor_path.write_text(json.dumps(doctor)) + + result = merge_reports(str(doctor_path), None, None) + assert result["overall_status"] == "pass" + assert result["reports"]["doctor"] == doctor + assert result["reports"]["bench"] == {} + assert result["reports"]["objbench"] == {} + + def test_merge_with_invalid_report(self): + """Test merging when one report file is invalid.""" + doctor = {"summary": {"status": "pass"}} + bench_invalid_path = "/nonexistent/bench.json" + + with tempfile.TemporaryDirectory() as tmpdir: + doctor_path = Path(tmpdir) / "doctor.json" + doctor_path.write_text(json.dumps(doctor)) + + result = merge_reports(str(doctor_path), bench_invalid_path, None) + assert result["overall_status"] == "pass" + assert result["reports"]["doctor"] == doctor + assert "error" in result["reports"]["bench"] + + def test_merge_timestamp_present(self): + """Test that merged report includes timestamp.""" + result = merge_reports(None, None, None) + assert "timestamp" in result + assert result["timestamp"].endswith("Z") + + def test_status_aggregation_priority(self): + """Test that status aggregation follows priority: fail > warn > pass.""" + test_cases = [ + (["pass", "pass", "pass"], "pass"), + (["pass", "warn", "pass"], "warn"), + (["pass", "fail", "pass"], "fail"), + (["warn", "fail"], "fail"), + (["fail", "fail", "fail"], "fail"), + (["pass"], "pass"), + (["warn"], "warn"), + (["fail"], "fail"), + ] + + for statuses, expected in test_cases: + with tempfile.TemporaryDirectory() as tmpdir: + paths = [] + for i, status in enumerate(statuses): + data = {"summary": {"status": status}} + path = Path(tmpdir) / f"report{i}.json" + path.write_text(json.dumps(data)) + paths.append(str(path)) + + # Pad with None for missing reports + while len(paths) < 3: + paths.append(None) + + result = merge_reports(*paths) + assert result["overall_status"] == expected, \ + f"Failed for statuses {statuses}: got {result['overall_status']}, expected {expected}" + + +class TestRenderHtml: + """Test cases for render_html function.""" + + def test_render_basic_html(self): + """Test basic HTML rendering.""" + combined_report = { + "timestamp": "2026-02-24T10:00:00Z", + "overall_status": "pass", + "reports": { + "doctor": { + "summary": {"status": "pass"}, + "checks": { + "fuse_device": {"status": "pass", "message": "FUSE available"} + }, + }, + "bench": {}, + "objbench": {}, + }, + } + + html = render_html(combined_report) + assert isinstance(html, str) + assert "Fusion Diagnostic Report" in html + assert "2026-02-24T10:00:00Z" in html + assert "pass" in html.lower() + + def test_render_with_status_badge(self): + """Test that status badges are rendered correctly.""" + combined_report = { + "timestamp": "2026-02-24T10:00:00Z", + "overall_status": "warn", + "reports": { + "doctor": { + "summary": {"status": "warn"}, + "checks": {}, + }, + "bench": {}, + "objbench": {}, + }, + } + + html = render_html(combined_report) + assert "warn" in html.lower() or "status-warn" in html + + def test_render_with_doctor_checks(self): + """Test rendering with doctor report checks.""" + combined_report = { + "timestamp": "2026-02-24T10:00:00Z", + "overall_status": "pass", + "reports": { + "doctor": { + "summary": {"status": "pass"}, + "version": "v1.5.0", + "checks": { + "memory": { + "status": "pass", + "message": "32GB available", + }, + "disk": { + "status": "warn", + "message": "Low disk space", + }, + }, + }, + "bench": {}, + "objbench": {}, + }, + } + + html = render_html(combined_report) + assert "v1.5.0" in html + assert "memory" in html.lower() or "Memory" in html + assert "32GB" in html + + def test_render_empty_reports(self): + """Test rendering with empty reports.""" + combined_report = { + "timestamp": "2026-02-24T10:00:00Z", + "overall_status": "pass", + "reports": { + "doctor": {}, + "bench": {}, + "objbench": {}, + }, + } + + html = render_html(combined_report) + assert isinstance(html, str) + assert len(html) > 0 + + def test_render_html_structure(self): + """Test that rendered HTML contains basic HTML structure.""" + combined_report = { + "timestamp": "2026-02-24T10:00:00Z", + "overall_status": "pass", + "reports": { + "doctor": {"summary": {"status": "pass"}}, + "bench": {}, + "objbench": {}, + }, + } + + html = render_html(combined_report) + assert "" in html.lower() + assert "" in html.lower() + + +class TestLoadTemplate: + """Test cases for load_template function.""" + + def test_load_template_exists(self): + """Test that template file is found and loaded.""" + template = load_template() + assert isinstance(template, str) + assert len(template) > 0 + + def test_load_template_contains_variables(self): + """Test that template contains Jinja2 template variables.""" + template = load_template() + assert "{{" in template or "{%" in template + + def test_load_template_is_html(self): + """Test that template is HTML content.""" + template = load_template() + assert "= 5.10", + }, + }, + } + + with tempfile.TemporaryDirectory() as tmpdir: + doctor_path = Path(tmpdir) / "doctor.json" + doctor_path.write_text(json.dumps(doctor_data)) + + # Merge + combined = merge_reports(str(doctor_path), None, None) + assert combined["overall_status"] == "pass" + + # Render + html = render_html(combined) + assert "v1.5.0" in html + assert "fuse" in html.lower() # Check name is rendered (case-insensitive) + + def test_full_workflow_with_multiple_reports(self): + """Test complete workflow with all three report types.""" + doctor_data = { + "summary": {"status": "pass"}, + "checks": {}, + } + bench_data = { + "summary": {"status": "warn"}, + "results": {"throughput": "5 GiB/s"}, + } + objbench_data = { + "summary": {"status": "pass"}, + "results": {}, + } + + with tempfile.TemporaryDirectory() as tmpdir: + doctor_path = Path(tmpdir) / "doctor.json" + bench_path = Path(tmpdir) / "bench.json" + objbench_path = Path(tmpdir) / "objbench.json" + + doctor_path.write_text(json.dumps(doctor_data)) + bench_path.write_text(json.dumps(bench_data)) + objbench_path.write_text(json.dumps(objbench_data)) + + # Merge + combined = merge_reports(str(doctor_path), str(bench_path), str(objbench_path)) + assert combined["overall_status"] == "warn" + + # Render + html = render_html(combined) + assert isinstance(html, str) + assert len(html) > 1000 # Should be substantial HTML + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From ab418fb8b1215384d858e132559834538e113a07 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 24 Feb 2026 17:15:22 +0100 Subject: [PATCH 03/53] feat(fusion-doctor): Add `FUSION_DOCTOR_GENERATE_REPORT` process Signed-off-by: Alberto Miranda --- main.nf | 35 +++++++++++- tests/main.nf.test | 42 ++++++++++----- ...n.test_fusion_doctor_empty_buckets.nf.test | 20 +++---- tests/main.test_fusion_doctor_report.nf.test | 53 +++++++++++++++++++ 4 files changed, 127 insertions(+), 23 deletions(-) create mode 100644 tests/main.test_fusion_doctor_report.nf.test diff --git a/main.nf b/main.nf index c2bdd41..32f8bac 100644 --- a/main.nf +++ b/main.nf @@ -419,6 +419,31 @@ process TEST_FUSION_DOCTOR { """ } +process FUSION_DOCTOR_GENERATE_REPORT { + /* + Aggregates doctor, bench, and objbench JSON reports into a single + consolidated HTML report and combined JSON report using the Python + generate_fusion_report.py script. + */ + + publishDir { params.outdir ?: file(workflow.workDir).resolve("outputs/fusion").toUriString() }, mode: 'copy' + + input: + path(doctor_report) + + output: + path("fusion_report.html"), emit: html_report + path("fusion_report.json"), emit: json_report + + script: + """ + generate_fusion_report.py \\ + --doctor ${doctor_report} \\ + --output-html fusion_report.html \\ + --output-json fusion_report.json + """ +} + workflow NF_CANARY { take: run_tools @@ -511,6 +536,12 @@ workflow NF_CANARY { TEST_FUSION_DOCTOR(run_ch.TEST_FUSION_DOCTOR, reference_profile_ch, rw_buckets_list, ro_buckets_list, params.fusion_cache_path) + // Generate consolidated fusion report from doctor output + // Only run FUSION_DOCTOR_GENERATE_REPORT if TEST_FUSION_DOCTOR produced output + FUSION_DOCTOR_GENERATE_REPORT( + TEST_FUSION_DOCTOR.out.report + ) + // POC of emitting the channel Channel.empty() .mix( @@ -530,7 +561,9 @@ workflow NF_CANARY { TEST_MV_FOLDER_CONTENTS.out, TEST_VAL_INPUT.out, TEST_GPU.out, - TEST_FUSION_DOCTOR.out + TEST_FUSION_DOCTOR.out, + FUSION_DOCTOR_GENERATE_REPORT.out.html_report.ifEmpty([]), + FUSION_DOCTOR_GENERATE_REPORT.out.json_report.ifEmpty([]) ) .set { ch_out } diff --git a/tests/main.nf.test b/tests/main.nf.test index 717683a..5f5179b 100644 --- a/tests/main.nf.test +++ b/tests/main.nf.test @@ -15,12 +15,18 @@ nextflow_pipeline{ then { assertAll( { assert workflow.success }, - { assert workflow.trace.tasks().size() == 16 }, - { assert workflow.trace.succeeded().size() == 15 }, + { assert workflow.trace.tasks().size() == 17 }, + { assert workflow.trace.succeeded().size() == 16 }, { assert workflow.trace.failed().size() == 1 }, - // Check fusion-doctor-report.json exists but don't snapshot its content (contains workDir paths) + // Check fusion reports exist but don't snapshot their content (contains workDir paths) { assert path(params.outdir).resolve("fusion-doctor-report.json").exists() }, - { assert snapshot(workflow, path(params.outdir).list().findAll { !it.toString().contains("fusion-doctor-report.json") }).match() } + { assert path(params.outdir).resolve("fusion_report.json").exists() }, + { assert path(params.outdir).resolve("fusion_report.html").exists() }, + { assert snapshot(workflow, path(params.outdir).list().findAll { + !it.toString().contains("fusion-doctor-report.json") && + !it.toString().contains("fusion_report.json") && + !it.toString().contains("fusion_report.html") + }).match() } ) } @@ -39,12 +45,18 @@ nextflow_pipeline{ then { assertAll( { assert workflow.success }, - { assert workflow.trace.tasks().size() == 15 }, - { assert workflow.trace.succeeded().size() == 14 }, + { assert workflow.trace.tasks().size() == 16 }, + { assert workflow.trace.succeeded().size() == 15 }, { assert workflow.trace.failed().size() == 1 }, - // Check fusion-doctor-report.json exists but don't snapshot its content (contains workDir paths) + // Check fusion reports exist but don't snapshot their content (contains workDir paths) { assert path(params.outdir).resolve("fusion-doctor-report.json").exists() }, - { assert snapshot(workflow, path(params.outdir).list().findAll { !it.toString().contains("fusion-doctor-report.json") }).match() } + { assert path(params.outdir).resolve("fusion_report.json").exists() }, + { assert path(params.outdir).resolve("fusion_report.html").exists() }, + { assert snapshot(workflow, path(params.outdir).list().findAll { + !it.toString().contains("fusion-doctor-report.json") && + !it.toString().contains("fusion_report.json") && + !it.toString().contains("fusion_report.html") + }).match() } ) } @@ -85,12 +97,18 @@ nextflow_pipeline{ then { assertAll( { assert workflow.success }, - { assert workflow.trace.tasks().size() == 16 }, - { assert workflow.trace.succeeded().size() == 15 }, + { assert workflow.trace.tasks().size() == 17 }, + { assert workflow.trace.succeeded().size() == 16 }, { assert workflow.trace.failed().size() == 1 }, - // Check fusion-doctor-report.json exists but don't snapshot its content (contains workDir paths) + // Check fusion reports exist but don't snapshot their content (contains workDir paths) { assert path("${launchDir}/output").resolve("fusion-doctor-report.json").exists() }, - { assert snapshot(workflow, path("${launchDir}/output").list().findAll { !it.toString().contains("fusion-doctor-report.json") }).match() } + { assert path("${launchDir}/output").resolve("fusion_report.json").exists() }, + { assert path("${launchDir}/output").resolve("fusion_report.html").exists() }, + { assert snapshot(workflow, path("${launchDir}/output").list().findAll { + !it.toString().contains("fusion-doctor-report.json") && + !it.toString().contains("fusion_report.json") && + !it.toString().contains("fusion_report.html") + }).match() } ) } diff --git a/tests/main.test_fusion_doctor_empty_buckets.nf.test b/tests/main.test_fusion_doctor_empty_buckets.nf.test index 81fc5db..34b2486 100644 --- a/tests/main.test_fusion_doctor_empty_buckets.nf.test +++ b/tests/main.test_fusion_doctor_empty_buckets.nf.test @@ -17,8 +17,8 @@ nextflow_pipeline { then { assertAll( { assert workflow.success }, - { assert workflow.trace.tasks().size() == 1 }, - { assert workflow.trace.succeeded().size() == 1 } + { assert workflow.trace.tasks().size() == 2 }, + { assert workflow.trace.succeeded().size() == 2 } ) } @@ -38,8 +38,8 @@ nextflow_pipeline { then { assertAll( { assert workflow.success }, - { assert workflow.trace.tasks().size() == 1 }, - { assert workflow.trace.succeeded().size() == 1 } + { assert workflow.trace.tasks().size() == 2 }, + { assert workflow.trace.succeeded().size() == 2 } ) } @@ -59,8 +59,8 @@ nextflow_pipeline { then { assertAll( { assert workflow.success }, - { assert workflow.trace.tasks().size() == 1 }, - { assert workflow.trace.succeeded().size() == 1 } + { assert workflow.trace.tasks().size() == 2 }, + { assert workflow.trace.succeeded().size() == 2 } ) } @@ -80,8 +80,8 @@ nextflow_pipeline { then { assertAll( { assert workflow.success }, - { assert workflow.trace.tasks().size() == 1 }, - { assert workflow.trace.succeeded().size() == 1 } + { assert workflow.trace.tasks().size() == 2 }, + { assert workflow.trace.succeeded().size() == 2 } ) } @@ -101,8 +101,8 @@ nextflow_pipeline { then { assertAll( { assert workflow.success }, - { assert workflow.trace.tasks().size() == 1 }, - { assert workflow.trace.succeeded().size() == 1 } + { assert workflow.trace.tasks().size() == 2 }, + { assert workflow.trace.succeeded().size() == 2 } ) } diff --git a/tests/main.test_fusion_doctor_report.nf.test b/tests/main.test_fusion_doctor_report.nf.test new file mode 100644 index 0000000..d50a526 --- /dev/null +++ b/tests/main.test_fusion_doctor_report.nf.test @@ -0,0 +1,53 @@ +nextflow_pipeline { + + name "Test FUSION_DOCTOR_GENERATE_REPORT conditional execution" + script "main.nf" + + test("Report process does NOT run when fusion is disabled") { + + when { + params { + fusion = false + run = 'TEST_SUCCESS' // Run a different test + outdir = "${launchDir}/output" + } + } + + then { + assertAll( + { assert workflow.success }, + // TEST_FUSION_DOCTOR should not run + { assert workflow.trace.tasks().findAll { it.name == 'NF_CANARY:TEST_FUSION_DOCTOR' }.size() == 0 }, + // FUSION_DOCTOR_GENERATE_REPORT should not run + { assert workflow.trace.tasks().findAll { it.name == 'NF_CANARY:FUSION_DOCTOR_GENERATE_REPORT' }.size() == 0 } + ) + } + + } + + test("Report process DOES run when fusion is enabled") { + + when { + params { + fusion = true + run = 'TEST_FUSION_DOCTOR' + outdir = "${launchDir}/output" + } + } + + then { + assertAll( + { assert workflow.success }, + // Both processes should run (2 tasks total) + { assert workflow.trace.tasks().size() == 2 }, + { assert workflow.trace.succeeded().size() == 2 }, + // Check that report files exist in outputs + { assert path("${launchDir}/output/fusion_report.html").exists() }, + { assert path("${launchDir}/output/fusion_report.json").exists() }, + { assert path("${launchDir}/output/fusion-doctor-report.json").exists() } + ) + } + + } + +} From b150c34bafe5efd414563fddfdf4ced33d7eded0 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 25 Feb 2026 07:50:05 +0100 Subject: [PATCH 04/53] chore: Update test snapshots Signed-off-by: Alberto Miranda --- tests/main.nf.test.snap | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/main.nf.test.snap b/tests/main.nf.test.snap index fdd5ded..4c78ed6 100644 --- a/tests/main.nf.test.snap +++ b/tests/main.nf.test.snap @@ -8,9 +8,9 @@ "errorReport": "", "failed": false, "trace": { - "tasksCount": 15, + "tasksCount": 16, "tasksFailed": 1, - "tasksSucceeded": 14 + "tasksSucceeded": 15 }, "stdout": [ @@ -28,7 +28,7 @@ "test.txt:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], - "timestamp": "2026-02-20T16:21:07.54100674", + "timestamp": "2026-02-25T07:46:12.113135429", "meta": { "nf-test": "0.9.4", "nextflow": "25.10.4" @@ -43,9 +43,9 @@ "errorReport": "", "failed": false, "trace": { - "tasksCount": 16, + "tasksCount": 17, "tasksFailed": 1, - "tasksSucceeded": 15 + "tasksSucceeded": 16 }, "stdout": [ @@ -63,7 +63,7 @@ "test.txt:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], - "timestamp": "2026-02-20T16:21:40.15320284", + "timestamp": "2026-02-25T07:46:19.211262425", "meta": { "nf-test": "0.9.4", "nextflow": "25.10.4" @@ -109,9 +109,9 @@ "errorReport": "", "failed": false, "trace": { - "tasksCount": 16, + "tasksCount": 17, "tasksFailed": 1, - "tasksSucceeded": 15 + "tasksSucceeded": 16 }, "stdout": [ @@ -129,7 +129,7 @@ "test.txt:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], - "timestamp": "2026-02-20T16:20:48.129435245", + "timestamp": "2026-02-25T07:46:08.101797557", "meta": { "nf-test": "0.9.4", "nextflow": "25.10.4" From 02a8dad510246d17f5e12bebe71515aaa89d811a Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 25 Feb 2026 07:55:58 +0100 Subject: [PATCH 05/53] ci: Integrate Python tests in CI pipelines Signed-off-by: Alberto Miranda --- .github/workflows/test.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 43829ab..422aafe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,6 +33,18 @@ jobs: - name: Run Prettier --check run: prettier --check . + python-tests: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v3 + + - name: Install uv + uses: astral-sh/setup-uv@v3 + + - name: Run generate_report tests + run: ./tests/test_generate_fusion_report.py + nf-test: runs-on: ubuntu-latest env: From fc6768eef28b8dfc1ef5f27be79a94ca89630ae1 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 25 Feb 2026 09:47:31 +0100 Subject: [PATCH 06/53] fix: Make `fusion_report_template.html` conformant with `prettier` Signed-off-by: Alberto Miranda --- assets/templates/fusion_report_template.html | 840 ++++++++++--------- 1 file changed, 433 insertions(+), 407 deletions(-) diff --git a/assets/templates/fusion_report_template.html b/assets/templates/fusion_report_template.html index 6752bcc..7979caa 100644 --- a/assets/templates/fusion_report_template.html +++ b/assets/templates/fusion_report_template.html @@ -1,437 +1,463 @@ - + - - - - Fusion Diagnostic Report - - - -
-
-

Fusion Diagnostic Report

-
-
- Generated - {{ timestamp }} + display: grid; + grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); + gap: 1rem; + margin-bottom: 1.5rem; + } + + .summary-card { + padding: 1rem; + background: #f5f6f7; + border: 1px solid #dadde1; + border-radius: 0.4rem; + } + + .summary-card-label { + font-size: 0.75rem; + color: #606770; + margin-bottom: 0.5rem; + font-weight: 500; + text-transform: uppercase; + letter-spacing: 0.02em; + } + + .summary-card-value { + font-size: 1.25rem; + font-weight: 700; + color: #1c1e21; + } + + h3 { + font-size: 1.1rem; + font-weight: 600; + color: #1c1e21; + margin: 1.5rem 0 1rem 0; + } + + footer { + text-align: center; + padding-top: 1.5rem; + border-top: 1px solid #dadde1; + color: #8d949e; + font-size: 0.85rem; + margin-top: 2rem; + } + + @media print { + body { + background: white; + } + .container { + max-width: none; + padding: 0; + } + .section-content { + display: block !important; + } + } + + @media (max-width: 600px) { + h1 { + font-size: 1.5rem; + } + .container { + padding: 1rem; + } + .metadata { + gap: 1rem; + } + .summary-grid { + grid-template-columns: 1fr; + } + } + + + +
+
+

+ + + + Fusion Diagnostic Report +

+
+
+ Generated + {{ timestamp }} +
+
+ Status + {{ overall_status }} +
-
- Status - {{ overall_status }} +
+ + + {% if doctor_report %} +
+
+

System & Validation

+ ▼
-
-
- - - {% if doctor_report %} -
-
-

System & Validation

- ▼ -
-
- {% if doctor_report.summary %} -
-
-
Overall Status
-
- {{ doctor_report.summary.status }} +
+ {% if doctor_report.summary %} +
+
+
Overall Status
+
+ {{ doctor_report.summary.status }} +
+ {% if doctor_report.version %} +
+
Fusion Version
+
{{ doctor_report.version }}
+
+ {% endif %}
- {% if doctor_report.version %} -
-
Fusion Version
-
{{ doctor_report.version }}
+ {% endif %} {% if doctor_report.checks %} +

Checks

+ {% for check_name, check_data in doctor_report.checks.items() %} +
+
+ + {% if check_data.status == 'pass' %}✓{% elif check_data.status == 'warn' %}⚠{% else + %}✗{% endif %} + + {{ check_name|title }} +
+
{{ check_data.message }}
+ {% if check_data.details %} +
+ Details: + 
{{ check_data.details|tojson(indent=2) }}
+
+ {% endif %} {% if check_data.remediation %} +
Remediation: {{ check_data.remediation }}
+ {% endif %}
- {% endif %} + {% endfor %} {% endif %} +
+
+ {% endif %} + + + {% if bench_report and bench_report.keys()|length > 0 and 'error' not in bench_report %} +
+
+

Filesystem Benchmark

+ ▼
- {% endif %} - - {% if doctor_report.checks %} -

Checks

- {% for check_name, check_data in doctor_report.checks.items() %} -
-
- - {% if check_data.status == 'pass' %}✓{% elif check_data.status == 'warn' %}⚠{% else %}✗{% endif %} - - {{ check_name|title }} +
+ {% if bench_report.summary %} +
+
+
Status
+
+ {{ bench_report.summary.status }} +
+
-
{{ check_data.message }}
- {% if check_data.details %} -
- Details: - 
{{ check_data.details|tojson(indent=2) }}
+ {% endif %} {% if bench_report.results %} +
+ 
+{{ bench_report.results|tojson(indent=2) }}
- {% endif %} - {% if check_data.remediation %} -
- Remediation: {{ check_data.remediation }} + {% elif bench_report.error %} +
+
Error: {{ bench_report.error }}
+ {% else %} +

No benchmark results available

{% endif %}
- {% endfor %} - {% endif %} -
-
- {% endif %} - - - {% if bench_report and bench_report.keys()|length > 0 and 'error' not in bench_report %} -
-
-

Filesystem Benchmark

- ▼ -
-
- {% if bench_report.summary %} -
-
-
Status
-
- {{ bench_report.summary.status }} -
-
-
- {% endif %} - - {% if bench_report.results %} -
- 
{{ bench_report.results|tojson(indent=2) }}
-
- {% elif bench_report.error %} -
-
Error: {{ bench_report.error }}
+
+ {% endif %} + + + {% if objbench_report and objbench_report.keys()|length > 0 and 'error' not in objbench_report %} +
+
+

Object Storage Benchmark

+ ▼
- {% else %} -

No benchmark results available

- {% endif %} -
- - {% endif %} - - - {% if objbench_report and objbench_report.keys()|length > 0 and 'error' not in objbench_report %} -
-
-

Object Storage Benchmark

- ▼ -
-
- {% if objbench_report.summary %} -
-
-
Status
-
- {{ objbench_report.summary.status }} +
+ {% if objbench_report.summary %} +
+
+
Status
+
+ {{ objbench_report.summary.status }} +
+ {% endif %} {% if objbench_report.results %} +
+ 
+{{ objbench_report.results|tojson(indent=2) }}
+
+ {% elif objbench_report.error %} +
+
Error: {{ objbench_report.error }}
+
+ {% else %} +

No benchmark results available

+ {% endif %}
- {% endif %} - - {% if objbench_report.results %} -
- 
{{ objbench_report.results|tojson(indent=2) }}
-
- {% elif objbench_report.error %} -
-
Error: {{ objbench_report.error }}
-
- {% else %} -

No benchmark results available

- {% endif %} -
-
- {% endif %} - -
- Fusion Diagnostic Report • Generated on {{ timestamp }} -
- - - - + + From 33ea6486456d0a106346b37abac7cd062ce4dbb9 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 25 Feb 2026 09:57:26 +0100 Subject: [PATCH 07/53] fix: Add `uv` container for `FUSION_DOCTOR_GENERATE_REPORT` Signed-off-by: Alberto Miranda --- main.nf | 1 + 1 file changed, 1 insertion(+) diff --git a/main.nf b/main.nf index 32f8bac..4cf8999 100644 --- a/main.nf +++ b/main.nf @@ -426,6 +426,7 @@ process FUSION_DOCTOR_GENERATE_REPORT { generate_fusion_report.py script. */ + container 'ghcr.io/astral-sh/uv:0.10.6' publishDir { params.outdir ?: file(workflow.workDir).resolve("outputs/fusion").toUriString() }, mode: 'copy' input: From 8d0d96d6b55104ff624561f8da6c48b4ef5b4cbd Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 25 Feb 2026 11:17:54 +0100 Subject: [PATCH 08/53] fix: Replace `uv:0.10.6` container with `uv:python3.12-bookworm-slim` Signed-off-by: Alberto Miranda --- main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.nf b/main.nf index 4cf8999..db57561 100644 --- a/main.nf +++ b/main.nf @@ -426,7 +426,7 @@ process FUSION_DOCTOR_GENERATE_REPORT { generate_fusion_report.py script. */ - container 'ghcr.io/astral-sh/uv:0.10.6' + container 'ghcr.io/astral-sh/uv:python3.12-bookworm-slim' publishDir { params.outdir ?: file(workflow.workDir).resolve("outputs/fusion").toUriString() }, mode: 'copy' input: From e814e4439533b7c1ee4867751d547b3524db7667 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 25 Feb 2026 14:47:57 +0100 Subject: [PATCH 09/53] ci: Add `setup-uv` to `nf-test` job Signed-off-by: Alberto Miranda --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 422aafe..95fcdfb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -52,6 +52,9 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Install uv + uses: astral-sh/setup-uv@v3 + - uses: actions/setup-java@v3 with: distribution: "temurin" From 53afe7ec307fb938561601c5fd43ce6061bafa8d Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 25 Feb 2026 15:44:02 +0100 Subject: [PATCH 10/53] fix: Replace `uv:python3.12-bookworm-slim` with `jinja2_python_uv:7113b0a0e59d95a6` Signed-off-by: Alberto Miranda --- main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.nf b/main.nf index db57561..3184d34 100644 --- a/main.nf +++ b/main.nf @@ -426,7 +426,7 @@ process FUSION_DOCTOR_GENERATE_REPORT { generate_fusion_report.py script. */ - container 'ghcr.io/astral-sh/uv:python3.12-bookworm-slim' + container 'community.wave.seqera.io/library/jinja2_python_uv:7113b0a0e59d95a6' publishDir { params.outdir ?: file(workflow.workDir).resolve("outputs/fusion").toUriString() }, mode: 'copy' input: From 047347baea02deceaa774ca93ea332af725a2819 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 27 Feb 2026 14:06:43 +0100 Subject: [PATCH 11/53] chore: Rename HTML report to `fusion-report.html` Signed-off-by: Alberto Miranda --- bin/generate_fusion_report.py | 4 ++-- tower.yml | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/bin/generate_fusion_report.py b/bin/generate_fusion_report.py index b746fbe..ba2c5f2 100755 --- a/bin/generate_fusion_report.py +++ b/bin/generate_fusion_report.py @@ -158,8 +158,8 @@ def main(): help="Path to fusion objbench JSON report" ) parser.add_argument( - "--output-html", type=str, default="fusion_report.html", - help="Output path for HTML report (default: fusion_report.html)" + "--output-html", type=str, default="fusion-report.html", + help="Output path for HTML report (default: fusion-report.html)" ) parser.add_argument( "--output-json", type=str, default="fusion_report.json", diff --git a/tower.yml b/tower.yml index 8a3dc2e..a496635 100644 --- a/tower.yml +++ b/tower.yml @@ -1,4 +1,8 @@ reports: - "fusion/fusion-doctor-report.json": - display: "Fusion Diagnostics Report" - mimeType: "text/plain" + "fusion/fusion-report.json": + display: "Fusion Filesystem Diagnostics Report (JSON)" + mimeType: "application/json" + + "fusion/fusion-report.html": + display: "Fusion Filesystem Diagnostics Report (HTML)" + mimeType: "text/html" From 09220c1673c35ad3837fa28c20f9f32889c47979 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 4 Mar 2026 11:49:36 +0100 Subject: [PATCH 12/53] feat(fusion-doctor): Split profiles into tiers Signed-off-by: Alberto Miranda --- README.md | 92 +++++++++++++++++++++++--- conf/fusion.config | 159 ++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 225 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index c71f3df..42f61b4 100644 --- a/README.md +++ b/README.md @@ -69,26 +69,38 @@ Each test includes a brief comment explaining its purpose. In case of failure, r This process should succeed automatically with exit status 0. +--- + ### `TEST_CREATE_FILE` This process creates a file on the worker machine and then moves it to the working directory. +--- + ### `TEST_CREATE_EMPTY_FILE` This process creates an empty file on the worker machine and then moves it to the working directory. +--- + ### `TEST_CREATE_FOLDER` This process creates a folder in the working directory. +--- + ### `TEST_INPUT` This process retrieves a file from the working directory and reads its contents on the worker machine. +--- + ### `TEST_BIN_SCRIPT` Tests a shell script in the `bin/` directory that creates a single file. +--- + ### `TEST_STAGE_REMOTE` _Note: Enabled only if the parameter `--remoteFile` is specified._ @@ -101,67 +113,127 @@ nextflow run seqeralabs/nf-canary --remoteFile 'https://raw.githubusercontent.co Use this parameter to specify a file to access during runtime. +--- + ### `TEST_PASS_FILE` This process stages a file from the working directory to the worker node, copies it, and stages it back to the working directory. +--- + ### `TEST_PASS_FOLDER` This process stages a folder from the working directory to the worker node, copies it, and stages it back to the working directory. +--- + ### `TEST_PUBLISH_FILE` This process creates a file on the worker machine and writes it to the publishDir directory. By default, this is written to a subfolder called `output` in the working directory, but it can be overridden using the `--outdir` parameter. Use this to demonstrate the ability to publish to the relevant output directory. +--- + ### `TEST_PUBLISH_FOLDER` This process creates a folder on the worker machine and writes it to the publishDir directory. By default, this is written to a subfolder called `output` in the working directory, but it can be overridden using the `--outdir` parameter. Use this to demonstrate the ability to publish to the relevant output directory. +--- + ### `TEST_IGNORED_FAIL` This process should fail immediately but be ignored using the default configuration. +--- + ### `TEST_MV_FILE` Tests moving a file within the working directory. +--- + ### `TEST_MV_FOLDER_CONTENTS` Tests moving the contents of a folder to a new folder within the working directory. +--- + ### `TEST_VAL_INPUT` Test a process can accept a value as input. +--- + ### `TEST_GPU` _Note: Enabled only if the parameter `--gpu` is specified._ This process tests the ability to use a GPU. It uses the `pytorch` conda environment to test CUDA is available and working. This is disabled by default as it requires a GPU to be available which may not be true. +--- + ### `TEST_FUSION_DOCTOR` _Note: Enabled only if the parameter `--fusion` is specified (or set to `true` by a profile)._ -This process runs `fusion doctor` to validate the Fusion filesystem configuration. It checks system requirements (kernel version, memory, and disk space), FUSE device availability, and cloud bucket accessibility. The process produces a JSON diagnostic report published to `${outdir}/fusion/`. - -This test requires the `fusion` binary to be available in the task environment and will fail if it is not found. +This process runs the `fusion doctor` diagnostics tool to validate the Fusion filesystem configuration. It checks system requirements (e.g. kernel version, memory, disk space, etc.), FUSE device availability, and cloud bucket accessibility among others. The process produces a JSON diagnostic report published to `${outdir}/fusion/`. #### Fusion Profiles -Use a built-in profile to enable Fusion validation with recommended thresholds: +Use a built-in profile to enable Fusion validation with predefined thresholds: ```bash nextflow run seqeralabs/nf-canary -profile fusion_aws_recommended ``` -Available profiles: +> [!TIP] +> +> **If in doubt, start with the `recommended` tier:** +> +> 1. Choose `low` for small workloads or quick smoke tests, and `high` for large-scale production with big datasets. +> 2. Choose `recommended` to match Seqera's documented minimum requirements for production workloads with Fusion. +> 3. Choose `high` (or a custom threshold of 400 GB+ storage) if your pipeline processes files larger than 100 GB. + +##### AWS + +> [!NOTE] +> +> Seqera Platform auto-selects NVMe-based instance families when Fusion is enabled (e.g. `m6id`, `c6id`, `r6id`). Fusion can also work without NVMe instances, but in this case the EBS disk shall be bumped to 100 GB (`gp3`, 325 MB/s); this is what the `low` profile validates. + +| Profile | Disk | Memory | Kernel | Based on | +| ------------------------ | ------ | ------ | ------ | ------------------- | +| `fusion_aws_low` | 100 GB | 4 GB | 5.10+ | EBS gp3 (no NVMe) | +| `fusion_aws_recommended` | 200 GB | 8 GB | 5.10+ | Seqera docs minimum | +| `fusion_aws_high` | 474 GB | 16 GB | 5.10+ | m6id.2xlarge NVMe | + +**Google Cloud** + +> [!NOTE] +> +> Seqera Platform auto-selects families supporting local SSDs (e.g. `n2`, `c2`, `n2d`) and provisions a 375 GB NVMe SSD per job. + +| Profile | Disk | Memory | Kernel | Based on | +| --------------------------- | ------ | ------ | ------ | ------------------- | +| `fusion_google_low` | 50 GB | 4 GB | 5.15+ | GCP persistent disk | +| `fusion_google_recommended` | 375 GB | 8 GB | 5.15+ | 1x local NVMe SSD | +| `fusion_google_high` | 750 GB | 16 GB | 5.15+ | 2x local NVMe SSDs | + +##### Azure + +> [!note] +> +> Unlike AWS and Google Cloud, there is no auto-selection: the user must pick the VM size. Seqera recommends E-series with a `d` suffix (e.g. `Standard_E8d_v5`, `Standard_E16d_v5`). + +| Profile | Disk | Memory | Kernel | Based on | +| -------------------------- | ------ | ------ | ------ | ---------------------------- | +| `fusion_azure_low` | 75 GB | 4 GB | 5.15+ | `Standard_E2d_v5` temp disk | +| `fusion_azure_recommended` | 300 GB | 8 GB | 5.15+ | `Standard_E8d_v5` temp disk | +| `fusion_azure_high` | 600 GB | 16 GB | 5.15+ | `Standard_E16d_v5` temp disk | + +#### Caveats + +In AWS Batch, the Seqera Platform UI allows selecting instance families (e.g. `m6id`) but not specific sizes. Small instances in an otherwise valid family may not meet the `recommended` disk threshold. For example, on AWS the `m6id` family NVMe ranges from 118 GB (`.large`) to 1,900 GB (`.8xlarge`), with only `.xlarge` and above meeting the 200 GB threshold. -| Profile | Description | -| --------------------------- | ----------------------------------------- | -| `fusion_aws_recommended` | AWS Batch recommended thresholds | -| `fusion_google_recommended` | Google Cloud Batch recommended thresholds | -| `fusion_azure_recommended` | Azure Batch recommended thresholds | +If `fusion doctor` reports a disk requirement failure, request more CPUs/memory to get a larger instance, or use the `low` profile for small tasks. #### Custom Requirements diff --git a/conf/fusion.config b/conf/fusion.config index 70c5508..d149aa2 100644 --- a/conf/fusion.config +++ b/conf/fusion.config @@ -1,43 +1,170 @@ /* * Fusion Validation Profiles * - * These profiles define recommended thresholds for different cloud environments. - * Parameters are internally used to build a reference profile that is passed - * to `fusion doctor`. + * These profiles define recommended thresholds for different cloud + * environments and workload sizes. Parameters are internally used to build + * a reference profile that is passed to `fusion doctor`. + * + * Tiers: + * low — small Nextflow workloads + * recommended — typical pipeline workloads + * high — large-scale production workloads * * Usage: nextflow run seqeralabs/nf-canary -profile fusion_aws_recommended + * + * --- Threshold review notes (2026-03-04) --- + * + * Sources: + * - https://docs.seqera.io/platform-cloud/compute-envs/aws-batch + * - https://docs.seqera.io/fusion/guide/aws-batch + * - https://docs.seqera.io/fusion/guide/gcp-batch + * - https://docs.seqera.io/platform-cloud/compute-envs/google-cloud-batch + * - https://docs.seqera.io/platform-cloud/compute-envs/azure-batch + * + * Seqera docs requirements for Fusion (all clouds): + * - Local temp storage: at least 200 GB, random read speed 1000 MBps+ + * - For files >100 GB: 400 GB+ temp storage + * + * kernel_version_min: + * Minimum kernel shipped by Seqera Forge images: + * AWS - ECS-Optimized AL2023: 6.1 | legacy AL2: 5.10 (EOL June 2026) + * GCP - Ubuntu 22.04 LTS: 5.15+ | Ubuntu 24.04 LTS: 6.8+ + * Azure - Ubuntu HPC 22.04: 5.15 | legacy Ubuntu 20.04: 5.4 + * + * Decision: Use cloud-specific minimums: + * - AWS: 5.10 (covers legacy AL2 until EOL June 2026) + * - GCP/Azure: 5.15 (current-gen Forge images) + * + * memory_gb_min: + * No Fusion-specific minimum documented. Thresholds reflect workload size. + * Tiers: 4 GB (low), 8 GB (recommended), 16 GB (high). + * + * disk_gb_min: + * Thresholds match Seqera's documented minimums for Fusion. Note that + * Platform's UI only allows selecting instance families, not sizes. Small + * instances in valid families (e.g. m6id.large = 118 GB) may not meet the + * recommended threshold — this is expected and signals the instance is + * undersized for Fusion despite being in the right family. + * + * AWS: + * Platform auto-selects NVMe-based instance families when Fusion is + * enabled (e.g. m6id, c6id, r6id). 8xlarge+ recommended for production. + * - Without NVMe: EBS bumped to 100 GB (gp3, 325 MB/s) + * - With NVMe: starts at 118 GB (.large), 1900 GB (.8xlarge) + * Tiers: + * - 100 GB (low — EBS gp3 when Fusion enabled without NVMe) + * - 200 GB (recommended — Seqera docs minimum) + * - 474 GB (high — .2xlarge NVMe, large datasets) + * + * GCP: + * Platform auto-selects families that support local SSDs (e.g. n2, c2, + * n2d). A 375 GB local NVMe SSD is provisioned per job. + * - Persistent disk: min 10 GiB (no local SSD) + * - Local NVMe SSD: 375 GiB increments (attached at creation time) + * - Production: n2-highmem-16 with local SSD, or larger + * Tiers: + * - 50 GB (low — persistent disk, small workloads) + * - 375 GB (recommended — 1x local NVMe SSD) + * - 750 GB (high — 2x local NVMe SSD) + * + * Azure: + * No auto-selection — user must pick VM size. Seqera recommends + * E-series with 'd' suffix (e.g. Standard_E8d_v5, Standard_E16d_v5). + * Standard SSDs only, no network-attached storage. + * - 'd' suffix VMs have local temp SSD (~37.5 GiB per vCPU) + * - Production: Standard_E16d_v5 or larger + * Tiers: + * - 75 GB (low — 2-vCPU 'd' VM, small workloads) + * - 300 GB (recommended — 8-vCPU 'd' VM, e.g. Standard_E8d_v5) + * - 600 GB (high — 16-vCPU 'd' VM, e.g. Standard_E16d_v5) */ profiles { + // ---- AWS profiles ---- + + fusion_aws_low { + // Small workloads — EBS-only, no NVMe + // 100 GB = EBS size Platform sets when Fusion is enabled without NVMe + params.fusion = true + params.fusion_kernel_version_min = "5.10" + params.fusion_memory_gb_min = 4 + params.fusion_disk_gb_min = 100 + } + fusion_aws_recommended { - // AWS-specific Fusion filesystem requirements - // For production workloads on AWS, consider instances with NVMe SSD storage - // Recommended: c5d, m5d, r5d instance families with local NVMe SSDs + // Seqera docs: NVMe-based families (e.g. m6id, c6id, r6id), 8xlarge+ for production + // 200 GB = Seqera docs minimum for Fusion local temp storage params.fusion = true params.fusion_kernel_version_min = "5.10" params.fusion_memory_gb_min = 8 params.fusion_disk_gb_min = 200 } - fusion_google_recommended { - // Google Cloud-specific Fusion filesystem requirements - // For production workloads on GCP, consider instances with local SSD scratch disks - // Recommended: n2, n2d, or c2 machine types with local SSD scratch disks + fusion_aws_high { + // Large-scale production — NVMe families, .2xlarge+ instances + // 474 GB = NVMe on .2xlarge; Seqera docs: 400 GB+ for files >100 GB params.fusion = true params.fusion_kernel_version_min = "5.10" + params.fusion_memory_gb_min = 16 + params.fusion_disk_gb_min = 474 + } + + // ---- Google Cloud profiles ---- + + fusion_google_low { + // Small workloads — persistent disk, no local SSD + // 50 GB = modest persistent disk for small workloads + params.fusion = true + params.fusion_kernel_version_min = "5.15" + params.fusion_memory_gb_min = 4 + params.fusion_disk_gb_min = 50 + } + + fusion_google_recommended { + // Seqera docs: families supporting local SSDs (e.g. n2, c2, n2d) + // 375 GB = 1x local NVMe SSD (GCP provisions per job) + params.fusion = true + params.fusion_kernel_version_min = "5.15" params.fusion_memory_gb_min = 8 - params.fusion_disk_gb_min = 200 + params.fusion_disk_gb_min = 375 + } + + fusion_google_high { + // Large-scale production — 2x local NVMe SSDs + // 750 GB = 2 x 375 GB; Seqera docs: 400 GB+ for files >100 GB + params.fusion = true + params.fusion_kernel_version_min = "5.15" + params.fusion_memory_gb_min = 16 + params.fusion_disk_gb_min = 750 + } + + // ---- Azure profiles ---- + + fusion_azure_low { + // Small workloads — smallest 'd' suffix VM (2 vCPU) + // 75 GB = local temp disk on 2-vCPU 'd' VM + params.fusion = true + params.fusion_kernel_version_min = "5.15" + params.fusion_memory_gb_min = 4 + params.fusion_disk_gb_min = 75 } fusion_azure_recommended { - // Azure-specific Fusion filesystem requirements - // For production workloads on Azure, consider VM series with local temp storage - // Recommended: D-series, E-series, or L-series VMs with local SSD storage + // Seqera docs: E-series with 'd' suffix (e.g. Standard_E8d_v5, Standard_E16d_v5) + // 300 GB = 8-vCPU 'd' VM temp disk params.fusion = true - params.fusion_kernel_version_min = "5.10" + params.fusion_kernel_version_min = "5.15" params.fusion_memory_gb_min = 8 - params.fusion_disk_gb_min = 200 + params.fusion_disk_gb_min = 300 } + fusion_azure_high { + // Large-scale production — 16-vCPU 'd' VM (e.g. Standard_E16d_v5) + // 600 GB = Standard_E16d_v5 temp disk (614 GiB) + params.fusion = true + params.fusion_kernel_version_min = "5.15" + params.fusion_memory_gb_min = 16 + params.fusion_disk_gb_min = 600 + } } From 676f39c670fdcbeefd849823fcf67d93a84223a0 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 4 Mar 2026 15:52:16 +0100 Subject: [PATCH 13/53] fix: Stage report template file as an input Signed-off-by: Alberto Miranda --- main.nf | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/main.nf b/main.nf index 3184d34..27ac6b5 100644 --- a/main.nf +++ b/main.nf @@ -431,6 +431,7 @@ process FUSION_DOCTOR_GENERATE_REPORT { input: path(doctor_report) + path(template_file) output: path("fusion_report.html"), emit: html_report @@ -438,6 +439,10 @@ process FUSION_DOCTOR_GENERATE_REPORT { script: """ + # Create the expected directory structure for the template + mkdir -p assets/templates + cp ${template_file} assets/templates/fusion_report_template.html + generate_fusion_report.py \\ --doctor ${doctor_report} \\ --output-html fusion_report.html \\ @@ -540,7 +545,8 @@ workflow NF_CANARY { // Generate consolidated fusion report from doctor output // Only run FUSION_DOCTOR_GENERATE_REPORT if TEST_FUSION_DOCTOR produced output FUSION_DOCTOR_GENERATE_REPORT( - TEST_FUSION_DOCTOR.out.report + TEST_FUSION_DOCTOR.out.report, + file("${projectDir}/assets/templates/fusion_report_template.html") ) // POC of emitting the channel From 6b77f7f1a95a8994a1f3f2ac5bf25c1c07cf7650 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 4 Mar 2026 16:03:29 +0100 Subject: [PATCH 14/53] fix: Ensure `uv` does not check for project files Signed-off-by: Alberto Miranda --- bin/generate_fusion_report.py | 2 +- tests/test_generate_fusion_report.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/generate_fusion_report.py b/bin/generate_fusion_report.py index ba2c5f2..80a6ac8 100755 --- a/bin/generate_fusion_report.py +++ b/bin/generate_fusion_report.py @@ -1,4 +1,4 @@ -#!/usr/bin/env -S uv run --script +#!/usr/bin/env -S uv run --no-project --script # /// script # requires-python = ">=3.12" diff --git a/tests/test_generate_fusion_report.py b/tests/test_generate_fusion_report.py index 585fe94..4a24705 100755 --- a/tests/test_generate_fusion_report.py +++ b/tests/test_generate_fusion_report.py @@ -1,4 +1,4 @@ -#!/usr/bin/env -S uv run --script +#!/usr/bin/env -S uv run --no-project --script # /// script # requires-python = ">=3.12" From 5e8a31e982a851093c21faf6a5a43f809fc388f1 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 4 Mar 2026 16:24:16 +0100 Subject: [PATCH 15/53] fix: Pass template file as parameter instead of copying to fixed location - Add --template argument to generate_fusion_report.py - Update TEST_FUSION_REPORT process to pass template file path directly - Removes need to copy template to /usr/local/assets/templates - Makes template handling more flexible and container-agnostic --- bin/generate_fusion_report.py | 35 ++++++++++++++++++++++++----------- main.nf | 5 +---- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/bin/generate_fusion_report.py b/bin/generate_fusion_report.py index 80a6ac8..04f378d 100755 --- a/bin/generate_fusion_report.py +++ b/bin/generate_fusion_report.py @@ -83,14 +83,15 @@ def merge_reports( return combined -def render_html(combined_report: Dict[str, Any], template_str: Optional[str] = None) -> str: +def render_html(combined_report: Dict[str, Any], template_str: Optional[str] = None, template_path: Optional[str] = None) -> str: """ Render self-contained HTML report from combined data. Args: combined_report: Merged diagnostic report dictionary template_str: The HTML template string (pre-loaded for robustness). - If None, will be loaded at call time (slower, for backward compatibility). + If None, will be loaded from template_path or default location. + template_path: Path to template file (optional, overrides default location) Returns: HTML string with inline CSS/JS @@ -99,7 +100,7 @@ def render_html(combined_report: Dict[str, Any], template_str: Optional[str] = N # Load template if not provided (for backward compatibility with tests) if template_str is None: - template_str = load_template() + template_str = load_template(template_path) template = Template(template_str) @@ -114,8 +115,11 @@ def render_html(combined_report: Dict[str, Any], template_str: Optional[str] = N return html -def load_template() -> str: - """Load the HTML template from assets/templates directory. +def load_template(template_path: Optional[str] = None) -> str: + """Load the HTML template from assets/templates directory or specified path. + + Args: + template_path: Optional path to template file. If None, uses default location. Returns: HTML template string @@ -124,18 +128,23 @@ def load_template() -> str: FileNotFoundError: If template file is not found IOError: If template file cannot be read """ - # Go up from bin/ to project root, then into assets/templates - template_path = Path(__file__).parent.parent / "assets" / "templates" / "fusion_report_template.html" + # Use provided path or default location + if template_path: + path = Path(template_path) + else: + # Go up from bin/ to project root, then into assets/templates + path = Path(__file__).parent.parent / "assets" / "templates" / "fusion_report_template.html" + try: - with open(template_path, 'r') as f: + with open(path, 'r') as f: return f.read() except FileNotFoundError: raise FileNotFoundError( - f"HTML template not found at {template_path}. " + f"HTML template not found at {path}. " "Ensure fusion_report_template.html is in 'assets/templates/' directory." ) except IOError as e: - raise IOError(f"Cannot read template file {template_path}: {str(e)}") + raise IOError(f"Cannot read template file {path}: {str(e)}") def main(): @@ -165,6 +174,10 @@ def main(): "--output-json", type=str, default="fusion_report.json", help="Output path for combined JSON report (default: fusion_report.json)" ) + parser.add_argument( + "--template", type=str, required=False, + help="Path to HTML template file (optional, uses default if not provided)" + ) args = parser.parse_args() @@ -178,7 +191,7 @@ def main(): # Load template at startup for early failure detection try: - template_str = load_template() + template_str = load_template(args.template) except (FileNotFoundError, IOError) as e: print(f"ERROR: {str(e)}", file=sys.stderr) sys.exit(1) diff --git a/main.nf b/main.nf index 27ac6b5..6be7cb2 100644 --- a/main.nf +++ b/main.nf @@ -439,12 +439,9 @@ process FUSION_DOCTOR_GENERATE_REPORT { script: """ - # Create the expected directory structure for the template - mkdir -p assets/templates - cp ${template_file} assets/templates/fusion_report_template.html - generate_fusion_report.py \\ --doctor ${doctor_report} \\ + --template ${template_file} \\ --output-html fusion_report.html \\ --output-json fusion_report.json """ From f3a7b7283c4910bbfec6de1037efdc78c98bb5c0 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 4 Mar 2026 16:39:46 +0100 Subject: [PATCH 16/53] fix: Handle real fusion doctor JSON schema in HTML report The real `fusion doctor` binary outputs a different JSON schema than the mock: `checks` as a list (not a dict), `check_summary.overall` instead of `summary.status`, and `fusion_version` instead of `version`. The template and merge logic now handle both formats. Signed-off-by: Alberto Miranda --- assets/templates/fusion_report_template.html | 53 +++++++++-- bin/generate_fusion_report.py | 6 +- tests/test_generate_fusion_report.py | 99 ++++++++++++++++++++ 3 files changed, 147 insertions(+), 11 deletions(-) diff --git a/assets/templates/fusion_report_template.html b/assets/templates/fusion_report_template.html index 7979caa..1685f17 100644 --- a/assets/templates/fusion_report_template.html +++ b/assets/templates/fusion_report_template.html @@ -332,33 +332,68 @@

System & Validation

▼
- {% if doctor_report.summary %} + {% if doctor_report.check_summary or doctor_report.summary %} {% set summary = + doctor_report.check_summary or doctor_report.summary %} {% set summary_status = summary.overall or + summary.status or 'unknown' %}
Overall Status
- {{ doctor_report.summary.status }} + {{ summary_status }}
- {% if doctor_report.version %} + {% if doctor_report.fusion_version or doctor_report.version %}
Fusion Version
-
{{ doctor_report.version }}
+
+ {{ doctor_report.fusion_version or doctor_report.version }} +
+
+ {% endif %} {% if summary.passed is defined or summary.failed is defined %} +
+
Checks
+
+ {{ summary.passed|default(0) }} passed, {{ summary.failed|default(0) }} failed{% if + summary.skipped %}, {{ summary.skipped }} skipped{% endif %} +
{% endif %}
{% endif %} {% if doctor_report.checks %}

Checks

- {% for check_name, check_data in doctor_report.checks.items() %} + {% if doctor_report.checks is mapping %} {% for check_name, check_data in + doctor_report.checks.items() %} +
+
+ + {% if check_data.status == 'pass' %}✓{% elif check_data.status == 'warn' %}⚠{% else + %}✗{% endif %} + + {{ check_name|replace('_', ' ')|title }} +
+
{{ check_data.message }}
+ {% if check_data.details %} +
+ Details: + 
{{ check_data.details|tojson(indent=2) }}
+
+ {% endif %} {% if check_data.remediation %} +
Remediation: {{ check_data.remediation }}
+ {% endif %} +
+ {% endfor %} {% else %} {% for check_data in doctor_report.checks %}
{% if check_data.status == 'pass' %}✓{% elif check_data.status == 'warn' %}⚠{% else %}✗{% endif %} - {{ check_name|title }} + {{ (check_data.check or check_data.name or 'Unknown')|replace('_', ' ')|title }} {% if + check_data.category %} + ({{ check_data.category }}) + {% endif %}
{{ check_data.message }}
{% if check_data.details %} @@ -370,7 +405,7 @@

Checks

Remediation: {{ check_data.remediation }}
{% endif %}
- {% endfor %} {% endif %} + {% endfor %} {% endif %} {% endif %}
{% endif %} diff --git a/bin/generate_fusion_report.py b/bin/generate_fusion_report.py index 04f378d..c7d71a2 100755 --- a/bin/generate_fusion_report.py +++ b/bin/generate_fusion_report.py @@ -66,12 +66,14 @@ def merge_reports( } # Compute overall status (fail > warn > pass) + # Supports both legacy "summary.status" and real fusion "check_summary.overall" statuses = [] for report in combined["reports"].values(): if report and "error" not in report: - if "summary" in report: + if "check_summary" in report: + statuses.append(report["check_summary"].get("overall", "unknown")) + elif "summary" in report: statuses.append(report["summary"].get("status", "unknown")) - # Note: Reports without "summary" are silently ignored in status aggregation if "fail" in statuses: combined["overall_status"] = "fail" diff --git a/tests/test_generate_fusion_report.py b/tests/test_generate_fusion_report.py index 4a24705..3b02327 100755 --- a/tests/test_generate_fusion_report.py +++ b/tests/test_generate_fusion_report.py @@ -435,5 +435,104 @@ def test_full_workflow_with_multiple_reports(self): assert len(html) > 1000 # Should be substantial HTML +class TestRealFusionDoctorFormat: + """Tests for the real fusion doctor output format (list-based checks, check_summary).""" + + def test_render_with_list_checks(self): + """Test rendering when checks is a list (real fusion doctor format).""" + combined_report = { + "timestamp": "2026-03-04T10:00:00Z", + "overall_status": "pass", + "reports": { + "doctor": { + "fusion_version": "2.6-develop-1f517df", + "check_summary": { + "overall": "pass", + "passed": 2, + "failed": 0, + "skipped": 0, + }, + "checks": [ + { + "check": "fuse_device", + "category": "critical", + "status": "pass", + "message": "/dev/fuse is available", + "details": {"path": "/dev/fuse"}, + "duration_ms": 0, + }, + { + "check": "disk_space", + "category": "warning", + "status": "warn", + "message": "Low disk space on /tmp", + "duration_ms": 5, + }, + ], + }, + "bench": {}, + "objbench": {}, + }, + } + + html = render_html(combined_report) + assert "2.6-develop-1f517df" in html + assert "Fuse Device" in html + assert "Disk Space" in html + assert "/dev/fuse" in html + assert "critical" in html + assert "2 passed" in html + + def test_merge_with_check_summary(self): + """Test that merge_reports reads status from check_summary.overall.""" + doctor_data = { + "check_summary": {"overall": "warn", "passed": 3, "failed": 0}, + "checks": [], + } + + with tempfile.TemporaryDirectory() as tmpdir: + doctor_path = Path(tmpdir) / "doctor.json" + doctor_path.write_text(json.dumps(doctor_data)) + + result = merge_reports(str(doctor_path), None, None) + assert result["overall_status"] == "warn" + + def test_full_workflow_real_format(self): + """Test complete workflow with real fusion doctor format.""" + doctor_data = { + "schema_version": "1.1", + "fusion_version": "2.6.0", + "timestamp": "2026-03-04T10:00:00Z", + "checks": [ + { + "check": "fuse_device", + "category": "critical", + "status": "pass", + "message": "/dev/fuse is available and accessible", + "details": {"path": "/dev/fuse", "permissions": "Dcrw-rw-rw-"}, + "duration_ms": 0, + }, + ], + "check_summary": { + "overall": "pass", + "passed": 1, + "failed": 0, + "skipped": 0, + }, + } + + with tempfile.TemporaryDirectory() as tmpdir: + doctor_path = Path(tmpdir) / "doctor.json" + doctor_path.write_text(json.dumps(doctor_data)) + + combined = merge_reports(str(doctor_path), None, None) + assert combined["overall_status"] == "pass" + + html = render_html(combined) + assert "2.6.0" in html + assert "Fuse Device" in html + assert "Dcrw-rw-rw-" in html + + if __name__ == "__main__": pytest.main([__file__, "-v"]) From d58124938c84bb5dd797b96795469d0135d7b8dd Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 4 Mar 2026 16:57:52 +0100 Subject: [PATCH 17/53] fix: Fix report generation Signed-off-by: Alberto Miranda --- bin/generate_fusion_report.py | 4 ++-- main.nf | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bin/generate_fusion_report.py b/bin/generate_fusion_report.py index c7d71a2..9243c30 100755 --- a/bin/generate_fusion_report.py +++ b/bin/generate_fusion_report.py @@ -173,8 +173,8 @@ def main(): help="Output path for HTML report (default: fusion-report.html)" ) parser.add_argument( - "--output-json", type=str, default="fusion_report.json", - help="Output path for combined JSON report (default: fusion_report.json)" + "--output-json", type=str, default="fusion-report.json", + help="Output path for combined JSON report (default: fusion-report.json)" ) parser.add_argument( "--template", type=str, required=False, diff --git a/main.nf b/main.nf index 6be7cb2..2e605c5 100644 --- a/main.nf +++ b/main.nf @@ -427,23 +427,23 @@ process FUSION_DOCTOR_GENERATE_REPORT { */ container 'community.wave.seqera.io/library/jinja2_python_uv:7113b0a0e59d95a6' - publishDir { params.outdir ?: file(workflow.workDir).resolve("outputs/fusion").toUriString() }, mode: 'copy' + publishDir { (params.outdir ? file(params.outdir) : file(workflow.workDir).resolve("outputs")).resolve("fusion").toUriString() }, mode: 'copy' input: path(doctor_report) path(template_file) output: - path("fusion_report.html"), emit: html_report - path("fusion_report.json"), emit: json_report + path("fusion-report.html"), emit: html_report + path("fusion-report.json"), emit: json_report script: """ generate_fusion_report.py \\ --doctor ${doctor_report} \\ --template ${template_file} \\ - --output-html fusion_report.html \\ - --output-json fusion_report.json + --output-html fusion-report.html \\ + --output-json fusion-report.json """ } From 9ee58b6d01cc2bac676d4df3138bd321b7ea6cc0 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 4 Mar 2026 18:45:03 +0100 Subject: [PATCH 18/53] feat: Enhance Fusion diagnostic report with system, storage, and resource sections Display rich system environment, storage, and resource limit data from fusion doctor JSON. Improve UX with native details/summary elements, structured check details, filesystem filtering/sorting, thousands separators, copy-to-clipboard buttons, and localized timestamps. Signed-off-by: Alberto Miranda --- assets/templates/fusion_report_template.html | 549 +++++++++++++++++-- bin/generate_fusion_report.py | 31 +- tests/test_generate_fusion_report.py | 282 ++++++++++ 3 files changed, 803 insertions(+), 59 deletions(-) diff --git a/assets/templates/fusion_report_template.html b/assets/templates/fusion_report_template.html index 1685f17..e43d2ca 100644 --- a/assets/templates/fusion_report_template.html +++ b/assets/templates/fusion_report_template.html @@ -90,6 +90,7 @@ color: #c41e1e; } + /* Collapsible sections using native
/ */ .section { margin-bottom: 2rem; border: 1px solid #dadde1; @@ -98,7 +99,7 @@ background: #fff; } - .section-header { + .section > summary { background: #f5f6f7; padding: 1rem; border-bottom: 1px solid #dadde1; @@ -108,20 +109,30 @@ align-items: center; transition: background-color 0.15s; user-select: none; + list-style: none; } - .section-header:hover { + .section > summary::-webkit-details-marker { + display: none; + } + + .section > summary::marker { + display: none; + content: ""; + } + + .section > summary:hover { background: #ebedf0; } - .section-header h2 { + .section > summary h2 { font-size: 1.15rem; font-weight: 600; color: #1c1e21; margin: 0; } - .section-header .toggle-icon { + .section > summary .toggle-icon { display: inline-block; width: 20px; height: 20px; @@ -131,26 +142,15 @@ font-weight: 700; font-size: 0.9rem; transition: transform 0.2s ease; + transform: rotate(-90deg); } - .section-header.collapsed .toggle-icon { - transform: rotate(-90deg); + .section[open] > summary .toggle-icon { + transform: rotate(0deg); } .section-content { padding: 1.5rem; - display: block; - max-height: 10000px; - overflow: hidden; - transition: - max-height 0.3s ease, - padding 0.3s ease; - } - - .section-content.collapsed { - max-height: 0; - padding: 0 1.5rem; - overflow: hidden; } .check-item { @@ -193,6 +193,12 @@ justify-content: center; } + .check-category { + font-size: 0.75em; + color: #606770; + font-weight: 400; + } + .check-message { color: #525860; margin-bottom: 0.5rem; @@ -245,6 +251,16 @@ color: #1c1e21; } + .summary-card-value.sm { + font-size: 1rem; + } + + .summary-card-sub { + font-size: 0.85rem; + color: #606770; + margin-top: 0.25rem; + } + h3 { font-size: 1.1rem; font-weight: 600; @@ -252,6 +268,165 @@ margin: 1.5rem 0 1rem 0; } + h3:first-child, + h3.mt-0 { + margin-top: 0; + } + + .info-table { + width: 100%; + border-collapse: collapse; + font-size: 0.95rem; + margin-bottom: 1rem; + } + + .info-table caption { + text-align: left; + font-weight: 600; + font-size: 1.1rem; + color: #1c1e21; + padding-bottom: 0.5rem; + } + + .info-table th { + text-align: left; + padding: 0.5rem 0.75rem; + background: #f5f6f7; + border-bottom: 1px solid #dadde1; + font-weight: 500; + color: #606770; + font-size: 0.8rem; + text-transform: uppercase; + letter-spacing: 0.02em; + } + + .info-table td { + padding: 0.5rem 0.75rem; + border-bottom: 1px solid #ebedf0; + color: #1c1e21; + } + + .info-table tr:last-child td { + border-bottom: none; + } + + .info-table .label { + color: #606770; + font-weight: 500; + width: 40%; + } + + .mono { + font-family: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace; + font-size: 0.9em; + } + + .usage-bar { + display: flex; + align-items: center; + gap: 0.5rem; + } + + .usage-bar-track { + flex: 1; + height: 8px; + background: #ebedf0; + border-radius: 4px; + overflow: hidden; + max-width: 120px; + } + + .usage-bar-fill { + height: 100%; + border-radius: 4px; + background: #00a400; + } + + .usage-bar-fill.warn { + background: #ffba00; + } + + .usage-bar-fill.critical { + background: #fa383e; + } + + .usage-pct { + font-size: 0.85rem; + min-width: 3em; + } + + .text-muted { + font-size: 0.85rem; + color: #606770; + } + + .inline-warn { + display: inline-block; + font-size: 0.8rem; + color: #b38200; + background: #fff8e6; + padding: 0.1rem 0.4rem; + border-radius: 0.2rem; + margin-left: 0.5rem; + font-weight: 500; + } + + .inline-critical { + display: inline-block; + font-size: 0.8rem; + color: #c41e1e; + background: #ffe6e6; + padding: 0.1rem 0.4rem; + border-radius: 0.2rem; + margin-left: 0.5rem; + font-weight: 500; + } + + .detail-kv { + display: flex; + gap: 0.5rem; + padding: 0.2rem 0; + font-size: 0.9rem; + } + + .detail-kv .dk-key { + color: #606770; + font-weight: 500; + min-width: 8em; + } + + .detail-kv .dk-val { + color: #1c1e21; + font-family: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace; + font-size: 0.85em; + } + + .copy-btn { + display: inline-flex; + align-items: center; + gap: 0.25rem; + background: none; + border: 1px solid #dadde1; + border-radius: 0.25rem; + padding: 0.15rem 0.4rem; + font-size: 0.75rem; + color: #606770; + cursor: pointer; + margin-left: 0.5rem; + vertical-align: middle; + transition: background 0.15s, color 0.15s; + } + + .copy-btn:hover { + background: #ebedf0; + color: #1c1e21; + } + + .copy-btn.copied { + color: #00a400; + border-color: #00a400; + } + footer { text-align: center; padding-top: 1.5rem; @@ -269,8 +444,17 @@ max-width: none; padding: 0; } - .section-content { - display: block !important; + .section { + break-inside: avoid; + } + .section > summary .toggle-icon { + display: none; + } + details.section { + display: block; + } + details.section > summary ~ * { + display: block; } } @@ -297,6 +481,7 @@

Generated - {{ timestamp }} +
+ {% if doctor_report.fusion_version or doctor_report.version %} + {% set version = doctor_report.fusion_version or doctor_report.version %} +
+ Fusion Version + {{ version }} +
+ {% endif %}
Status {{ overall_status }} @@ -324,13 +516,13 @@

- + {% if doctor_report %} -
-
-

System & Validation

+
+ +

Validation Checks

▼ -
+

{% if doctor_report.check_summary or doctor_report.summary %} {% set summary = doctor_report.check_summary or doctor_report.summary %} {% set summary_status = summary.overall or @@ -342,14 +534,7 @@

System & Validation

{{ summary_status }}
- {% if doctor_report.fusion_version or doctor_report.version %} -
-
Fusion Version
-
- {{ doctor_report.fusion_version or doctor_report.version }} -
-
- {% endif %} {% if summary.passed is defined or summary.failed is defined %} + {% if summary.passed is defined or summary.failed is defined %}
Checks
@@ -374,8 +559,17 @@

Checks

{{ check_data.message }}
{% if check_data.details %}
- Details: + {% if check_data.details is mapping %} + {% for key, val in check_data.details.items() %} +
+ {{ key|replace('_', ' ')|title }} + {{ val }} +
+ {% endfor %} + {% else %} 
{{ check_data.details|tojson(indent=2) }}
+ {% endif %} +
{% endif %} {% if check_data.remediation %}
Remediation: {{ check_data.remediation }}
@@ -390,16 +584,23 @@

Checks

{{ (check_data.check or check_data.name or 'Unknown')|replace('_', ' ')|title }} {% if check_data.category %} - ({{ check_data.category }}) + ({{ check_data.category }}) {% endif %}
{{ check_data.message }}
{% if check_data.details %}
- Details: + {% if check_data.details is mapping %} + {% for key, val in check_data.details.items() %} +
+ {{ key|replace('_', ' ')|title }} + {{ val }} +
+ {% endfor %} + {% else %} 
{{ check_data.details|tojson(indent=2) }}
+ {% endif %} +
{% endif %} {% if check_data.remediation %}
Remediation: {{ check_data.remediation }}
@@ -407,16 +608,229 @@

Checks

{% endfor %} {% endif %} {% endif %} - +
+ {% endif %} + + + {% if doctor_report.system %} +
+ +

System Environment

+ ▼ + +
+ {% set sys = doctor_report.system %} +
+ {% if sys.os %} +
+
Operating System
+
+ {{ sys.os.name|default('Unknown')|title }} {{ sys.os.version|default('') }} +
+
+ Kernel {{ sys.os.kernel|default('N/A') }} · {{ sys.os.architecture|default('N/A') }} +
+
+ {% endif %} + {% if sys.cpu %} +
+
CPU
+
+ {{ sys.cpu.cores|default('?') }} cores / {{ sys.cpu.threads|default('?') }} threads +
+
+ {{ sys.cpu.model|default('Unknown') }} +
+
+ {% endif %} + {% if sys.memory %} + {% set mem_total_gb = (sys.memory.total_bytes / 1073741824)|round(1) %} + {% set mem_used_gb = ((sys.memory.total_bytes - sys.memory.available_bytes) / 1073741824)|round(1) %} + {% set mem_avail_gb = (sys.memory.available_bytes / 1073741824)|round(1) %} + {% set mem_used_pct = (((sys.memory.total_bytes - sys.memory.available_bytes) / sys.memory.total_bytes) * 100)|round(0)|int %} +
+
Memory
+
+ {{ mem_used_gb }} GB used of {{ mem_total_gb }} GB +
+
+
+
+
+ {{ mem_used_pct }}% + ({{ mem_avail_gb }} GB available) +
+ {% if sys.memory.swap_total_bytes and sys.memory.swap_total_bytes > 0 %} + {% set swap_total_bytes = sys.memory.swap_total_bytes %} + {% set swap_free_bytes = sys.memory.swap_free_bytes %} + {% set swap_used_pct = (((swap_total_bytes - swap_free_bytes) / swap_total_bytes) * 100)|round(0)|int %} + {% set swap_total_gb = (swap_total_bytes / 1073741824)|round(1) %} +
+ Swap: + {% if swap_free_bytes < 1073741824 %} + {{ (swap_free_bytes / 1048576)|round(0)|int }} MB free + {% else %} + {{ (swap_free_bytes / 1073741824)|round(1) }} GB free + {% endif %} + of {{ swap_total_gb }} GB + {% if swap_used_pct >= 95 %} + {{ swap_used_pct }}% used + {% elif swap_used_pct >= 80 %} + {{ swap_used_pct }}% used + {% endif %} +
+ {% endif %} +
+ {% endif %} +
+
+
+ {% endif %} + + + {% if doctor_report.storage %} +
+ +

Storage

+ ▼ + +
+ {% if doctor_report.storage.nvme_devices %} + + + + + + + + + + + {% for dev in doctor_report.storage.nvme_devices %} + + + + + + {% endfor %} + +
NVMe Devices
DeviceModelSize
{{ dev.name }}{{ dev.model }}{{ (dev.size_bytes / 1073741824)|round(0)|int }} GB
+ {% endif %} + + {% if doctor_report.storage.filesystems %} + + + + + + + + + + + + {% for fs in doctor_report.storage.filesystems %} + {% if fs.total_bytes and fs.total_bytes > 0 %} + {% set fs_total_gb = (fs.total_bytes / 1073741824)|round(1) %} + {% set fs_avail_gb = (fs.available_bytes / 1073741824)|round(1) %} + {% set fs_used_pct = (((fs.total_bytes - fs.available_bytes) / fs.total_bytes) * 100)|round(0)|int %} + + + + + + + {% endif %} + {% endfor %} + +
Filesystems
MountDeviceTypeUsage
{{ fs.mount_point }}{{ fs.device }}{{ fs.type }} +
+
+
+
+ {{ fs_used_pct }}% + {{ fs_avail_gb }} / {{ fs_total_gb }} GB free +
+
+ {% endif %} +
+
+ {% endif %} + + + {% if doctor_report.resources %} +
+ +

Resource Limits

+ ▼ + +
+ {% set res = doctor_report.resources %} + {% set unlimited = 18446744073709551615 %} + + + + + + + + + + {% if res.open_files %} + + + + + + {% endif %} + {% if res.max_procs %} + + + + + + {% endif %} + {% if res.mem_lock %} + + + + + + {% endif %} + {% if res.stack_size %} + + + + + + {% endif %} + {% if res.file_size %} + + + + + + {% endif %} + {% if res.address_space %} + + + + + + {% endif %} + +
ResourceSoft LimitHard Limit
Open Files{{ res.open_files.soft|intcomma if res.open_files.soft != unlimited else 'unlimited' }}{% if res.open_files.soft != unlimited and res.open_files.soft < 65536 %} Low{% endif %}{{ res.open_files.hard|intcomma if res.open_files.hard != unlimited else 'unlimited' }}
Max Processes{{ res.max_procs.soft|intcomma if res.max_procs.soft != unlimited else 'unlimited' }}{{ res.max_procs.hard|intcomma if res.max_procs.hard != unlimited else 'unlimited' }}
Memory Lock{% if res.mem_lock.soft == unlimited %}unlimited{% else %}{{ (res.mem_lock.soft / 1073741824)|round(1) }} GB{% endif %}{% if res.mem_lock.hard == unlimited %}unlimited{% else %}{{ (res.mem_lock.hard / 1073741824)|round(1) }} GB{% endif %}
Stack Size{% if res.stack_size.soft == unlimited %}unlimited{% else %}{{ (res.stack_size.soft / 1048576)|round(1) }} MB{% endif %}{% if res.stack_size.hard == unlimited %}unlimited{% else %}{{ (res.stack_size.hard / 1048576)|round(1) }} MB{% endif %}
Max File Size{{ 'unlimited' if res.file_size.soft == unlimited else res.file_size.soft }}{{ 'unlimited' if res.file_size.hard == unlimited else res.file_size.hard }}
Address Space{{ 'unlimited' if res.address_space.soft == unlimited else res.address_space.soft }}{{ 'unlimited' if res.address_space.hard == unlimited else res.address_space.hard }}
+
+
{% endif %} {% if bench_report and bench_report.keys()|length > 0 and 'error' not in bench_report %} -
-
+
+

Filesystem Benchmark

▼ -
+
{% if bench_report.summary %}
@@ -440,19 +854,19 @@

Filesystem Benchmark

Error: {{ bench_report.error }}
{% else %} -

No benchmark results available

+

No benchmark results available

{% endif %}
-
+ {% endif %} {% if objbench_report and objbench_report.keys()|length > 0 and 'error' not in objbench_report %} -
-
+
+

Object Storage Benchmark

▼ -
+
{% if objbench_report.summary %}
@@ -476,23 +890,46 @@

Object Storage Benchmark

Error: {{ objbench_report.error }}
{% else %} -

No benchmark results available

+

No benchmark results available

{% endif %}
-
+ {% endif %} -
Fusion Diagnostic Report • Generated on {{ timestamp }}
+
Fusion Diagnostic Report • Generated on
diff --git a/bin/generate_fusion_report.py b/bin/generate_fusion_report.py index 9243c30..b40a9f4 100755 --- a/bin/generate_fusion_report.py +++ b/bin/generate_fusion_report.py @@ -98,18 +98,43 @@ def render_html(combined_report: Dict[str, Any], template_str: Optional[str] = N Returns: HTML string with inline CSS/JS """ - from jinja2 import Template + from jinja2 import Environment # Load template if not provided (for backward compatibility with tests) if template_str is None: template_str = load_template(template_path) - template = Template(template_str) + env = Environment() + env.filters['intcomma'] = lambda v: f"{int(v):,}" if isinstance(v, (int, float)) else str(v) + template = env.from_string(template_str) + + doctor_report = combined_report.get("reports", {}).get("doctor", {}) + + # Sort filesystems by usage% descending and filter out zero-size / noise mounts + storage = doctor_report.get("storage", {}) + if storage.get("filesystems"): + noise_prefixes = ("/var/snap/", "/snap/") + noise_types = ("squashfs", "tmpfs") + filtered = [] + for fs in storage["filesystems"]: + total = fs.get("total_bytes", 0) + if total <= 0: + continue + mount = fs.get("mount_point", "") + fs_type = fs.get("type", "") + if any(mount.startswith(p) for p in noise_prefixes): + continue + if fs_type in noise_types and mount != "/tmp": + continue + # Attach computed usage% for sorting + fs["_used_pct"] = ((total - fs.get("available_bytes", 0)) / total) * 100 + filtered.append(fs) + storage["filesystems"] = sorted(filtered, key=lambda f: f["_used_pct"], reverse=True) html = template.render( timestamp=combined_report.get("timestamp", ""), overall_status=combined_report.get("overall_status", "unknown"), - doctor_report=combined_report.get("reports", {}).get("doctor", {}), + doctor_report=doctor_report, bench_report=combined_report.get("reports", {}).get("bench", {}), objbench_report=combined_report.get("reports", {}).get("objbench", {}), ) diff --git a/tests/test_generate_fusion_report.py b/tests/test_generate_fusion_report.py index 3b02327..bc1dea3 100755 --- a/tests/test_generate_fusion_report.py +++ b/tests/test_generate_fusion_report.py @@ -534,5 +534,287 @@ def test_full_workflow_real_format(self): assert "Dcrw-rw-rw-" in html +class TestNewSections: + """Tests for the new System Environment, Storage, Resource Limits sections.""" + + FULL_DOCTOR_DATA = { + "schema_version": "1.1", + "fusion_version": "2.6-develop-1f517df", + "timestamp": "2026-03-04T15:35:35Z", + "system": { + "os": { + "name": "ubuntu", + "version": "24.04", + "kernel": "6.14.0-27-generic", + "architecture": "x86_64", + }, + "cpu": { + "model": "13th Gen Intel(R) Core(TM) i7-1355U", + "cores": 10, + "threads": 12, + }, + "memory": { + "total_bytes": 33301454848, + "available_bytes": 3561734144, + "swap_total_bytes": 2046816256, + "swap_free_bytes": 417792, + }, + }, + "storage": { + "filesystems": [ + { + "device": "/dev/dm-1", + "mount_point": "/", + "type": "ext4", + "total_bytes": 981132795904, + "available_bytes": 152799973376, + }, + ], + "nvme_devices": [ + { + "name": "nvme0", + "model": "WD PC SN810", + "size_bytes": 1024209543168, + "block_devices": ["nvme0n1"], + }, + ], + }, + "resources": { + "open_files": {"soft": 1048576, "hard": 1048576}, + "max_procs": {"soft": 125971, "hard": 125971}, + "mem_lock": {"soft": 4162678784, "hard": 4162678784}, + "stack_size": {"soft": 12800000, "hard": 18446744073709551615}, + }, + "checks": [ + { + "check": "fuse_device", + "category": "critical", + "status": "pass", + "message": "/dev/fuse is available and accessible", + "details": {"path": "/dev/fuse", "permissions": "Dcrw-rw-rw-"}, + "duration_ms": 0, + }, + ], + "check_summary": { + "overall": "pass", + "passed": 1, + "failed": 0, + "skipped": 0, + }, + } + + def _render(self, doctor_data=None): + combined = { + "timestamp": "2026-03-04T15:35:35Z", + "overall_status": "pass", + "reports": { + "doctor": doctor_data or self.FULL_DOCTOR_DATA, + "bench": {}, + "objbench": {}, + }, + } + return render_html(combined) + + def test_system_environment_section_present(self): + """Test that System Environment section is rendered.""" + html = self._render() + assert "System Environment" in html + + def test_os_info_rendered(self): + """Test OS name, version, kernel, architecture appear.""" + html = self._render() + assert "Ubuntu" in html + assert "24.04" in html + assert "6.14.0-27-generic" in html + assert "x86_64" in html + + def test_cpu_info_rendered(self): + """Test CPU model, cores, threads appear.""" + html = self._render() + assert "10 cores" in html + assert "12 threads" in html + assert "i7-1355U" in html + + def test_memory_info_rendered(self): + """Test memory used and total appear with consistent framing.""" + html = self._render() + assert "27.7 GB used" in html # used ~27.7 GB + assert "31.0 GB" in html # total ~31 GB + assert "3.3 GB available" in html # available shown in parentheses + + def test_swap_warning_rendered(self): + """Test swap critical warning when swap is nearly full.""" + html = self._render() + assert "inline-critical" in html # swap is 99%+ used + + def test_swap_shows_mb_when_under_1gb(self): + """Test swap free shows MB when under 1 GB (avoids '0.0 GB free').""" + html = self._render() + # swap_free_bytes = 417792 (~0.4 MB), should show MB not GB + assert "MB free" in html + assert "0.0 GB free" not in html + + def test_storage_section_present(self): + """Test that Storage section is rendered.""" + html = self._render() + assert "Storage" in html + + def test_nvme_devices_rendered(self): + """Test NVMe device info appears.""" + html = self._render() + assert "nvme0" in html + assert "WD PC SN810" in html + + def test_filesystem_table_rendered(self): + """Test filesystem table with mount point and device.""" + html = self._render() + assert "/dev/dm-1" in html + assert "ext4" in html + + def test_resource_limits_section_present(self): + """Test that Resource Limits section is rendered.""" + html = self._render() + assert "Resource Limits" in html + + def test_resource_limits_values_rendered(self): + """Test key resource limit values appear (with thousands separators).""" + html = self._render() + assert "1,048,576" in html # open files + assert "125,971" in html # max procs + + def test_resource_limits_unlimited_rendered(self): + """Test unlimited values displayed correctly.""" + html = self._render() + assert "unlimited" in html # stack_size hard limit + + def test_check_details_structured(self): + """Test that check details use structured key-value instead of raw JSON.""" + html = self._render() + assert "dk-key" in html # structured detail key class + assert "dk-val" in html # structured detail value class + assert "Dcrw-rw-rw-" in html + + def test_no_system_section_when_missing(self): + """Test System Environment section is absent when no system data.""" + html = self._render({"checks": [], "check_summary": {"overall": "pass", "passed": 0, "failed": 0}}) + assert "

System Environment

" not in html + + def test_no_storage_section_when_missing(self): + """Test Storage section is absent when no storage data.""" + html = self._render({"checks": [], "check_summary": {"overall": "pass", "passed": 0, "failed": 0}}) + assert "NVMe Devices" not in html + + def test_no_resource_section_when_missing(self): + """Test Resource Limits section is absent when no resource data.""" + html = self._render({"checks": [], "check_summary": {"overall": "pass", "passed": 0, "failed": 0}}) + assert "

Resource Limits

" not in html + + def test_section_renamed_to_validation_checks(self): + """Test that old 'System & Validation' is now 'Validation Checks'.""" + html = self._render() + assert "Validation Checks" in html + assert "System & Validation" not in html + + +class TestP2Improvements: + """Tests for P2 improvements: context-dependent collapse, copy buttons, filesystem filtering, timestamps.""" + + FULL_DOCTOR_DATA = TestNewSections.FULL_DOCTOR_DATA + + def _render(self, doctor_data=None, overall_status="pass"): + combined = { + "timestamp": "2026-03-04T15:35:35Z", + "overall_status": overall_status, + "reports": { + "doctor": doctor_data or self.FULL_DOCTOR_DATA, + "bench": {}, + "objbench": {}, + }, + } + return render_html(combined) + + def test_system_env_collapsed_on_pass(self): + """System Environment should NOT have 'open' when overall status is pass.""" + html = self._render(overall_status="pass") + # Find the system-environment details tag + import re + match = re.search(r'
]*>', html) + assert match, "system-environment section not found" + assert "open" not in match.group(0) + + def test_system_env_open_on_fail(self): + """System Environment should be open when overall status is fail.""" + doctor = dict(self.FULL_DOCTOR_DATA) + doctor["check_summary"] = {"overall": "fail", "passed": 0, "failed": 1} + html = self._render(doctor_data=doctor, overall_status="fail") + import re + match = re.search(r'
]*>', html) + assert match, "system-environment section not found" + assert "open" in match.group(0) + + def test_copy_button_on_version(self): + """Copy button should appear next to Fusion version.""" + html = self._render() + assert 'data-copy="2.6-develop-1f517df"' in html + assert "copy-btn" in html + + def test_copy_button_on_check_details(self): + """Copy button should appear in check details.""" + html = self._render() + # Check details copy button should contain the JSON of details + assert 'data-copy="{' in html + + def test_timestamps_use_time_element(self): + """Timestamps should use
{% endif %} -
Environment
+

Environment

{% if doctor_report.system %} {% set sys = doctor_report.system %} {% set cloud = doctor_report.cloud or {} %}
-
Instance
+

Instance

Provider {{ cloud.provider|default('N/A')|upper }} @@ -979,7 +986,7 @@
-
System
+

System

{% if sys.os %} OS @@ -1019,7 +1026,7 @@ {% if doctor_report.storage and doctor_report.storage.nvme_devices %}
0 %} open{% endif %}> - Storage Devices +

Storage Devices

▼
@@ -1049,7 +1056,7 @@ {% if doctor_report.storage and doctor_report.storage.filesystems %}
0 %} open{% endif %}> - Mounted Filesystems +

Mounted Filesystems

▼
@@ -1095,7 +1102,7 @@ {% if doctor_report.resources %}
0 %} open{% endif %}> - Resource Limits +

Resource Limits

▼
@@ -1185,7 +1192,7 @@ {% if bench_report and bench_report.keys()|length > 0 and 'error' not in bench_report %}
- Filesystem Benchmark +

Filesystem Benchmark

▼
@@ -1223,7 +1230,7 @@ {% if objbench_report and objbench_report.keys()|length > 0 and 'error' not in objbench_report %}
- Object Storage Benchmark +

Object Storage Benchmark

▼
@@ -1260,14 +1267,25 @@
", + "remediation": "", + }, + ], + }, + "bench": {}, + "objbench": {}, + }, + } + + html = render_html(combined_report) + assert ""}, + }, + ], + }, + "bench": {}, + "objbench": {}, + }, + } + + html = render_html(combined_report) + assert "" not in html + + def test_render_does_not_mutate_input(self): + """Test that render_html does not modify the input dictionary.""" + combined_report = { + "timestamp": "2026-02-24T10:00:00Z", + "overall_status": "pass", + "reports": { + "doctor": { + "storage": { + "filesystems": [ + { + "device": "/dev/sda1", + "mount_point": "/", + "type": "ext4", + "total_bytes": 100000000000, + "available_bytes": 50000000000, + }, + ], + }, + "checks": [], + "check_summary": {"overall": "pass", "passed": 0, "failed": 0}, + }, + "bench": {}, + "objbench": {}, + }, + } + + import copy + original = copy.deepcopy(combined_report) + render_html(combined_report) + assert "_used_pct" not in combined_report["reports"]["doctor"]["storage"]["filesystems"][0] + assert combined_report == original + class TestLoadTemplate: """Test cases for load_template function.""" @@ -398,72 +420,300 @@ def test_load_template_is_html(self): assert "ulimit -n" in str(result) + + def test_inline_code_escapes_html(self): + result = inline_code("") + assert " + + \ No newline at end of file diff --git a/examples/fusion-doctor-report-degraded.json b/examples/fusion-doctor-report-degraded.json new file mode 100644 index 0000000..137738c --- /dev/null +++ b/examples/fusion-doctor-report-degraded.json @@ -0,0 +1,358 @@ +{ + "schema_version": "1.2", + "fusion_version": "2.6-develop-b6a157c", + "timestamp": "2026-03-05T16:17:08.334755668Z", + "system": { + "os": { + "name": "ubuntu", + "version": "24.04", + "kernel": "6.1.161-183.298.amzn2023.x86_64", + "architecture": "x86_64" + }, + "cpu": { + "model": "Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz", + "cores": 1, + "threads": 2 + }, + "memory": { + "total_bytes": 8161673216, + "available_bytes": 7473061888, + "swap_total_bytes": 0, + "swap_free_bytes": 0 + } + }, + "cloud": { + "provider": "aws", + "instance_id": "i-022224f636401f5cc", + "instance_type": "m6id.large", + "region": "eu-west-1", + "zone": "eu-west-1a" + }, + "storage": { + "filesystems": [ + { + "device": "none", + "mount_point": "/", + "type": "overlay", + "total_bytes": 32132542464, + "available_bytes": 28244172800 + }, + { + "device": "/dev/nvme1n1", + "mount_point": "/tmp", + "type": "xfs", + "total_bytes": 117932888064, + "available_bytes": 117076611072 + }, + { + "device": "/dev/nvme0n1p1", + "mount_point": "/etc/resolv.conf", + "type": "xfs", + "total_bytes": 32132542464, + "available_bytes": 28244172800 + }, + { + "device": "/etc/resolv.conf", + "mount_point": "/etc/hostname", + "type": "xfs", + "total_bytes": 32132542464, + "available_bytes": 28244172800 + }, + { + "device": "/etc/hostname", + "mount_point": "/etc/hosts", + "type": "xfs", + "total_bytes": 32132542464, + "available_bytes": 28244172800 + }, + { + "device": "none", + "mount_point": "/fusion", + "type": "fuse.fusion", + "total_bytes": 8589934592, + "available_bytes": 4294967296 + } + ], + "nvme_devices": [ + { + "name": "nvme0", + "model": "Amazon Elastic Block Store", + "size_bytes": 32212254720, + "block_devices": ["nvme0n1"] + }, + { + "name": "nvme1", + "model": "Amazon EC2 NVMe Instance Storage", + "size_bytes": 118000000000, + "block_devices": ["nvme1n1"] + } + ] + }, + "resources": { + "cpu_time": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "file_size": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "data_seg_size": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "stack_size": { + "soft": 10485760, + "hard": 10485760 + }, + "core_file_size": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "resident_set": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "max_procs": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "open_files": { + "soft": 1024, + "hard": 65536 + }, + "mem_lock": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "address_space": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "file_locks": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "pending_signals": { + "soft": 30446, + "hard": 30446 + }, + "msg_queue_size": { + "soft": 819200, + "hard": 819200 + }, + "max_nice": { + "soft": 0, + "hard": 0 + }, + "max_rt_priority": { + "soft": 0, + "hard": 0 + }, + "rt_time": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + } + }, + "checks": [ + { + "check": "fuse_device", + "category": "critical", + "status": "pass", + "message": "/dev/fuse is available and accessible", + "details": { + "path": "/dev/fuse", + "permissions": "Dcrw-rw-rw-" + }, + "duration_ms": 0 + }, + { + "check": "kernel_version", + "category": "critical", + "status": "pass", + "message": "6.1.161-183.298.amzn2023.x86_64 >= 5.10", + "details": { + "kernel_version": "6.1.161-183.298.amzn2023.x86_64", + "required_min": "5.10" + }, + "duration_ms": 0 + }, + { + "check": "memory", + "category": "warning", + "status": "pass", + "message": "7.6 GiB total >= 4.0 GiB required", + "details": { + "total_bytes": 8161673216, + "available_bytes": 7473061888, + "required_bytes": 4294967296 + }, + "duration_ms": 0 + }, + { + "check": "disk_space", + "category": "warning", + "status": "pass", + "message": "109.0 GiB available >= 100.0 GiB required", + "details": { + "path": "/tmp", + "mount_point": "/tmp", + "total_bytes": 117932888064, + "available_bytes": 117076611072, + "required_bytes": 107374182400 + }, + "duration_ms": 0 + }, + { + "check": "cpu_cores", + "category": "warning", + "status": "fail", + "message": "1 cores available < 2 required", + "details": { + "cores_available": 1, + "cores_required": 2 + }, + "remediation": "Choose an instance type with more CPU cores.", + "duration_ms": 0 + }, + { + "check": "open_files", + "category": "warning", + "status": "fail", + "message": "open files soft limit 1024 < 65535 required", + "details": { + "soft_limit": 1024, + "hard_limit": 65536, + "required_min": 65535 + }, + "remediation": "Increase the open files limit with `ulimit -n` or update /etc/security/limits.conf.", + "duration_ms": 0 + }, + { + "check": "bucket_access_rw", + "category": "critical", + "status": "pass", + "message": "bucket s3://fusion-ci is accessible", + "details": { + "uri": "s3://fusion-ci" + }, + "duration_ms": 186, + "sub_checks": [ + { + "check": "exists", + "category": "critical", + "status": "pass", + "message": "check bucket exists: ok", + "duration_ms": 47 + }, + { + "check": "list", + "category": "critical", + "status": "pass", + "message": "list objects: ok", + "duration_ms": 39 + }, + { + "check": "upload", + "category": "critical", + "status": "pass", + "message": "upload file: ok", + "duration_ms": 26 + }, + { + "check": "download", + "category": "critical", + "status": "pass", + "message": "download file: ok", + "duration_ms": 51 + }, + { + "check": "delete", + "category": "critical", + "status": "pass", + "message": "delete file: ok", + "duration_ms": 20 + } + ] + }, + { + "check": "bucket_access_rw", + "category": "critical", + "status": "pass", + "message": "bucket s3://fusion-ci/scratch/60y3Jy9oQRkhjw is accessible", + "details": { + "uri": "s3://fusion-ci/scratch/60y3Jy9oQRkhjw" + }, + "duration_ms": 149, + "sub_checks": [ + { + "check": "exists", + "category": "critical", + "status": "pass", + "message": "check bucket exists: ok", + "duration_ms": 6 + }, + { + "check": "list", + "category": "critical", + "status": "pass", + "message": "list objects: ok", + "duration_ms": 18 + }, + { + "check": "upload", + "category": "critical", + "status": "pass", + "message": "upload file: ok", + "duration_ms": 18 + }, + { + "check": "download", + "category": "critical", + "status": "pass", + "message": "download file: ok", + "duration_ms": 88 + }, + { + "check": "delete", + "category": "critical", + "status": "pass", + "message": "delete file: ok", + "duration_ms": 17 + } + ] + }, + { + "check": "bucket_access_ro", + "category": "critical", + "status": "pass", + "message": "bucket s3://ngi-igenomes is accessible", + "details": { + "uri": "s3://ngi-igenomes" + }, + "duration_ms": 82, + "sub_checks": [ + { + "check": "exists", + "category": "critical", + "status": "pass", + "message": "check bucket exists: ok", + "duration_ms": 62 + }, + { + "check": "list", + "category": "critical", + "status": "pass", + "message": "list objects: ok", + "duration_ms": 19 + } + ] + } + ], + "check_summary": { + "overall": "fail", + "passed": 7, + "failed": 2, + "skipped": 0 + } +} diff --git a/examples/fusion-doctor-report-failed.html b/examples/fusion-doctor-report-failed.html new file mode 100644 index 0000000..552b7f9 --- /dev/null +++ b/examples/fusion-doctor-report-failed.html @@ -0,0 +1,1522 @@ + + + + + + Fusion Filesystem Diagnostics + + + + + + +
+
+
+
+ +
+
+

Fusion Filesystem Diagnostics

+
+ +
+
+
+ fail +
+
+ +
+ +
+

Overview

+
+
+
+ 11 +
+
Total Checks
+
+
+
8
+
Passed
+
+
+
1
+
Warnings
+
+
+
2
+
Critical
+
+
+
+ + +
+ +

Recommendations

+2 items ▼ + +
+
    +
  • + critical + The current credentials do not have sufficient permissions for this bucket. +
    • +
  • + warning + Choose an instance type with more CPU cores. +
    • +
+
+
+ + +
+ +

System Checks

+ 4/5 passed + ▼ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CheckResultStatusDetails
FUSE DeviceDetectedpass
Kernel Version6.1.161-183.298.amzn2023.x86_64pass
Memory7.6 GiBpass
CPU Cores1warn
Open Files65,535pass
+
+
+ + +
+ +

Storage Checks

+ 1/1 passed + ▼ + +
+ + + + + + + + + + + + + + + + + +
PathTotal CapacityStatusDetails
/tmp109.8 GiBpass
+
+
+ + +
+ +

Object Storage Checks

+ 3/5 passed + ▼ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
URIResultStatusDetails
read-writerws3://fusion-ci5/5 checks passedpass
read-writerws3://fusion-qa0/5 checks passedfail
read-writerws3://ngi-igenomes2/5 checks passedfail
read-writerws3://fusion-ci/scratch/60y3Jy9oQRkhjw5/5 checks passedpass
read-onlyros3://ngi-igenomes2/2 checks passedpass
+
+
+ +

Environment

+ + +
+
+

Instance

+
+ Provider + AWS + + ID + i-022224f636401f5cc + + Type + m6id.large + + Region + eu-west-1 + + Zone + eu-west-1a +
+
+ +
+

System

+
+ OS + Ubuntu 24.04 + + Kernel + 6.1.161-183.298.amzn2023.x86_64 + + Architecture + x86_64 + CPU + Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz + + Cores / Threads + 1 / 2 + Memory + 7.6 GiB +
+
+
+ + +
+ +

Storage Devices

+ ▼ + +
+ + + + + + + + + + + + + + + + + + + + +
NVMe DeviceModelSize
nvme0Amazon Elastic Block Store30 GiB
nvme1Amazon EC2 NVMe Instance Storage110 GiB
+
+
+ + +
+ +

Mounted Filesystems

+ ▼ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MountDeviceTypeUsage
/noneoverlay +
+
+
+
+ 12% + 3.6 / 29.9 GiB +
+
/tmp/dev/nvme1n1xfs +
+
+
+
+ 1% + 0.8 / 109.8 GiB +
+
/etc/resolv.conf/dev/nvme0n1p1xfs +
+
+
+
+ 12% + 3.6 / 29.9 GiB +
+
/etc/hostname/etc/resolv.confxfs +
+
+
+
+ 12% + 3.6 / 29.9 GiB +
+
/etc/hosts/etc/hostnamexfs +
+
+
+
+ 12% + 3.6 / 29.9 GiB +
+
/fusionnonefuse.fusion +
+
+
+
+ 50% + 4.0 / 8.0 GiB +
+
+
+
+ + +
+ +

Resource Limits

+ ▼ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ResourceSoft LimitHard LimitDetails
Open Files + 65,535 65,536
Max Processes + unlimited unlimited
Memory Lock + unlimited unlimited
Stack Size + 10.0 MB 10.0 MB
Max File Size + unlimited unlimited
Address Space + unlimited unlimited
+
+
+ + + + +
+ + +
+
+
+ © 2019-2026 Seqera | + 2.6-develop-b6a157c
+ +
+
+ + + + \ No newline at end of file diff --git a/examples/fusion-doctor-report-failed.json b/examples/fusion-doctor-report-failed.json new file mode 100644 index 0000000..f4b8be2 --- /dev/null +++ b/examples/fusion-doctor-report-failed.json @@ -0,0 +1,459 @@ +{ + "schema_version": "1.2", + "fusion_version": "2.6-develop-b6a157c", + "timestamp": "2026-03-05T16:17:08.334755668Z", + "system": { + "os": { + "name": "ubuntu", + "version": "24.04", + "kernel": "6.1.161-183.298.amzn2023.x86_64", + "architecture": "x86_64" + }, + "cpu": { + "model": "Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz", + "cores": 1, + "threads": 2 + }, + "memory": { + "total_bytes": 8161673216, + "available_bytes": 7473061888, + "swap_total_bytes": 0, + "swap_free_bytes": 0 + } + }, + "cloud": { + "provider": "aws", + "instance_id": "i-022224f636401f5cc", + "instance_type": "m6id.large", + "region": "eu-west-1", + "zone": "eu-west-1a" + }, + "storage": { + "filesystems": [ + { + "device": "none", + "mount_point": "/", + "type": "overlay", + "total_bytes": 32132542464, + "available_bytes": 28244172800 + }, + { + "device": "/dev/nvme1n1", + "mount_point": "/tmp", + "type": "xfs", + "total_bytes": 117932888064, + "available_bytes": 117076611072 + }, + { + "device": "/dev/nvme0n1p1", + "mount_point": "/etc/resolv.conf", + "type": "xfs", + "total_bytes": 32132542464, + "available_bytes": 28244172800 + }, + { + "device": "/etc/resolv.conf", + "mount_point": "/etc/hostname", + "type": "xfs", + "total_bytes": 32132542464, + "available_bytes": 28244172800 + }, + { + "device": "/etc/hostname", + "mount_point": "/etc/hosts", + "type": "xfs", + "total_bytes": 32132542464, + "available_bytes": 28244172800 + }, + { + "device": "none", + "mount_point": "/fusion", + "type": "fuse.fusion", + "total_bytes": 8589934592, + "available_bytes": 4294967296 + } + ], + "nvme_devices": [ + { + "name": "nvme0", + "model": "Amazon Elastic Block Store", + "size_bytes": 32212254720, + "block_devices": ["nvme0n1"] + }, + { + "name": "nvme1", + "model": "Amazon EC2 NVMe Instance Storage", + "size_bytes": 118000000000, + "block_devices": ["nvme1n1"] + } + ] + }, + "resources": { + "cpu_time": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "file_size": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "data_seg_size": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "stack_size": { + "soft": 10485760, + "hard": 10485760 + }, + "core_file_size": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "resident_set": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "max_procs": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "open_files": { + "soft": 65535, + "hard": 65536 + }, + "mem_lock": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "address_space": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "file_locks": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "pending_signals": { + "soft": 30446, + "hard": 30446 + }, + "msg_queue_size": { + "soft": 819200, + "hard": 819200 + }, + "max_nice": { + "soft": 0, + "hard": 0 + }, + "max_rt_priority": { + "soft": 0, + "hard": 0 + }, + "rt_time": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + } + }, + "checks": [ + { + "check": "fuse_device", + "category": "critical", + "status": "pass", + "message": "/dev/fuse is available and accessible", + "details": { + "path": "/dev/fuse", + "permissions": "Dcrw-rw-rw-" + }, + "duration_ms": 0 + }, + { + "check": "kernel_version", + "category": "critical", + "status": "pass", + "message": "6.1.161-183.298.amzn2023.x86_64 \u003e= 5.10", + "details": { + "kernel_version": "6.1.161-183.298.amzn2023.x86_64", + "required_min": "5.10" + }, + "duration_ms": 0 + }, + { + "check": "memory", + "category": "warning", + "status": "pass", + "message": "7.6 GiB total \u003e= 4.0 GiB required", + "details": { + "total_bytes": 8161673216, + "available_bytes": 7473061888, + "required_bytes": 4294967296 + }, + "duration_ms": 0 + }, + { + "check": "disk_space", + "category": "warning", + "status": "pass", + "message": "109.0 GiB available \u003e= 100.0 GiB required", + "details": { + "path": "/tmp", + "mount_point": "/tmp", + "total_bytes": 117932888064, + "available_bytes": 117076611072, + "required_bytes": 107374182400 + }, + "duration_ms": 0 + }, + { + "check": "cpu_cores", + "category": "warning", + "status": "fail", + "message": "1 cores available \u003c 2 required", + "details": { + "cores_available": 1, + "cores_required": 2 + }, + "remediation": "Choose an instance type with more CPU cores.", + "duration_ms": 0 + }, + { + "check": "open_files", + "category": "warning", + "status": "pass", + "message": "open files soft limit 65535 >= 65535 required", + "details": { + "soft_limit": 65535, + "hard_limit": 65536, + "required_min": 65535 + }, + "duration_ms": 0 + }, + { + "check": "bucket_access_rw", + "category": "critical", + "status": "pass", + "message": "bucket s3://fusion-ci is accessible", + "details": { + "uri": "s3://fusion-ci" + }, + "duration_ms": 186, + "sub_checks": [ + { + "check": "exists", + "category": "critical", + "status": "pass", + "message": "check bucket exists: ok", + "duration_ms": 47 + }, + { + "check": "list", + "category": "critical", + "status": "pass", + "message": "list objects: ok", + "duration_ms": 39 + }, + { + "check": "upload", + "category": "critical", + "status": "pass", + "message": "upload file: ok", + "duration_ms": 26 + }, + { + "check": "download", + "category": "critical", + "status": "pass", + "message": "download file: ok", + "duration_ms": 51 + }, + { + "check": "delete", + "category": "critical", + "status": "pass", + "message": "delete file: ok", + "duration_ms": 20 + } + ] + }, + { + "check": "bucket_access_rw", + "category": "critical", + "status": "fail", + "message": "bucket check failed for s3://fusion-qa", + "details": { + "uri": "s3://fusion-qa" + }, + "remediation": "The current credentials do not have sufficient permissions for this bucket.", + "duration_ms": 37, + "sub_checks": [ + { + "check": "exists", + "category": "critical", + "status": "fail", + "message": "check bucket exists: not found (HTTP 404)", + "details": { + "error": "element not found\noperation error S3: HeadBucket, https response error StatusCode: 404, RequestID: R6F5BRJABEPT67ZQ, HostID: eUeGSSs0BHHnluIdkYAUhvjUCnwD6ZAlJRQCGs1p7+RyXvUYEP2ogF0JHq68Y+Lbf49Og9GG/KoaT/bPphxrCmtshnHav8dO, NotFound: " + }, + "duration_ms": 37 + }, + { + "check": "list", + "category": "critical", + "status": "skip", + "message": "", + "duration_ms": 0 + }, + { + "check": "upload", + "category": "critical", + "status": "skip", + "message": "", + "duration_ms": 0 + }, + { + "check": "download", + "category": "critical", + "status": "skip", + "message": "", + "duration_ms": 0 + }, + { + "check": "delete", + "category": "critical", + "status": "skip", + "message": "", + "duration_ms": 0 + } + ] + }, + { + "check": "bucket_access_rw", + "category": "critical", + "status": "fail", + "message": "bucket check failed for s3://ngi-igenomes", + "details": { + "uri": "s3://ngi-igenomes" + }, + "remediation": "The current credentials do not have sufficient permissions for this bucket.", + "duration_ms": 158, + "sub_checks": [ + { + "check": "exists", + "category": "critical", + "status": "pass", + "message": "check bucket exists: ok", + "duration_ms": 74 + }, + { + "check": "list", + "category": "critical", + "status": "pass", + "message": "list objects: ok", + "duration_ms": 45 + }, + { + "check": "upload", + "category": "critical", + "status": "fail", + "message": "upload file: operation error S3: PutObject, https response error StatusCode: 403, RequestID: R6FC1C0FWMRCKXGW, HostID: 8cEbiE4lTY7YgV077uBHWkjjGsBezqW4Tlp1VmoODTVDmops4E3U2Ym67RlcfGHeCm+05p1g2QwCr8vmvXYoOmvlysCF2ApI, api error AccessDenied: Access Denied", + "details": { + "error": "operation error S3: PutObject, https response error StatusCode: 403, RequestID: R6FC1C0FWMRCKXGW, HostID: 8cEbiE4lTY7YgV077uBHWkjjGsBezqW4Tlp1VmoODTVDmops4E3U2Ym67RlcfGHeCm+05p1g2QwCr8vmvXYoOmvlysCF2ApI, api error AccessDenied: Access Denied" + }, + "duration_ms": 37 + }, + { + "check": "download", + "category": "critical", + "status": "skip", + "message": "", + "duration_ms": 0 + }, + { + "check": "delete", + "category": "critical", + "status": "skip", + "message": "", + "duration_ms": 0 + } + ] + }, + { + "check": "bucket_access_rw", + "category": "critical", + "status": "pass", + "message": "bucket s3://fusion-ci/scratch/60y3Jy9oQRkhjw is accessible", + "details": { + "uri": "s3://fusion-ci/scratch/60y3Jy9oQRkhjw" + }, + "duration_ms": 149, + "sub_checks": [ + { + "check": "exists", + "category": "critical", + "status": "pass", + "message": "check bucket exists: ok", + "duration_ms": 6 + }, + { + "check": "list", + "category": "critical", + "status": "pass", + "message": "list objects: ok", + "duration_ms": 18 + }, + { + "check": "upload", + "category": "critical", + "status": "pass", + "message": "upload file: ok", + "duration_ms": 18 + }, + { + "check": "download", + "category": "critical", + "status": "pass", + "message": "download file: ok", + "duration_ms": 88 + }, + { + "check": "delete", + "category": "critical", + "status": "pass", + "message": "delete file: ok", + "duration_ms": 17 + } + ] + }, + { + "check": "bucket_access_ro", + "category": "critical", + "status": "pass", + "message": "bucket s3://ngi-igenomes is accessible", + "details": { + "uri": "s3://ngi-igenomes" + }, + "duration_ms": 82, + "sub_checks": [ + { + "check": "exists", + "category": "critical", + "status": "pass", + "message": "check bucket exists: ok", + "duration_ms": 62 + }, + { + "check": "list", + "category": "critical", + "status": "pass", + "message": "list objects: ok", + "duration_ms": 19 + } + ] + } + ], + "check_summary": { + "overall": "fail", + "passed": 8, + "failed": 3, + "skipped": 0 + } +} diff --git a/examples/fusion-doctor-report-passed.html b/examples/fusion-doctor-report-passed.html new file mode 100644 index 0000000..0d82e36 --- /dev/null +++ b/examples/fusion-doctor-report-passed.html @@ -0,0 +1,1456 @@ + + + + + + Fusion Filesystem Diagnostics + + + + + + +
+
+
+
+ +
+
+

Fusion Filesystem Diagnostics

+
+ +
+
+
+ pass +
+
+ +
+ +
+

Overview

+
+
+
+ 9 +
+
Total Checks
+
+
+
9
+
Passed
+
+
+
0
+
Warnings
+
+
+
0
+
Critical
+
+
+
+ + +
+ +

Recommendations

+ ▼ + +
+

All checks passed, no recommendations.

+
+
+ + +
+ +

System Checks

+ 5/5 passed + ▼ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CheckResultStatusDetails
FUSE DeviceDetectedpass
Kernel Version6.1.161-183.298.amzn2023.x86_64pass
Memory7.6 GiBpass
CPU Cores2pass
Open Files65,535pass
+
+
+ + +
+ +

Storage Checks

+ 1/1 passed + ▼ + +
+ + + + + + + + + + + + + + + + + +
PathTotal CapacityStatusDetails
/tmp109.8 GiBpass
+
+
+ + +
+ +

Object Storage Checks

+ 3/3 passed + ▼ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
URIResultStatusDetails
read-writerws3://fusion-ci5/5 checks passedpass
read-writerws3://fusion-ci/scratch/60y3Jy9oQRkhjw5/5 checks passedpass
read-onlyros3://ngi-igenomes2/2 checks passedpass
+
+
+ +

Environment

+ + +
+
+

Instance

+
+ Provider + AWS + + ID + i-022224f636401f5cc + + Type + m6id.large + + Region + eu-west-1 + + Zone + eu-west-1a +
+
+ +
+

System

+
+ OS + Ubuntu 24.04 + + Kernel + 6.1.161-183.298.amzn2023.x86_64 + + Architecture + x86_64 + CPU + Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz + + Cores / Threads + 2 / 2 + Memory + 7.6 GiB +
+
+
+ + +
+ +

Storage Devices

+ ▼ + +
+ + + + + + + + + + + + + + + + + + + + +
NVMe DeviceModelSize
nvme0Amazon Elastic Block Store30 GiB
nvme1Amazon EC2 NVMe Instance Storage110 GiB
+
+
+ + +
+ +

Mounted Filesystems

+ ▼ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MountDeviceTypeUsage
/noneoverlay +
+
+
+
+ 12% + 3.6 / 29.9 GiB +
+
/tmp/dev/nvme1n1xfs +
+
+
+
+ 1% + 0.8 / 109.8 GiB +
+
/etc/resolv.conf/dev/nvme0n1p1xfs +
+
+
+
+ 12% + 3.6 / 29.9 GiB +
+
/etc/hostname/etc/resolv.confxfs +
+
+
+
+ 12% + 3.6 / 29.9 GiB +
+
/etc/hosts/etc/hostnamexfs +
+
+
+
+ 12% + 3.6 / 29.9 GiB +
+
/fusionnonefuse.fusion +
+
+
+
+ 50% + 4.0 / 8.0 GiB +
+
+
+
+ + +
+ +

Resource Limits

+ ▼ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ResourceSoft LimitHard LimitDetails
Open Files + 65,535 65,536
Max Processes + unlimited unlimited
Memory Lock + unlimited unlimited
Stack Size + 10.0 MB 10.0 MB
Max File Size + unlimited unlimited
Address Space + unlimited unlimited
+
+
+ + + + +
+ + +
+
+
+ © 2019-2026 Seqera | + 2.6-develop-b6a157c
+ +
+
+ + + + \ No newline at end of file diff --git a/examples/fusion-doctor-report-passed.json b/examples/fusion-doctor-report-passed.json new file mode 100644 index 0000000..004e5ff --- /dev/null +++ b/examples/fusion-doctor-report-passed.json @@ -0,0 +1,356 @@ +{ + "schema_version": "1.2", + "fusion_version": "2.6-develop-b6a157c", + "timestamp": "2026-03-05T16:17:08.334755668Z", + "system": { + "os": { + "name": "ubuntu", + "version": "24.04", + "kernel": "6.1.161-183.298.amzn2023.x86_64", + "architecture": "x86_64" + }, + "cpu": { + "model": "Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz", + "cores": 2, + "threads": 2 + }, + "memory": { + "total_bytes": 8161673216, + "available_bytes": 7473061888, + "swap_total_bytes": 0, + "swap_free_bytes": 0 + } + }, + "cloud": { + "provider": "aws", + "instance_id": "i-022224f636401f5cc", + "instance_type": "m6id.large", + "region": "eu-west-1", + "zone": "eu-west-1a" + }, + "storage": { + "filesystems": [ + { + "device": "none", + "mount_point": "/", + "type": "overlay", + "total_bytes": 32132542464, + "available_bytes": 28244172800 + }, + { + "device": "/dev/nvme1n1", + "mount_point": "/tmp", + "type": "xfs", + "total_bytes": 117932888064, + "available_bytes": 117076611072 + }, + { + "device": "/dev/nvme0n1p1", + "mount_point": "/etc/resolv.conf", + "type": "xfs", + "total_bytes": 32132542464, + "available_bytes": 28244172800 + }, + { + "device": "/etc/resolv.conf", + "mount_point": "/etc/hostname", + "type": "xfs", + "total_bytes": 32132542464, + "available_bytes": 28244172800 + }, + { + "device": "/etc/hostname", + "mount_point": "/etc/hosts", + "type": "xfs", + "total_bytes": 32132542464, + "available_bytes": 28244172800 + }, + { + "device": "none", + "mount_point": "/fusion", + "type": "fuse.fusion", + "total_bytes": 8589934592, + "available_bytes": 4294967296 + } + ], + "nvme_devices": [ + { + "name": "nvme0", + "model": "Amazon Elastic Block Store", + "size_bytes": 32212254720, + "block_devices": ["nvme0n1"] + }, + { + "name": "nvme1", + "model": "Amazon EC2 NVMe Instance Storage", + "size_bytes": 118000000000, + "block_devices": ["nvme1n1"] + } + ] + }, + "resources": { + "cpu_time": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "file_size": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "data_seg_size": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "stack_size": { + "soft": 10485760, + "hard": 10485760 + }, + "core_file_size": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "resident_set": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "max_procs": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "open_files": { + "soft": 65535, + "hard": 65536 + }, + "mem_lock": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "address_space": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "file_locks": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + }, + "pending_signals": { + "soft": 30446, + "hard": 30446 + }, + "msg_queue_size": { + "soft": 819200, + "hard": 819200 + }, + "max_nice": { + "soft": 0, + "hard": 0 + }, + "max_rt_priority": { + "soft": 0, + "hard": 0 + }, + "rt_time": { + "soft": 18446744073709551615, + "hard": 18446744073709551615 + } + }, + "checks": [ + { + "check": "fuse_device", + "category": "critical", + "status": "pass", + "message": "/dev/fuse is available and accessible", + "details": { + "path": "/dev/fuse", + "permissions": "Dcrw-rw-rw-" + }, + "duration_ms": 0 + }, + { + "check": "kernel_version", + "category": "critical", + "status": "pass", + "message": "6.1.161-183.298.amzn2023.x86_64 >= 5.10", + "details": { + "kernel_version": "6.1.161-183.298.amzn2023.x86_64", + "required_min": "5.10" + }, + "duration_ms": 0 + }, + { + "check": "memory", + "category": "warning", + "status": "pass", + "message": "7.6 GiB total >= 4.0 GiB required", + "details": { + "total_bytes": 8161673216, + "available_bytes": 7473061888, + "required_bytes": 4294967296 + }, + "duration_ms": 0 + }, + { + "check": "disk_space", + "category": "warning", + "status": "pass", + "message": "109.0 GiB available >= 100.0 GiB required", + "details": { + "path": "/tmp", + "mount_point": "/tmp", + "total_bytes": 117932888064, + "available_bytes": 117076611072, + "required_bytes": 107374182400 + }, + "duration_ms": 0 + }, + { + "check": "cpu_cores", + "category": "warning", + "status": "pass", + "message": "2 cores available >= 2 required", + "details": { + "cores_available": 2, + "cores_required": 2 + }, + "duration_ms": 0 + }, + { + "check": "open_files", + "category": "warning", + "status": "pass", + "message": "open files soft limit 65535 >= 65535 required", + "details": { + "soft_limit": 65535, + "hard_limit": 65536, + "required_min": 65535 + }, + "duration_ms": 0 + }, + { + "check": "bucket_access_rw", + "category": "critical", + "status": "pass", + "message": "bucket s3://fusion-ci is accessible", + "details": { + "uri": "s3://fusion-ci" + }, + "duration_ms": 186, + "sub_checks": [ + { + "check": "exists", + "category": "critical", + "status": "pass", + "message": "check bucket exists: ok", + "duration_ms": 47 + }, + { + "check": "list", + "category": "critical", + "status": "pass", + "message": "list objects: ok", + "duration_ms": 39 + }, + { + "check": "upload", + "category": "critical", + "status": "pass", + "message": "upload file: ok", + "duration_ms": 26 + }, + { + "check": "download", + "category": "critical", + "status": "pass", + "message": "download file: ok", + "duration_ms": 51 + }, + { + "check": "delete", + "category": "critical", + "status": "pass", + "message": "delete file: ok", + "duration_ms": 20 + } + ] + }, + { + "check": "bucket_access_rw", + "category": "critical", + "status": "pass", + "message": "bucket s3://fusion-ci/scratch/60y3Jy9oQRkhjw is accessible", + "details": { + "uri": "s3://fusion-ci/scratch/60y3Jy9oQRkhjw" + }, + "duration_ms": 149, + "sub_checks": [ + { + "check": "exists", + "category": "critical", + "status": "pass", + "message": "check bucket exists: ok", + "duration_ms": 6 + }, + { + "check": "list", + "category": "critical", + "status": "pass", + "message": "list objects: ok", + "duration_ms": 18 + }, + { + "check": "upload", + "category": "critical", + "status": "pass", + "message": "upload file: ok", + "duration_ms": 18 + }, + { + "check": "download", + "category": "critical", + "status": "pass", + "message": "download file: ok", + "duration_ms": 88 + }, + { + "check": "delete", + "category": "critical", + "status": "pass", + "message": "delete file: ok", + "duration_ms": 17 + } + ] + }, + { + "check": "bucket_access_ro", + "category": "critical", + "status": "pass", + "message": "bucket s3://ngi-igenomes is accessible", + "details": { + "uri": "s3://ngi-igenomes" + }, + "duration_ms": 82, + "sub_checks": [ + { + "check": "exists", + "category": "critical", + "status": "pass", + "message": "check bucket exists: ok", + "duration_ms": 62 + }, + { + "check": "list", + "category": "critical", + "status": "pass", + "message": "list objects: ok", + "duration_ms": 19 + } + ] + } + ], + "check_summary": { + "overall": "pass", + "passed": 9, + "failed": 0, + "skipped": 0 + } +} diff --git a/examples/generate_reports.sh b/examples/generate_reports.sh new file mode 100755 index 0000000..7774d40 --- /dev/null +++ b/examples/generate_reports.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Generate HTML reports from example doctor JSONs +set -euo pipefail +cd "$(dirname "$0")/.." +for json in examples/fusion-doctor-report-*.json; do + html="${json%.json}.html" + uv run --no-project --script bin/generate_fusion_report.py \ + --doctor "$json" --output-html "$html" --output-json /dev/null + echo "$html" +done From bffe20e6002e814af6046391cb11a7f4ff1ebd9a Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Mon, 9 Mar 2026 12:14:18 +0100 Subject: [PATCH 42/53] refactor(fusion): Remove duplicate details from resource limits table The collapsible details and remediation are already shown in the System Checks section; resource limits now only shows values with status color and Low tag. Signed-off-by: Alberto Miranda --- assets/templates/fusion_report_template.html | 27 -------------------- examples/fusion-doctor-report-degraded.html | 25 +----------------- examples/fusion-doctor-report-failed.html | 24 +---------------- examples/fusion-doctor-report-passed.html | 24 +---------------- 4 files changed, 3 insertions(+), 97 deletions(-) diff --git a/assets/templates/fusion_report_template.html b/assets/templates/fusion_report_template.html index 72e73a2..8f8b912 100644 --- a/assets/templates/fusion_report_template.html +++ b/assets/templates/fusion_report_template.html @@ -1249,7 +1249,6 @@

Res {% macro rlimit_row(label, check_name, soft_display, hard_display) %} {% set chk = system_checks|selectattr('check', 'defined')|selectattr('check', 'equalto', check_name)|first if system_checks is iterable and system_checks is not mapping else none %} {% set req_min = chk.details.required_min if chk and chk.details and chk.details.required_min is defined else none %} - {% set has_details = chk and (chk.details or chk.remediation) %} {{ label }} @@ -1260,32 +1259,7 @@

Res {% endif %} {{ hard_display }} - {% if has_details %}{% endif %} - {% if has_details %} - - - {% if chk.details %} -
-
Details
-
- {% if chk.details is mapping %} - {% for key, val in chk.details.items() %} - {{ detail_label(key, chk.category|default('')) }} - {{ format_detail_value(val, key) }} - {% endfor %} - {% else %} - 
{{ chk.details|tojson(indent=2) }}
- {% endif %} -
-
- {% endif %} - {% if chk.remediation %} -
{{ chk.remediation|inline_code }}
- {% endif %} - - - {% endif %} {% endmacro %} @@ -1293,7 +1267,6 @@

Res

- diff --git a/examples/fusion-doctor-report-degraded.html b/examples/fusion-doctor-report-degraded.html index 83ba036..c3094ea 100644 --- a/examples/fusion-doctor-report-degraded.html +++ b/examples/fusion-doctor-report-degraded.html @@ -852,7 +852,7 @@

Fusion Filesystem Diagnostics

- +
@@ -1333,7 +1333,6 @@

Res

- @@ -1342,23 +1341,6 @@

Res

- - - - @@ -1366,7 +1348,6 @@

Res

- @@ -1374,7 +1355,6 @@

Res

- @@ -1382,7 +1362,6 @@

Res

- @@ -1390,7 +1369,6 @@

Res

- @@ -1398,7 +1376,6 @@

Res

- diff --git a/examples/fusion-doctor-report-failed.html b/examples/fusion-doctor-report-failed.html index 552b7f9..c228e29 100644 --- a/examples/fusion-doctor-report-failed.html +++ b/examples/fusion-doctor-report-failed.html @@ -852,7 +852,7 @@

Fusion Filesystem Diagnostics

- +
@@ -1388,7 +1388,6 @@

Res

- @@ -1397,22 +1396,6 @@

Res

- - - - @@ -1420,7 +1403,6 @@

Res

- @@ -1428,7 +1410,6 @@

Res

- @@ -1436,7 +1417,6 @@

Res

- @@ -1444,7 +1424,6 @@

Res

- @@ -1452,7 +1431,6 @@

Res

- diff --git a/examples/fusion-doctor-report-passed.html b/examples/fusion-doctor-report-passed.html index 0d82e36..43a843f 100644 --- a/examples/fusion-doctor-report-passed.html +++ b/examples/fusion-doctor-report-passed.html @@ -852,7 +852,7 @@

Fusion Filesystem Diagnostics

- +
@@ -1322,7 +1322,6 @@

Res

- @@ -1331,22 +1330,6 @@

Res

- - - - @@ -1354,7 +1337,6 @@

Res

- @@ -1362,7 +1344,6 @@

Res

- @@ -1370,7 +1351,6 @@

Res

- @@ -1378,7 +1358,6 @@

Res

- @@ -1386,7 +1365,6 @@

Res

- From 90b8b8cf5f001342187f5d682e0af0014de5a682 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Mon, 9 Mar 2026 14:43:45 +0100 Subject: [PATCH 43/53] feat(fusion): Add `fusion_redact` param to mask PII in diagnostics Adds a boolean parameter (disabled by default) that passes --redact to fusion doctor, stripping hostnames, IPs, bucket names, etc. from the diagnostic output. Signed-off-by: Alberto Miranda --- main.nf | 2 ++ nextflow_schema.json | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/main.nf b/main.nf index 3f6a332..3906990 100644 --- a/main.nf +++ b/main.nf @@ -392,6 +392,7 @@ process TEST_FUSION_DOCTOR { script: def disk_flag = "--check-disk-usage ${cache_path ?: '/tmp'}" + def redact_flag = params.fusion_redact ? "--redact" : "" // Build bucket args from lists def rw_bucket_args = rw_buckets ? rw_buckets.collect { bucket -> "--check-bucket-read-write ${bucket}" }.join(' ') : "" @@ -417,6 +418,7 @@ process TEST_FUSION_DOCTOR { --output fusion-doctor-report.json \\ --reference-profile ${reference_profile} \\ ${disk_flag} \\ + ${redact_flag} \\ ${rw_bucket_args} \\ ${ro_bucket_args} EXIT_CODE=\$? diff --git a/nextflow_schema.json b/nextflow_schema.json index 6042218..bc63a80 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -57,6 +57,13 @@ "help_text": "When enabled, run the TEST_FUSION_DOCTOR process to validate Fusion requirements.", "fa_icon": "fas fa-stethoscope" }, + "fusion_redact": { + "type": "boolean", + "description": "Redact PII from the Fusion diagnostics report.", + "help_text": "When enabled, instructs 'fusion doctor' to mask personally identifiable information (hostnames, IPs, bucket names, etc.) from the diagnostic output.", + "fa_icon": "fas fa-mask", + "default": false + }, "fusion_cache_path": { "type": "string", "description": "Filesystem path for Fusion cache directory.", From 0766a8e9e119ac99a7a7f2f348f05a10ecfce2e2 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 10 Mar 2026 11:52:54 +0100 Subject: [PATCH 44/53] chore: Leave profile handling to Nextflow Co-authored-by: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Signed-off-by: Alberto Miranda --- main.nf | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/main.nf b/main.nf index 3906990..acdf033 100644 --- a/main.nf +++ b/main.nf @@ -596,29 +596,5 @@ workflow NF_CANARY { } workflow { - // Validate that only one Fusion profile is specified - def fusionProfiles = workflow.profile.tokenize(',') - .findAll { it.trim().startsWith('fusion_') } - - if (fusionProfiles.size() > 1) { - log.error """ - ================================================================================ - ERROR: Multiple Fusion profiles detected: ${fusionProfiles.join(', ')} - - Only ONE Fusion profile can be specified at a time to ensure consistent - validation thresholds for the fusion doctor process. - - Please select only one profile from: - - fusion_aws_low, fusion_aws_recommended, fusion_aws_high - - fusion_google_low, fusion_google_recommended, fusion_google_high - - fusion_azure_low, fusion_azure_recommended, fusion_azure_high - - Example: nextflow run . -profile fusion_aws_recommended - ================================================================================ - """.stripIndent() - - System.exit(1) - } - NF_CANARY(params.run, params.skip, params.gpu, params.fusion) } From 18f2a2de3ca0741109de412998e53b137c523d4d Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 10 Mar 2026 11:56:42 +0100 Subject: [PATCH 45/53] chore: Replace horizontal dividers with subheadings Signed-off-by: Alberto Miranda --- README.md | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/README.md b/README.md index 42f61b4..8651999 100644 --- a/README.md +++ b/README.md @@ -69,38 +69,26 @@ Each test includes a brief comment explaining its purpose. In case of failure, r This process should succeed automatically with exit status 0. ---- - ### `TEST_CREATE_FILE` This process creates a file on the worker machine and then moves it to the working directory. ---- - ### `TEST_CREATE_EMPTY_FILE` This process creates an empty file on the worker machine and then moves it to the working directory. ---- - ### `TEST_CREATE_FOLDER` This process creates a folder in the working directory. ---- - ### `TEST_INPUT` This process retrieves a file from the working directory and reads its contents on the worker machine. ---- - ### `TEST_BIN_SCRIPT` Tests a shell script in the `bin/` directory that creates a single file. ---- - ### `TEST_STAGE_REMOTE` _Note: Enabled only if the parameter `--remoteFile` is specified._ @@ -113,64 +101,44 @@ nextflow run seqeralabs/nf-canary --remoteFile 'https://raw.githubusercontent.co Use this parameter to specify a file to access during runtime. ---- - ### `TEST_PASS_FILE` This process stages a file from the working directory to the worker node, copies it, and stages it back to the working directory. ---- - ### `TEST_PASS_FOLDER` This process stages a folder from the working directory to the worker node, copies it, and stages it back to the working directory. ---- - ### `TEST_PUBLISH_FILE` This process creates a file on the worker machine and writes it to the publishDir directory. By default, this is written to a subfolder called `output` in the working directory, but it can be overridden using the `--outdir` parameter. Use this to demonstrate the ability to publish to the relevant output directory. ---- - ### `TEST_PUBLISH_FOLDER` This process creates a folder on the worker machine and writes it to the publishDir directory. By default, this is written to a subfolder called `output` in the working directory, but it can be overridden using the `--outdir` parameter. Use this to demonstrate the ability to publish to the relevant output directory. ---- - ### `TEST_IGNORED_FAIL` This process should fail immediately but be ignored using the default configuration. ---- - ### `TEST_MV_FILE` Tests moving a file within the working directory. ---- - ### `TEST_MV_FOLDER_CONTENTS` Tests moving the contents of a folder to a new folder within the working directory. ---- - ### `TEST_VAL_INPUT` Test a process can accept a value as input. ---- - ### `TEST_GPU` _Note: Enabled only if the parameter `--gpu` is specified._ This process tests the ability to use a GPU. It uses the `pytorch` conda environment to test CUDA is available and working. This is disabled by default as it requires a GPU to be available which may not be true. ---- - ### `TEST_FUSION_DOCTOR` _Note: Enabled only if the parameter `--fusion` is specified (or set to `true` by a profile)._ From f887634ec76b9a834d0b4680da03244b0140d3b5 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 10 Mar 2026 12:01:55 +0100 Subject: [PATCH 46/53] fix: Clarify NVMe auto-selection for AWS CEs Signed-off-by: Alberto Miranda --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8651999..a71ea88 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,7 @@ nextflow run seqeralabs/nf-canary -profile fusion_aws_recommended > [!NOTE] > -> Seqera Platform auto-selects NVMe-based instance families when Fusion is enabled (e.g. `m6id`, `c6id`, `r6id`). Fusion can also work without NVMe instances, but in this case the EBS disk shall be bumped to 100 GB (`gp3`, 325 MB/s); this is what the `low` profile validates. +> Seqera Platform will auto-select NVMe-based instance families when Fusion is enabled (e.g. `m6id`, `c6id`, `r6id`) and the "Fast instance storage" toggle is active in the CE settings. Fusion can also work without NVMe instances, but in this case the EBS disk shall be bumped to 100 GB (`gp3`, 325 MB/s); this is what the `low` profile validates. | Profile | Disk | Memory | Kernel | Based on | | ------------------------ | ------ | ------ | ------ | ------------------- | From 01ccfe5f75c10f90a4fb3275488616472baf4922 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 10 Mar 2026 12:09:42 +0100 Subject: [PATCH 47/53] fix: Make path mandatory in `load_json_report` Signed-off-by: Alberto Miranda --- bin/generate_fusion_report.py | 14 ++++++-------- tests/test_generate_fusion_report.py | 6 ------ 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/bin/generate_fusion_report.py b/bin/generate_fusion_report.py index df63d4f..bdd9754 100755 --- a/bin/generate_fusion_report.py +++ b/bin/generate_fusion_report.py @@ -21,17 +21,15 @@ import humanize -def load_json_report(path: Optional[str]) -> Dict[str, Any]: - """Load a JSON report file, return empty dict if path is None. +def load_json_report(path: str) -> Dict[str, Any]: + """Load a JSON report file. Args: - path: Path to JSON report file, or None + path: Path to JSON report file Returns: Parsed JSON dictionary, or dict with "error" key if loading fails """ - if not path: - return {} try: with open(path, 'r') as f: @@ -63,9 +61,9 @@ def merge_reports( combined = { "timestamp": datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z'), "reports": { - "doctor": load_json_report(doctor_report), - "bench": load_json_report(bench_report), - "objbench": load_json_report(objbench_report), + "doctor": load_json_report(doctor_report) if doctor_report else {}, + "bench": load_json_report(bench_report) if bench_report else {}, + "objbench": load_json_report(objbench_report) if objbench_report else {}, }, } diff --git a/tests/test_generate_fusion_report.py b/tests/test_generate_fusion_report.py index ec02ff4..b1d6ba3 100755 --- a/tests/test_generate_fusion_report.py +++ b/tests/test_generate_fusion_report.py @@ -51,12 +51,6 @@ def test_load_valid_json(self, tmp_path): result = load_json_report(str(path)) assert result == {"status": "pass", "message": "All checks passed"} - def test_load_none_path(self): - assert load_json_report(None) == {} - - def test_load_empty_string_path(self): - assert load_json_report("") == {} - def test_load_missing_file(self): result = load_json_report("/nonexistent/path/file.json") assert "error" in result From 0e7735d9c7950085dfc783b7d6fb6e71441d12cf Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 10 Mar 2026 12:29:46 +0100 Subject: [PATCH 48/53] refactor(tests): Convert fusion doctor report test to process-level Signed-off-by: Alberto Miranda --- tests/main.test_fusion_doctor_report.nf.test | 68 ++++++++++---------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/tests/main.test_fusion_doctor_report.nf.test b/tests/main.test_fusion_doctor_report.nf.test index ec3d060..3e31787 100644 --- a/tests/main.test_fusion_doctor_report.nf.test +++ b/tests/main.test_fusion_doctor_report.nf.test @@ -1,50 +1,50 @@ -nextflow_pipeline { +nextflow_process { - name "Test FUSION_DOCTOR_GENERATE_REPORT conditional execution" + name "Test Process FUSION_DOCTOR_GENERATE_REPORT" script "main.nf" - - test("Report process does NOT run when fusion is disabled") { - - when { - params { - fusion = false - run = 'TEST_SUCCESS' // Run a different test - outdir = "${launchDir}/output" + process "FUSION_DOCTOR_GENERATE_REPORT" + + test("generates and publishes HTML and JSON reports when fusion is enabled") { + + setup { + run("TEST_FUSION_DOCTOR") { + script "main.nf" + params { + fusion = true + outdir = "${outputDir}/output" + } + process { + """ + input[0] = 'dummy' + def ref_profile = file("${outputDir}/fusion-reference-profile.yaml"); ref_profile.text = '' + input[1] = ref_profile // reference_profile + input[2] = [] // rw_buckets + input[3] = [] // ro_buckets + input[4] = '/tmp/cache' // cache_path + """ + } } } - then { - assertAll( - { assert workflow.success }, - // TEST_FUSION_DOCTOR should not run - { assert workflow.trace.tasks().findAll { it.name == 'NF_CANARY:TEST_FUSION_DOCTOR' }.size() == 0 }, - // FUSION_DOCTOR_GENERATE_REPORT should not run - { assert workflow.trace.tasks().findAll { it.name == 'NF_CANARY:FUSION_DOCTOR_GENERATE_REPORT' }.size() == 0 } - ) - } - - } - - test("Report process DOES run when fusion is enabled") { - when { params { - fusion = true - run = 'TEST_FUSION_DOCTOR' - outdir = "${launchDir}/output" + outdir = "${outputDir}/output" + } + process { + """ + input[0] = TEST_FUSION_DOCTOR.out.report + input[1] = file("${projectDir}/assets/templates/fusion_report_template.html") + """ } } then { assertAll( - { assert workflow.success }, - // Both processes should run (2 tasks total) - { assert workflow.trace.tasks().size() == 2 }, - { assert workflow.trace.succeeded().size() == 2 }, + { assert process.success }, // Check that report files exist in outputs - { assert path("${launchDir}/output/fusion/fusion-report.html").exists() }, - { assert path("${launchDir}/output/fusion/fusion-report.json").exists() }, - { assert path("${launchDir}/output/fusion-doctor-report.json").exists() } + { assert path("${outputDir}/output/fusion/fusion-report.html").exists() }, + { assert path("${outputDir}/output/fusion/fusion-report.json").exists() }, + { assert path("${outputDir}/output/fusion-doctor-report.json").exists() } ) } From 66e90f8a72fa7be03a24674157a7e8dbb8789248 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 10 Mar 2026 12:35:13 +0100 Subject: [PATCH 49/53] chore: Remove example report fixtures Signed-off-by: Alberto Miranda --- .prettierignore | 1 - examples/fusion-doctor-report-degraded.html | 1445 ------------------ examples/fusion-doctor-report-degraded.json | 358 ----- examples/fusion-doctor-report-failed.html | 1500 ------------------- examples/fusion-doctor-report-failed.json | 459 ------ examples/fusion-doctor-report-passed.html | 1434 ------------------ examples/fusion-doctor-report-passed.json | 356 ----- examples/generate_reports.sh | 10 - 8 files changed, 5563 deletions(-) delete mode 100644 examples/fusion-doctor-report-degraded.html delete mode 100644 examples/fusion-doctor-report-degraded.json delete mode 100644 examples/fusion-doctor-report-failed.html delete mode 100644 examples/fusion-doctor-report-failed.json delete mode 100644 examples/fusion-doctor-report-passed.html delete mode 100644 examples/fusion-doctor-report-passed.json delete mode 100755 examples/generate_reports.sh diff --git a/.prettierignore b/.prettierignore index 358dcf3..8c18fbc 100644 --- a/.prettierignore +++ b/.prettierignore @@ -14,5 +14,4 @@ results/ *.test.snap *.txt assets/templates/ -examples/ diff --git a/examples/fusion-doctor-report-degraded.html b/examples/fusion-doctor-report-degraded.html deleted file mode 100644 index c3094ea..0000000 --- a/examples/fusion-doctor-report-degraded.html +++ /dev/null @@ -1,1445 +0,0 @@ - - - - - - Fusion Filesystem Diagnostics - - - - - - -
-
-
-
- -
-
-

Fusion Filesystem Diagnostics

-
- -
-
-
- degraded -
-
- -
- -
-

Overview

-
-
-
- 9 -
-
Total Checks
-
-
-
7
-
Passed
-
-
-
2
-
Warnings
-
-
-
0
-
Critical
-
-
-
- - -
- -

Recommendations

-2 items ▼ - -
-
    -
  • - warning - Choose an instance type with more CPU cores. -
    • -
  • - warning - Increase the open files limit with ulimit -n or update /etc/security/limits.conf. -
    • -
-
-
- - -
- -

System Checks

- 3/5 passed - ▼ - -
-
Resource Soft Limit Hard LimitDetails
Resource Soft Limit Hard LimitDetails
1,024Low 65,536
unlimited unlimited
unlimited unlimited
10.0 MB 10.0 MB
unlimited unlimited
unlimited unlimited
Resource Soft Limit Hard LimitDetails
65,535 65,536
unlimited unlimited
unlimited unlimited
10.0 MB 10.0 MB
unlimited unlimited
unlimited unlimited
Resource Soft Limit Hard LimitDetails
65,535 65,536
unlimited unlimited
unlimited unlimited
10.0 MB 10.0 MB
unlimited unlimited
unlimited unlimited
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
CheckResultStatusDetails
FUSE DeviceDetectedpass
Kernel Version6.1.161-183.298.amzn2023.x86_64pass
Memory7.6 GiBpass
CPU Cores1warn
Open Files1,024warn
-

-
- - -
- -

Storage Checks

- 1/1 passed - ▼ - -
- - - - - - - - - - - - - - - - - -
PathTotal CapacityStatusDetails
/tmp109.8 GiBpass
-
-
- - -
- -

Object Storage Checks

- 3/3 passed - ▼ - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
URIResultStatusDetails
read-writerws3://fusion-ci5/5 checks passedpass
read-writerws3://fusion-ci/scratch/60y3Jy9oQRkhjw5/5 checks passedpass
read-onlyros3://ngi-igenomes2/2 checks passedpass
-
-
- -

Environment

- - -
-
-

Instance

-
- Provider - AWS - - ID - i-022224f636401f5cc - - Type - m6id.large - - Region - eu-west-1 - - Zone - eu-west-1a -
-
- -
-

System

-
- OS - Ubuntu 24.04 - - Kernel - 6.1.161-183.298.amzn2023.x86_64 - - Architecture - x86_64 - CPU - Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz - - Cores / Threads - 1 / 2 - Memory - 7.6 GiB -
-
-
- - -
- -

Storage Devices

- ▼ - -
- - - - - - - - - - - - - - - - - - - - -
NVMe DeviceModelSize
nvme0Amazon Elastic Block Store30 GiB
nvme1Amazon EC2 NVMe Instance Storage110 GiB
-
-
- - -
- -

Mounted Filesystems

- ▼ - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
MountDeviceTypeUsage
/noneoverlay -
-
-
-
- 12% - 3.6 / 29.9 GiB -
-
/tmp/dev/nvme1n1xfs -
-
-
-
- 1% - 0.8 / 109.8 GiB -
-
/etc/resolv.conf/dev/nvme0n1p1xfs -
-
-
-
- 12% - 3.6 / 29.9 GiB -
-
/etc/hostname/etc/resolv.confxfs -
-
-
-
- 12% - 3.6 / 29.9 GiB -
-
/etc/hosts/etc/hostnamexfs -
-
-
-
- 12% - 3.6 / 29.9 GiB -
-
/fusionnonefuse.fusion -
-
-
-
- 50% - 4.0 / 8.0 GiB -
-
-
-
- - -
- -

Resource Limits

- ▼ - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ResourceSoft LimitHard Limit
Open Files - 1,024Low 65,536
Max Processes - unlimited unlimited
Memory Lock - unlimited unlimited
Stack Size - 10.0 MB 10.0 MB
Max File Size - unlimited unlimited
Address Space - unlimited unlimited
-
-
- - - - -
- - -
-
-
- © 2019-2026 Seqera | - 2.6-develop-b6a157c
- -
-
- - - - \ No newline at end of file diff --git a/examples/fusion-doctor-report-degraded.json b/examples/fusion-doctor-report-degraded.json deleted file mode 100644 index 137738c..0000000 --- a/examples/fusion-doctor-report-degraded.json +++ /dev/null @@ -1,358 +0,0 @@ -{ - "schema_version": "1.2", - "fusion_version": "2.6-develop-b6a157c", - "timestamp": "2026-03-05T16:17:08.334755668Z", - "system": { - "os": { - "name": "ubuntu", - "version": "24.04", - "kernel": "6.1.161-183.298.amzn2023.x86_64", - "architecture": "x86_64" - }, - "cpu": { - "model": "Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz", - "cores": 1, - "threads": 2 - }, - "memory": { - "total_bytes": 8161673216, - "available_bytes": 7473061888, - "swap_total_bytes": 0, - "swap_free_bytes": 0 - } - }, - "cloud": { - "provider": "aws", - "instance_id": "i-022224f636401f5cc", - "instance_type": "m6id.large", - "region": "eu-west-1", - "zone": "eu-west-1a" - }, - "storage": { - "filesystems": [ - { - "device": "none", - "mount_point": "/", - "type": "overlay", - "total_bytes": 32132542464, - "available_bytes": 28244172800 - }, - { - "device": "/dev/nvme1n1", - "mount_point": "/tmp", - "type": "xfs", - "total_bytes": 117932888064, - "available_bytes": 117076611072 - }, - { - "device": "/dev/nvme0n1p1", - "mount_point": "/etc/resolv.conf", - "type": "xfs", - "total_bytes": 32132542464, - "available_bytes": 28244172800 - }, - { - "device": "/etc/resolv.conf", - "mount_point": "/etc/hostname", - "type": "xfs", - "total_bytes": 32132542464, - "available_bytes": 28244172800 - }, - { - "device": "/etc/hostname", - "mount_point": "/etc/hosts", - "type": "xfs", - "total_bytes": 32132542464, - "available_bytes": 28244172800 - }, - { - "device": "none", - "mount_point": "/fusion", - "type": "fuse.fusion", - "total_bytes": 8589934592, - "available_bytes": 4294967296 - } - ], - "nvme_devices": [ - { - "name": "nvme0", - "model": "Amazon Elastic Block Store", - "size_bytes": 32212254720, - "block_devices": ["nvme0n1"] - }, - { - "name": "nvme1", - "model": "Amazon EC2 NVMe Instance Storage", - "size_bytes": 118000000000, - "block_devices": ["nvme1n1"] - } - ] - }, - "resources": { - "cpu_time": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "file_size": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "data_seg_size": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "stack_size": { - "soft": 10485760, - "hard": 10485760 - }, - "core_file_size": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "resident_set": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "max_procs": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "open_files": { - "soft": 1024, - "hard": 65536 - }, - "mem_lock": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "address_space": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "file_locks": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "pending_signals": { - "soft": 30446, - "hard": 30446 - }, - "msg_queue_size": { - "soft": 819200, - "hard": 819200 - }, - "max_nice": { - "soft": 0, - "hard": 0 - }, - "max_rt_priority": { - "soft": 0, - "hard": 0 - }, - "rt_time": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - } - }, - "checks": [ - { - "check": "fuse_device", - "category": "critical", - "status": "pass", - "message": "/dev/fuse is available and accessible", - "details": { - "path": "/dev/fuse", - "permissions": "Dcrw-rw-rw-" - }, - "duration_ms": 0 - }, - { - "check": "kernel_version", - "category": "critical", - "status": "pass", - "message": "6.1.161-183.298.amzn2023.x86_64 >= 5.10", - "details": { - "kernel_version": "6.1.161-183.298.amzn2023.x86_64", - "required_min": "5.10" - }, - "duration_ms": 0 - }, - { - "check": "memory", - "category": "warning", - "status": "pass", - "message": "7.6 GiB total >= 4.0 GiB required", - "details": { - "total_bytes": 8161673216, - "available_bytes": 7473061888, - "required_bytes": 4294967296 - }, - "duration_ms": 0 - }, - { - "check": "disk_space", - "category": "warning", - "status": "pass", - "message": "109.0 GiB available >= 100.0 GiB required", - "details": { - "path": "/tmp", - "mount_point": "/tmp", - "total_bytes": 117932888064, - "available_bytes": 117076611072, - "required_bytes": 107374182400 - }, - "duration_ms": 0 - }, - { - "check": "cpu_cores", - "category": "warning", - "status": "fail", - "message": "1 cores available < 2 required", - "details": { - "cores_available": 1, - "cores_required": 2 - }, - "remediation": "Choose an instance type with more CPU cores.", - "duration_ms": 0 - }, - { - "check": "open_files", - "category": "warning", - "status": "fail", - "message": "open files soft limit 1024 < 65535 required", - "details": { - "soft_limit": 1024, - "hard_limit": 65536, - "required_min": 65535 - }, - "remediation": "Increase the open files limit with `ulimit -n` or update /etc/security/limits.conf.", - "duration_ms": 0 - }, - { - "check": "bucket_access_rw", - "category": "critical", - "status": "pass", - "message": "bucket s3://fusion-ci is accessible", - "details": { - "uri": "s3://fusion-ci" - }, - "duration_ms": 186, - "sub_checks": [ - { - "check": "exists", - "category": "critical", - "status": "pass", - "message": "check bucket exists: ok", - "duration_ms": 47 - }, - { - "check": "list", - "category": "critical", - "status": "pass", - "message": "list objects: ok", - "duration_ms": 39 - }, - { - "check": "upload", - "category": "critical", - "status": "pass", - "message": "upload file: ok", - "duration_ms": 26 - }, - { - "check": "download", - "category": "critical", - "status": "pass", - "message": "download file: ok", - "duration_ms": 51 - }, - { - "check": "delete", - "category": "critical", - "status": "pass", - "message": "delete file: ok", - "duration_ms": 20 - } - ] - }, - { - "check": "bucket_access_rw", - "category": "critical", - "status": "pass", - "message": "bucket s3://fusion-ci/scratch/60y3Jy9oQRkhjw is accessible", - "details": { - "uri": "s3://fusion-ci/scratch/60y3Jy9oQRkhjw" - }, - "duration_ms": 149, - "sub_checks": [ - { - "check": "exists", - "category": "critical", - "status": "pass", - "message": "check bucket exists: ok", - "duration_ms": 6 - }, - { - "check": "list", - "category": "critical", - "status": "pass", - "message": "list objects: ok", - "duration_ms": 18 - }, - { - "check": "upload", - "category": "critical", - "status": "pass", - "message": "upload file: ok", - "duration_ms": 18 - }, - { - "check": "download", - "category": "critical", - "status": "pass", - "message": "download file: ok", - "duration_ms": 88 - }, - { - "check": "delete", - "category": "critical", - "status": "pass", - "message": "delete file: ok", - "duration_ms": 17 - } - ] - }, - { - "check": "bucket_access_ro", - "category": "critical", - "status": "pass", - "message": "bucket s3://ngi-igenomes is accessible", - "details": { - "uri": "s3://ngi-igenomes" - }, - "duration_ms": 82, - "sub_checks": [ - { - "check": "exists", - "category": "critical", - "status": "pass", - "message": "check bucket exists: ok", - "duration_ms": 62 - }, - { - "check": "list", - "category": "critical", - "status": "pass", - "message": "list objects: ok", - "duration_ms": 19 - } - ] - } - ], - "check_summary": { - "overall": "fail", - "passed": 7, - "failed": 2, - "skipped": 0 - } -} diff --git a/examples/fusion-doctor-report-failed.html b/examples/fusion-doctor-report-failed.html deleted file mode 100644 index c228e29..0000000 --- a/examples/fusion-doctor-report-failed.html +++ /dev/null @@ -1,1500 +0,0 @@ - - - - - - Fusion Filesystem Diagnostics - - - - - - -
-
-
-
- -
-
-

Fusion Filesystem Diagnostics

-
- -
-
-
- fail -
-
- -
- -
-

Overview

-
-
-
- 11 -
-
Total Checks
-
-
-
8
-
Passed
-
-
-
1
-
Warnings
-
-
-
2
-
Critical
-
-
-
- - -
- -

Recommendations

-2 items ▼ - -
-
    -
  • - critical - The current credentials do not have sufficient permissions for this bucket. -
    • -
  • - warning - Choose an instance type with more CPU cores. -
    • -
-
-
- - -
- -

System Checks

- 4/5 passed - ▼ - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
CheckResultStatusDetails
FUSE DeviceDetectedpass
Kernel Version6.1.161-183.298.amzn2023.x86_64pass
Memory7.6 GiBpass
CPU Cores1warn
Open Files65,535pass
-
-
- - -
- -

Storage Checks

- 1/1 passed - ▼ - -
- - - - - - - - - - - - - - - - - -
PathTotal CapacityStatusDetails
/tmp109.8 GiBpass
-
-
- - -
- -

Object Storage Checks

- 3/5 passed - ▼ - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
URIResultStatusDetails
read-writerws3://fusion-ci5/5 checks passedpass
read-writerws3://fusion-qa0/5 checks passedfail
read-writerws3://ngi-igenomes2/5 checks passedfail
read-writerws3://fusion-ci/scratch/60y3Jy9oQRkhjw5/5 checks passedpass
read-onlyros3://ngi-igenomes2/2 checks passedpass
-
-
- -

Environment

- - -
-
-

Instance

-
- Provider - AWS - - ID - i-022224f636401f5cc - - Type - m6id.large - - Region - eu-west-1 - - Zone - eu-west-1a -
-
- -
-

System

-
- OS - Ubuntu 24.04 - - Kernel - 6.1.161-183.298.amzn2023.x86_64 - - Architecture - x86_64 - CPU - Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz - - Cores / Threads - 1 / 2 - Memory - 7.6 GiB -
-
-
- - -
- -

Storage Devices

- ▼ - -
- - - - - - - - - - - - - - - - - - - - -
NVMe DeviceModelSize
nvme0Amazon Elastic Block Store30 GiB
nvme1Amazon EC2 NVMe Instance Storage110 GiB
-
-
- - -
- -

Mounted Filesystems

- ▼ - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
MountDeviceTypeUsage
/noneoverlay -
-
-
-
- 12% - 3.6 / 29.9 GiB -
-
/tmp/dev/nvme1n1xfs -
-
-
-
- 1% - 0.8 / 109.8 GiB -
-
/etc/resolv.conf/dev/nvme0n1p1xfs -
-
-
-
- 12% - 3.6 / 29.9 GiB -
-
/etc/hostname/etc/resolv.confxfs -
-
-
-
- 12% - 3.6 / 29.9 GiB -
-
/etc/hosts/etc/hostnamexfs -
-
-
-
- 12% - 3.6 / 29.9 GiB -
-
/fusionnonefuse.fusion -
-
-
-
- 50% - 4.0 / 8.0 GiB -
-
-
-
- - -
- -

Resource Limits

- ▼ - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ResourceSoft LimitHard Limit
Open Files - 65,535 65,536
Max Processes - unlimited unlimited
Memory Lock - unlimited unlimited
Stack Size - 10.0 MB 10.0 MB
Max File Size - unlimited unlimited
Address Space - unlimited unlimited
-
-
- - - - -
- - -
-
-
- © 2019-2026 Seqera | - 2.6-develop-b6a157c
- -
-
- - - - \ No newline at end of file diff --git a/examples/fusion-doctor-report-failed.json b/examples/fusion-doctor-report-failed.json deleted file mode 100644 index f4b8be2..0000000 --- a/examples/fusion-doctor-report-failed.json +++ /dev/null @@ -1,459 +0,0 @@ -{ - "schema_version": "1.2", - "fusion_version": "2.6-develop-b6a157c", - "timestamp": "2026-03-05T16:17:08.334755668Z", - "system": { - "os": { - "name": "ubuntu", - "version": "24.04", - "kernel": "6.1.161-183.298.amzn2023.x86_64", - "architecture": "x86_64" - }, - "cpu": { - "model": "Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz", - "cores": 1, - "threads": 2 - }, - "memory": { - "total_bytes": 8161673216, - "available_bytes": 7473061888, - "swap_total_bytes": 0, - "swap_free_bytes": 0 - } - }, - "cloud": { - "provider": "aws", - "instance_id": "i-022224f636401f5cc", - "instance_type": "m6id.large", - "region": "eu-west-1", - "zone": "eu-west-1a" - }, - "storage": { - "filesystems": [ - { - "device": "none", - "mount_point": "/", - "type": "overlay", - "total_bytes": 32132542464, - "available_bytes": 28244172800 - }, - { - "device": "/dev/nvme1n1", - "mount_point": "/tmp", - "type": "xfs", - "total_bytes": 117932888064, - "available_bytes": 117076611072 - }, - { - "device": "/dev/nvme0n1p1", - "mount_point": "/etc/resolv.conf", - "type": "xfs", - "total_bytes": 32132542464, - "available_bytes": 28244172800 - }, - { - "device": "/etc/resolv.conf", - "mount_point": "/etc/hostname", - "type": "xfs", - "total_bytes": 32132542464, - "available_bytes": 28244172800 - }, - { - "device": "/etc/hostname", - "mount_point": "/etc/hosts", - "type": "xfs", - "total_bytes": 32132542464, - "available_bytes": 28244172800 - }, - { - "device": "none", - "mount_point": "/fusion", - "type": "fuse.fusion", - "total_bytes": 8589934592, - "available_bytes": 4294967296 - } - ], - "nvme_devices": [ - { - "name": "nvme0", - "model": "Amazon Elastic Block Store", - "size_bytes": 32212254720, - "block_devices": ["nvme0n1"] - }, - { - "name": "nvme1", - "model": "Amazon EC2 NVMe Instance Storage", - "size_bytes": 118000000000, - "block_devices": ["nvme1n1"] - } - ] - }, - "resources": { - "cpu_time": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "file_size": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "data_seg_size": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "stack_size": { - "soft": 10485760, - "hard": 10485760 - }, - "core_file_size": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "resident_set": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "max_procs": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "open_files": { - "soft": 65535, - "hard": 65536 - }, - "mem_lock": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "address_space": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "file_locks": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "pending_signals": { - "soft": 30446, - "hard": 30446 - }, - "msg_queue_size": { - "soft": 819200, - "hard": 819200 - }, - "max_nice": { - "soft": 0, - "hard": 0 - }, - "max_rt_priority": { - "soft": 0, - "hard": 0 - }, - "rt_time": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - } - }, - "checks": [ - { - "check": "fuse_device", - "category": "critical", - "status": "pass", - "message": "/dev/fuse is available and accessible", - "details": { - "path": "/dev/fuse", - "permissions": "Dcrw-rw-rw-" - }, - "duration_ms": 0 - }, - { - "check": "kernel_version", - "category": "critical", - "status": "pass", - "message": "6.1.161-183.298.amzn2023.x86_64 \u003e= 5.10", - "details": { - "kernel_version": "6.1.161-183.298.amzn2023.x86_64", - "required_min": "5.10" - }, - "duration_ms": 0 - }, - { - "check": "memory", - "category": "warning", - "status": "pass", - "message": "7.6 GiB total \u003e= 4.0 GiB required", - "details": { - "total_bytes": 8161673216, - "available_bytes": 7473061888, - "required_bytes": 4294967296 - }, - "duration_ms": 0 - }, - { - "check": "disk_space", - "category": "warning", - "status": "pass", - "message": "109.0 GiB available \u003e= 100.0 GiB required", - "details": { - "path": "/tmp", - "mount_point": "/tmp", - "total_bytes": 117932888064, - "available_bytes": 117076611072, - "required_bytes": 107374182400 - }, - "duration_ms": 0 - }, - { - "check": "cpu_cores", - "category": "warning", - "status": "fail", - "message": "1 cores available \u003c 2 required", - "details": { - "cores_available": 1, - "cores_required": 2 - }, - "remediation": "Choose an instance type with more CPU cores.", - "duration_ms": 0 - }, - { - "check": "open_files", - "category": "warning", - "status": "pass", - "message": "open files soft limit 65535 >= 65535 required", - "details": { - "soft_limit": 65535, - "hard_limit": 65536, - "required_min": 65535 - }, - "duration_ms": 0 - }, - { - "check": "bucket_access_rw", - "category": "critical", - "status": "pass", - "message": "bucket s3://fusion-ci is accessible", - "details": { - "uri": "s3://fusion-ci" - }, - "duration_ms": 186, - "sub_checks": [ - { - "check": "exists", - "category": "critical", - "status": "pass", - "message": "check bucket exists: ok", - "duration_ms": 47 - }, - { - "check": "list", - "category": "critical", - "status": "pass", - "message": "list objects: ok", - "duration_ms": 39 - }, - { - "check": "upload", - "category": "critical", - "status": "pass", - "message": "upload file: ok", - "duration_ms": 26 - }, - { - "check": "download", - "category": "critical", - "status": "pass", - "message": "download file: ok", - "duration_ms": 51 - }, - { - "check": "delete", - "category": "critical", - "status": "pass", - "message": "delete file: ok", - "duration_ms": 20 - } - ] - }, - { - "check": "bucket_access_rw", - "category": "critical", - "status": "fail", - "message": "bucket check failed for s3://fusion-qa", - "details": { - "uri": "s3://fusion-qa" - }, - "remediation": "The current credentials do not have sufficient permissions for this bucket.", - "duration_ms": 37, - "sub_checks": [ - { - "check": "exists", - "category": "critical", - "status": "fail", - "message": "check bucket exists: not found (HTTP 404)", - "details": { - "error": "element not found\noperation error S3: HeadBucket, https response error StatusCode: 404, RequestID: R6F5BRJABEPT67ZQ, HostID: eUeGSSs0BHHnluIdkYAUhvjUCnwD6ZAlJRQCGs1p7+RyXvUYEP2ogF0JHq68Y+Lbf49Og9GG/KoaT/bPphxrCmtshnHav8dO, NotFound: " - }, - "duration_ms": 37 - }, - { - "check": "list", - "category": "critical", - "status": "skip", - "message": "", - "duration_ms": 0 - }, - { - "check": "upload", - "category": "critical", - "status": "skip", - "message": "", - "duration_ms": 0 - }, - { - "check": "download", - "category": "critical", - "status": "skip", - "message": "", - "duration_ms": 0 - }, - { - "check": "delete", - "category": "critical", - "status": "skip", - "message": "", - "duration_ms": 0 - } - ] - }, - { - "check": "bucket_access_rw", - "category": "critical", - "status": "fail", - "message": "bucket check failed for s3://ngi-igenomes", - "details": { - "uri": "s3://ngi-igenomes" - }, - "remediation": "The current credentials do not have sufficient permissions for this bucket.", - "duration_ms": 158, - "sub_checks": [ - { - "check": "exists", - "category": "critical", - "status": "pass", - "message": "check bucket exists: ok", - "duration_ms": 74 - }, - { - "check": "list", - "category": "critical", - "status": "pass", - "message": "list objects: ok", - "duration_ms": 45 - }, - { - "check": "upload", - "category": "critical", - "status": "fail", - "message": "upload file: operation error S3: PutObject, https response error StatusCode: 403, RequestID: R6FC1C0FWMRCKXGW, HostID: 8cEbiE4lTY7YgV077uBHWkjjGsBezqW4Tlp1VmoODTVDmops4E3U2Ym67RlcfGHeCm+05p1g2QwCr8vmvXYoOmvlysCF2ApI, api error AccessDenied: Access Denied", - "details": { - "error": "operation error S3: PutObject, https response error StatusCode: 403, RequestID: R6FC1C0FWMRCKXGW, HostID: 8cEbiE4lTY7YgV077uBHWkjjGsBezqW4Tlp1VmoODTVDmops4E3U2Ym67RlcfGHeCm+05p1g2QwCr8vmvXYoOmvlysCF2ApI, api error AccessDenied: Access Denied" - }, - "duration_ms": 37 - }, - { - "check": "download", - "category": "critical", - "status": "skip", - "message": "", - "duration_ms": 0 - }, - { - "check": "delete", - "category": "critical", - "status": "skip", - "message": "", - "duration_ms": 0 - } - ] - }, - { - "check": "bucket_access_rw", - "category": "critical", - "status": "pass", - "message": "bucket s3://fusion-ci/scratch/60y3Jy9oQRkhjw is accessible", - "details": { - "uri": "s3://fusion-ci/scratch/60y3Jy9oQRkhjw" - }, - "duration_ms": 149, - "sub_checks": [ - { - "check": "exists", - "category": "critical", - "status": "pass", - "message": "check bucket exists: ok", - "duration_ms": 6 - }, - { - "check": "list", - "category": "critical", - "status": "pass", - "message": "list objects: ok", - "duration_ms": 18 - }, - { - "check": "upload", - "category": "critical", - "status": "pass", - "message": "upload file: ok", - "duration_ms": 18 - }, - { - "check": "download", - "category": "critical", - "status": "pass", - "message": "download file: ok", - "duration_ms": 88 - }, - { - "check": "delete", - "category": "critical", - "status": "pass", - "message": "delete file: ok", - "duration_ms": 17 - } - ] - }, - { - "check": "bucket_access_ro", - "category": "critical", - "status": "pass", - "message": "bucket s3://ngi-igenomes is accessible", - "details": { - "uri": "s3://ngi-igenomes" - }, - "duration_ms": 82, - "sub_checks": [ - { - "check": "exists", - "category": "critical", - "status": "pass", - "message": "check bucket exists: ok", - "duration_ms": 62 - }, - { - "check": "list", - "category": "critical", - "status": "pass", - "message": "list objects: ok", - "duration_ms": 19 - } - ] - } - ], - "check_summary": { - "overall": "fail", - "passed": 8, - "failed": 3, - "skipped": 0 - } -} diff --git a/examples/fusion-doctor-report-passed.html b/examples/fusion-doctor-report-passed.html deleted file mode 100644 index 43a843f..0000000 --- a/examples/fusion-doctor-report-passed.html +++ /dev/null @@ -1,1434 +0,0 @@ - - - - - - Fusion Filesystem Diagnostics - - - - - - -
-
-
-
- -
-
-

Fusion Filesystem Diagnostics

-
- -
-
-
- pass -
-
- -
- -
-

Overview

-
-
-
- 9 -
-
Total Checks
-
-
-
9
-
Passed
-
-
-
0
-
Warnings
-
-
-
0
-
Critical
-
-
-
- - -
- -

Recommendations

- ▼ - -
-

All checks passed, no recommendations.

-
-
- - -
- -

System Checks

- 5/5 passed - ▼ - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
CheckResultStatusDetails
FUSE DeviceDetectedpass
Kernel Version6.1.161-183.298.amzn2023.x86_64pass
Memory7.6 GiBpass
CPU Cores2pass
Open Files65,535pass
-
-
- - -
- -

Storage Checks

- 1/1 passed - ▼ - -
- - - - - - - - - - - - - - - - - -
PathTotal CapacityStatusDetails
/tmp109.8 GiBpass
-
-
- - -
- -

Object Storage Checks

- 3/3 passed - ▼ - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
URIResultStatusDetails
read-writerws3://fusion-ci5/5 checks passedpass
read-writerws3://fusion-ci/scratch/60y3Jy9oQRkhjw5/5 checks passedpass
read-onlyros3://ngi-igenomes2/2 checks passedpass
-
-
- -

Environment

- - -
-
-

Instance

-
- Provider - AWS - - ID - i-022224f636401f5cc - - Type - m6id.large - - Region - eu-west-1 - - Zone - eu-west-1a -
-
- -
-

System

-
- OS - Ubuntu 24.04 - - Kernel - 6.1.161-183.298.amzn2023.x86_64 - - Architecture - x86_64 - CPU - Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz - - Cores / Threads - 2 / 2 - Memory - 7.6 GiB -
-
-
- - -
- -

Storage Devices

- ▼ - -
- - - - - - - - - - - - - - - - - - - - -
NVMe DeviceModelSize
nvme0Amazon Elastic Block Store30 GiB
nvme1Amazon EC2 NVMe Instance Storage110 GiB
-
-
- - -
- -

Mounted Filesystems

- ▼ - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
MountDeviceTypeUsage
/noneoverlay -
-
-
-
- 12% - 3.6 / 29.9 GiB -
-
/tmp/dev/nvme1n1xfs -
-
-
-
- 1% - 0.8 / 109.8 GiB -
-
/etc/resolv.conf/dev/nvme0n1p1xfs -
-
-
-
- 12% - 3.6 / 29.9 GiB -
-
/etc/hostname/etc/resolv.confxfs -
-
-
-
- 12% - 3.6 / 29.9 GiB -
-
/etc/hosts/etc/hostnamexfs -
-
-
-
- 12% - 3.6 / 29.9 GiB -
-
/fusionnonefuse.fusion -
-
-
-
- 50% - 4.0 / 8.0 GiB -
-
-
-
- - -
- -

Resource Limits

- ▼ - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ResourceSoft LimitHard Limit
Open Files - 65,535 65,536
Max Processes - unlimited unlimited
Memory Lock - unlimited unlimited
Stack Size - 10.0 MB 10.0 MB
Max File Size - unlimited unlimited
Address Space - unlimited unlimited
-
-
- - - - -
- - -
-
-
- © 2019-2026 Seqera | - 2.6-develop-b6a157c
- -
-
- - - - \ No newline at end of file diff --git a/examples/fusion-doctor-report-passed.json b/examples/fusion-doctor-report-passed.json deleted file mode 100644 index 004e5ff..0000000 --- a/examples/fusion-doctor-report-passed.json +++ /dev/null @@ -1,356 +0,0 @@ -{ - "schema_version": "1.2", - "fusion_version": "2.6-develop-b6a157c", - "timestamp": "2026-03-05T16:17:08.334755668Z", - "system": { - "os": { - "name": "ubuntu", - "version": "24.04", - "kernel": "6.1.161-183.298.amzn2023.x86_64", - "architecture": "x86_64" - }, - "cpu": { - "model": "Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz", - "cores": 2, - "threads": 2 - }, - "memory": { - "total_bytes": 8161673216, - "available_bytes": 7473061888, - "swap_total_bytes": 0, - "swap_free_bytes": 0 - } - }, - "cloud": { - "provider": "aws", - "instance_id": "i-022224f636401f5cc", - "instance_type": "m6id.large", - "region": "eu-west-1", - "zone": "eu-west-1a" - }, - "storage": { - "filesystems": [ - { - "device": "none", - "mount_point": "/", - "type": "overlay", - "total_bytes": 32132542464, - "available_bytes": 28244172800 - }, - { - "device": "/dev/nvme1n1", - "mount_point": "/tmp", - "type": "xfs", - "total_bytes": 117932888064, - "available_bytes": 117076611072 - }, - { - "device": "/dev/nvme0n1p1", - "mount_point": "/etc/resolv.conf", - "type": "xfs", - "total_bytes": 32132542464, - "available_bytes": 28244172800 - }, - { - "device": "/etc/resolv.conf", - "mount_point": "/etc/hostname", - "type": "xfs", - "total_bytes": 32132542464, - "available_bytes": 28244172800 - }, - { - "device": "/etc/hostname", - "mount_point": "/etc/hosts", - "type": "xfs", - "total_bytes": 32132542464, - "available_bytes": 28244172800 - }, - { - "device": "none", - "mount_point": "/fusion", - "type": "fuse.fusion", - "total_bytes": 8589934592, - "available_bytes": 4294967296 - } - ], - "nvme_devices": [ - { - "name": "nvme0", - "model": "Amazon Elastic Block Store", - "size_bytes": 32212254720, - "block_devices": ["nvme0n1"] - }, - { - "name": "nvme1", - "model": "Amazon EC2 NVMe Instance Storage", - "size_bytes": 118000000000, - "block_devices": ["nvme1n1"] - } - ] - }, - "resources": { - "cpu_time": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "file_size": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "data_seg_size": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "stack_size": { - "soft": 10485760, - "hard": 10485760 - }, - "core_file_size": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "resident_set": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "max_procs": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "open_files": { - "soft": 65535, - "hard": 65536 - }, - "mem_lock": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "address_space": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "file_locks": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - }, - "pending_signals": { - "soft": 30446, - "hard": 30446 - }, - "msg_queue_size": { - "soft": 819200, - "hard": 819200 - }, - "max_nice": { - "soft": 0, - "hard": 0 - }, - "max_rt_priority": { - "soft": 0, - "hard": 0 - }, - "rt_time": { - "soft": 18446744073709551615, - "hard": 18446744073709551615 - } - }, - "checks": [ - { - "check": "fuse_device", - "category": "critical", - "status": "pass", - "message": "/dev/fuse is available and accessible", - "details": { - "path": "/dev/fuse", - "permissions": "Dcrw-rw-rw-" - }, - "duration_ms": 0 - }, - { - "check": "kernel_version", - "category": "critical", - "status": "pass", - "message": "6.1.161-183.298.amzn2023.x86_64 >= 5.10", - "details": { - "kernel_version": "6.1.161-183.298.amzn2023.x86_64", - "required_min": "5.10" - }, - "duration_ms": 0 - }, - { - "check": "memory", - "category": "warning", - "status": "pass", - "message": "7.6 GiB total >= 4.0 GiB required", - "details": { - "total_bytes": 8161673216, - "available_bytes": 7473061888, - "required_bytes": 4294967296 - }, - "duration_ms": 0 - }, - { - "check": "disk_space", - "category": "warning", - "status": "pass", - "message": "109.0 GiB available >= 100.0 GiB required", - "details": { - "path": "/tmp", - "mount_point": "/tmp", - "total_bytes": 117932888064, - "available_bytes": 117076611072, - "required_bytes": 107374182400 - }, - "duration_ms": 0 - }, - { - "check": "cpu_cores", - "category": "warning", - "status": "pass", - "message": "2 cores available >= 2 required", - "details": { - "cores_available": 2, - "cores_required": 2 - }, - "duration_ms": 0 - }, - { - "check": "open_files", - "category": "warning", - "status": "pass", - "message": "open files soft limit 65535 >= 65535 required", - "details": { - "soft_limit": 65535, - "hard_limit": 65536, - "required_min": 65535 - }, - "duration_ms": 0 - }, - { - "check": "bucket_access_rw", - "category": "critical", - "status": "pass", - "message": "bucket s3://fusion-ci is accessible", - "details": { - "uri": "s3://fusion-ci" - }, - "duration_ms": 186, - "sub_checks": [ - { - "check": "exists", - "category": "critical", - "status": "pass", - "message": "check bucket exists: ok", - "duration_ms": 47 - }, - { - "check": "list", - "category": "critical", - "status": "pass", - "message": "list objects: ok", - "duration_ms": 39 - }, - { - "check": "upload", - "category": "critical", - "status": "pass", - "message": "upload file: ok", - "duration_ms": 26 - }, - { - "check": "download", - "category": "critical", - "status": "pass", - "message": "download file: ok", - "duration_ms": 51 - }, - { - "check": "delete", - "category": "critical", - "status": "pass", - "message": "delete file: ok", - "duration_ms": 20 - } - ] - }, - { - "check": "bucket_access_rw", - "category": "critical", - "status": "pass", - "message": "bucket s3://fusion-ci/scratch/60y3Jy9oQRkhjw is accessible", - "details": { - "uri": "s3://fusion-ci/scratch/60y3Jy9oQRkhjw" - }, - "duration_ms": 149, - "sub_checks": [ - { - "check": "exists", - "category": "critical", - "status": "pass", - "message": "check bucket exists: ok", - "duration_ms": 6 - }, - { - "check": "list", - "category": "critical", - "status": "pass", - "message": "list objects: ok", - "duration_ms": 18 - }, - { - "check": "upload", - "category": "critical", - "status": "pass", - "message": "upload file: ok", - "duration_ms": 18 - }, - { - "check": "download", - "category": "critical", - "status": "pass", - "message": "download file: ok", - "duration_ms": 88 - }, - { - "check": "delete", - "category": "critical", - "status": "pass", - "message": "delete file: ok", - "duration_ms": 17 - } - ] - }, - { - "check": "bucket_access_ro", - "category": "critical", - "status": "pass", - "message": "bucket s3://ngi-igenomes is accessible", - "details": { - "uri": "s3://ngi-igenomes" - }, - "duration_ms": 82, - "sub_checks": [ - { - "check": "exists", - "category": "critical", - "status": "pass", - "message": "check bucket exists: ok", - "duration_ms": 62 - }, - { - "check": "list", - "category": "critical", - "status": "pass", - "message": "list objects: ok", - "duration_ms": 19 - } - ] - } - ], - "check_summary": { - "overall": "pass", - "passed": 9, - "failed": 0, - "skipped": 0 - } -} diff --git a/examples/generate_reports.sh b/examples/generate_reports.sh deleted file mode 100755 index 7774d40..0000000 --- a/examples/generate_reports.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash -# Generate HTML reports from example doctor JSONs -set -euo pipefail -cd "$(dirname "$0")/.." -for json in examples/fusion-doctor-report-*.json; do - html="${json%.json}.html" - uv run --no-project --script bin/generate_fusion_report.py \ - --doctor "$json" --output-html "$html" --output-json /dev/null - echo "$html" -done From 8c38d400c12158a7c8d468d03e6df31fe6935343 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 10 Mar 2026 12:54:17 +0100 Subject: [PATCH 50/53] refactor: Remove try/catch from `load_json_report`, catch at `main()` boundary Let errors propagate from library code instead of returning error dicts. Bad file paths now fail fast with a clean error message instead of producing partial results. Signed-off-by: Alberto Miranda --- bin/generate_fusion_report.py | 40 +++++++++++++--------------- tests/test_generate_fusion_report.py | 23 +++++++--------- 2 files changed, 28 insertions(+), 35 deletions(-) diff --git a/bin/generate_fusion_report.py b/bin/generate_fusion_report.py index bdd9754..357d4b1 100755 --- a/bin/generate_fusion_report.py +++ b/bin/generate_fusion_report.py @@ -28,18 +28,15 @@ def load_json_report(path: str) -> Dict[str, Any]: path: Path to JSON report file Returns: - Parsed JSON dictionary, or dict with "error" key if loading fails - """ + Parsed JSON dictionary - try: - with open(path, 'r') as f: - return json.load(f) - except FileNotFoundError: - return {"error": f"Report file not found: {path}"} - except json.JSONDecodeError as e: - return {"error": f"Malformed JSON in {path}: {str(e)}"} - except IOError as e: - return {"error": f"Cannot read {path}: {str(e)}"} + Raises: + FileNotFoundError: If the report file does not exist + json.JSONDecodeError: If the file contains malformed JSON + IOError: If the file cannot be read + """ + with open(path, 'r') as f: + return json.load(f) def merge_reports( @@ -72,7 +69,7 @@ def merge_reports( # Supports both legacy "summary.status" and real fusion "check_summary.overall" statuses = [] for report in combined["reports"].values(): - if report and "error" not in report: + if report: if "check_summary" in report: statuses.append(report["check_summary"].get("overall", "unknown")) elif "summary" in report: @@ -82,7 +79,7 @@ def merge_reports( # Check if failures are only in warning-category checks has_critical_failure = False for report in combined["reports"].values(): - if not report or "error" in report: + if not report: continue checks = report.get("checks", []) if isinstance(checks, list): @@ -415,11 +412,15 @@ def main(): sys.exit(1) # Merge all reports - combined = merge_reports( - doctor_report=args.doctor, - bench_report=args.bench, - objbench_report=args.objbench, - ) + try: + combined = merge_reports( + doctor_report=args.doctor, + bench_report=args.bench, + objbench_report=args.objbench, + ) + except (FileNotFoundError, json.JSONDecodeError, IOError) as e: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(1) # Write combined JSON try: @@ -437,9 +438,6 @@ def main(): except IOError as e: print(f"ERROR: Failed to write HTML report to {args.output_html}: {str(e)}", file=sys.stderr) sys.exit(1) - except Exception as e: - print(f"ERROR: Failed to render HTML report: {str(e)}", file=sys.stderr) - sys.exit(1) overall_status = combined.get("overall_status", "unknown") print(f"Reports generated:") diff --git a/tests/test_generate_fusion_report.py b/tests/test_generate_fusion_report.py index b1d6ba3..6fa1e12 100755 --- a/tests/test_generate_fusion_report.py +++ b/tests/test_generate_fusion_report.py @@ -52,22 +52,20 @@ def test_load_valid_json(self, tmp_path): assert result == {"status": "pass", "message": "All checks passed"} def test_load_missing_file(self): - result = load_json_report("/nonexistent/path/file.json") - assert "error" in result - assert "not found" in result["error"] + with pytest.raises(FileNotFoundError): + load_json_report("/nonexistent/path/file.json") def test_load_invalid_json(self, tmp_path): path = tmp_path / "bad.json" path.write_text("{invalid json content") - result = load_json_report(str(path)) - assert "error" in result - assert "Malformed JSON" in result["error"] + with pytest.raises(json.JSONDecodeError): + load_json_report(str(path)) def test_load_empty_json_file(self, tmp_path): path = tmp_path / "empty.json" path.write_text("") - result = load_json_report(str(path)) - assert "error" in result + with pytest.raises(json.JSONDecodeError): + load_json_report(str(path)) def test_load_complex_json(self, tmp_path): test_data = { @@ -150,12 +148,9 @@ def test_merge_partial_reports(self, write_reports): assert result["reports"]["objbench"] == {} def test_merge_with_invalid_report(self, write_reports): - doctor = {"summary": {"status": "pass"}} - paths = write_reports(doctor=doctor) - result = merge_reports(paths["doctor"], "/nonexistent/bench.json", None) - assert result["overall_status"] == "pass" - assert result["reports"]["doctor"] == doctor - assert "error" in result["reports"]["bench"] + paths = write_reports(doctor={"summary": {"status": "pass"}}) + with pytest.raises(FileNotFoundError): + merge_reports(paths["doctor"], "/nonexistent/bench.json", None) def test_merge_timestamp_present(self): result = merge_reports(None, None, None) From 061ccc35713a995e4b4b94ffd4a2ea992d20bb31 Mon Sep 17 00:00:00 2001 From: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com> Date: Thu, 12 Mar 2026 18:47:58 +0000 Subject: [PATCH 51/53] feat(fusion): add parameter sweep for fusion doctor validation Replace tiered fusion profiles (low/recommended/high per cloud) with simple per-cloud profiles and comma-separated sweep parameters. The FUSION_DOCTOR workflow builds a Cartesian product of all sweep values (kernel, memory, disk, NVMe, CPU, open files) and runs fusion doctor once per combination, collecting all reports into a single consolidated HTML/JSON output. Generated by Claude Code --- conf/fusion.config | 219 ++---------- main.nf | 208 +++++++++--- nextflow.config | 24 +- nextflow_schema.json | 60 +++- tests/main.fusion_doctor_workflow.nf.test | 385 ++++++++++++++++++++++ tests/main.nf.test | 39 +-- 6 files changed, 643 insertions(+), 292 deletions(-) create mode 100644 tests/main.fusion_doctor_workflow.nf.test diff --git a/conf/fusion.config b/conf/fusion.config index 90a4867..d060321 100644 --- a/conf/fusion.config +++ b/conf/fusion.config @@ -1,214 +1,51 @@ /* * Fusion Validation Profiles * - * These profiles define recommended thresholds for different cloud - * environments and workload sizes. Parameters are internally used to build - * a reference profile that is passed to `fusion doctor`. + * These profiles configure cloud-specific settings for running FUSION_DOCTOR. + * The parameter sweep values (kernel, memory, disk, CPU, open files) are + * defined as comma-separated defaults in nextflow.config and apply to all + * profiles automatically. * - * Tiers: - * low — small Nextflow workloads - * recommended — typical pipeline workloads - * high — large-scale production workloads - * - * Usage: nextflow run seqeralabs/nf-canary -profile fusion_aws_recommended - * - * --- Threshold review notes (2026-03-04) --- + * Usage: nextflow run seqeralabs/nf-canary -profile fusion_aws + * nextflow run seqeralabs/nf-canary -profile fusion_gcp + * nextflow run seqeralabs/nf-canary -profile fusion_azure * * Sources: - * - https://docs.seqera.io/platform-cloud/compute-envs/aws-batch * - https://docs.seqera.io/fusion/guide/aws-batch * - https://docs.seqera.io/fusion/guide/gcp-batch * - https://docs.seqera.io/platform-cloud/compute-envs/google-cloud-batch * - https://docs.seqera.io/platform-cloud/compute-envs/azure-batch - * - * Seqera docs requirements for Fusion (all clouds): - * - Local temp storage: at least 200 GB, random read speed 1000 MBps+ - * - For files >100 GB: 400 GB+ temp storage - * - * kernel_version_min: - * Minimum kernel shipped by Seqera Forge images: - * AWS - ECS-Optimized AL2023: 6.1 | legacy AL2: 5.10 (EOL June 2026) - * GCP - Ubuntu 22.04 LTS: 5.15+ | Ubuntu 24.04 LTS: 6.8+ - * Azure - Ubuntu HPC 22.04: 5.15 | legacy Ubuntu 20.04: 5.4 - * - * Decision: Use cloud-specific minimums: - * - AWS: 5.10 (covers legacy AL2 until EOL June 2026) - * - GCP/Azure: 5.15 (current-gen Forge images) - * - * memory_gb_min: - * No Fusion-specific minimum documented. Thresholds reflect workload size. - * Tiers: 4 GB (low), 8 GB (recommended), 16 GB (high). - * - * disk_gb_min: - * Thresholds match Seqera's documented minimums for Fusion. Note that - * Platform's UI only allows selecting instance families, not sizes. Small - * instances in valid families (e.g. m6id.large = 118 GB) may not meet the - * recommended threshold — this is expected and signals the instance is - * undersized for Fusion despite being in the right family. - * - * AWS: - * Platform auto-selects NVMe-based instance families when Fusion is - * enabled (e.g. m6id, c6id, r6id). 8xlarge+ recommended for production. - * - Without NVMe: EBS bumped to 100 GB (gp3, 325 MB/s) - * - With NVMe: starts at 118 GB (.large), 1900 GB (.8xlarge) - * Tiers: - * - 100 GB (low — EBS gp3 when Fusion enabled without NVMe) - * - 200 GB (recommended — Seqera docs minimum) - * - 950 GB (high — .4xlarge NVMe, 16 vCPUs, large datasets) - * - * GCP: - * Platform auto-selects families that support local SSDs (e.g. n2, c2, - * n2d). A 375 GB local NVMe SSD is provisioned per job. - * - Persistent disk: min 10 GiB (no local SSD) - * - Local NVMe SSD: 375 GiB increments (attached at creation time) - * - Production: n2-highmem-16 with local SSD, or larger - * Tiers: - * - 50 GB (low — persistent disk, small workloads) - * - 375 GB (recommended — 1x local NVMe SSD) - * - 750 GB (high — 2x local NVMe SSD) - * - * Azure: - * No auto-selection — user must pick VM size. Seqera recommends - * E-series with 'd' suffix (e.g. Standard_E8d_v5, Standard_E16d_v5). - * Standard SSDs only, no network-attached storage. - * - 'd' suffix VMs have local temp SSD (~37.5 GiB per vCPU) - * - Production: Standard_E16d_v5 or larger - * Tiers: - * - 75 GB (low — 2-vCPU 'd' VM, small workloads) - * - 300 GB (recommended — 8-vCPU 'd' VM, e.g. Standard_E8d_v5) - * - 600 GB (high — 16-vCPU 'd' VM, e.g. Standard_E16d_v5) - * - * nvme_required: - * Whether NVMe local storage is required. Fusion performs best with - * NVMe-backed instance storage for temp/scratch. Set to true for - * recommended/high tiers where NVMe families are expected, false for - * low tiers that may use EBS/persistent disk. - * - * cpu_cores_min: - * Minimum CPU cores. Seqera internal benchmarking recommends instances - * with 16 vCPUs for large, long-lived production pipelines. - * Tiers: 2 (low), 4 (recommended), 16 (high — matches docs). - * - * open_files_min: - * Minimum open file descriptor soft limit. Fusion opens many file handles - * for FUSE operations and cloud storage connections. 65535 is the common - * Linux default; production workloads benefit from higher limits. - * Tiers: 65535 (low), 131072 (recommended), 1048576 (high — matches docs). */ profiles { - // ---- AWS profiles ---- + // ---- AWS ---- + // Platform auto-selects NVMe-backed instance families (e.g. m6id, c6id, r6id) + // when Fusion is enabled — no machineType override needed. - fusion_aws_low { - // Small workloads — EBS-only, no NVMe - // 100 GB = EBS size Platform sets when Fusion is enabled without NVMe - params.fusion = true - params.fusion_kernel_version_min = "5.10" - params.fusion_memory_gb_min = 4 - params.fusion_disk_gb_min = 100 - params.fusion_nvme_required = false - params.fusion_cpu_cores_min = 2 - params.fusion_open_files_min = 65535 + fusion_aws { + params.fusion = true } - fusion_aws_recommended { - // Seqera docs: NVMe-based families (e.g. m6id, c6id, r6id), 8xlarge+ for production - // 200 GB = Seqera docs minimum for Fusion local temp storage - params.fusion = true - params.fusion_kernel_version_min = "5.10" - params.fusion_memory_gb_min = 8 - params.fusion_disk_gb_min = 200 - params.fusion_nvme_required = true - params.fusion_cpu_cores_min = 4 - params.fusion_open_files_min = 131072 - } - - fusion_aws_high { - // Large-scale production — NVMe families, .4xlarge+ instances (16 vCPUs) - // 950 GB = NVMe on .4xlarge; Seqera docs: 400 GB+ for files >100 GB - params.fusion = true - params.fusion_kernel_version_min = "5.10" - params.fusion_memory_gb_min = 16 - params.fusion_disk_gb_min = 950 - params.fusion_nvme_required = true - params.fusion_cpu_cores_min = 16 - params.fusion_open_files_min = 1048576 - } - - // ---- Google Cloud profiles ---- - - fusion_google_low { - // Small workloads — persistent disk, no local SSD - // 50 GB = modest persistent disk for small workloads - params.fusion = true - params.fusion_kernel_version_min = "5.15" - params.fusion_memory_gb_min = 4 - params.fusion_disk_gb_min = 50 - params.fusion_nvme_required = false - params.fusion_cpu_cores_min = 2 - params.fusion_open_files_min = 65535 - } + // ---- Google Cloud ---- + // Local NVMe SSDs are provisioned in 375 GB increments. + // Families supporting local SSDs: n2, c2, n2d (auto-selected by Platform). + // Recommended instance: n2-standard-8 or larger with local SSD attached. - fusion_google_recommended { - // Seqera docs: families supporting local SSDs (e.g. n2, c2, n2d) - // 375 GB = 1x local NVMe SSD (GCP provisions per job) - params.fusion = true - params.fusion_kernel_version_min = "5.15" - params.fusion_memory_gb_min = 8 - params.fusion_disk_gb_min = 375 - params.fusion_nvme_required = true - params.fusion_cpu_cores_min = 4 - params.fusion_open_files_min = 131072 + fusion_gcp { + params.fusion = true + process.machineType = 'n2-standard-8' + params.fusion_disk_gb_min = '375,750' // 1x or 2x local NVMe SSD } - fusion_google_high { - // Large-scale production — 2x local NVMe SSDs - // 750 GB = 2 x 375 GB; Seqera docs: 400 GB+ for files >100 GB - params.fusion = true - params.fusion_kernel_version_min = "5.15" - params.fusion_memory_gb_min = 16 - params.fusion_disk_gb_min = 750 - params.fusion_nvme_required = true - params.fusion_cpu_cores_min = 16 - params.fusion_open_files_min = 1048576 - } - - // ---- Azure profiles ---- - - fusion_azure_low { - // Small workloads — smallest 'd' suffix VM (2 vCPU) - // 75 GB = local temp disk on 2-vCPU 'd' VM - params.fusion = true - params.fusion_kernel_version_min = "5.15" - params.fusion_memory_gb_min = 4 - params.fusion_disk_gb_min = 75 - params.fusion_nvme_required = false - params.fusion_cpu_cores_min = 2 - params.fusion_open_files_min = 65535 - } - - fusion_azure_recommended { - // Seqera docs: E-series with 'd' suffix (e.g. Standard_E8d_v5, Standard_E16d_v5) - // 300 GB = 8-vCPU 'd' VM temp disk - params.fusion = true - params.fusion_kernel_version_min = "5.15" - params.fusion_memory_gb_min = 8 - params.fusion_disk_gb_min = 300 - params.fusion_nvme_required = true - params.fusion_cpu_cores_min = 4 - params.fusion_open_files_min = 131072 - } + // ---- Azure ---- + // User must select a VM with a 'd' suffix for local temp SSD. + // Seqera recommends E-series 'd' VMs (e.g. Standard_E8d_v5, Standard_E16d_v5). + // Local disk size is ~37.5 GiB per vCPU on these VM families. - fusion_azure_high { - // Large-scale production — 16-vCPU 'd' VM (e.g. Standard_E16d_v5) - // 600 GB = Standard_E16d_v5 temp disk (614 GiB) - params.fusion = true - params.fusion_kernel_version_min = "5.15" - params.fusion_memory_gb_min = 16 - params.fusion_disk_gb_min = 600 - params.fusion_nvme_required = true - params.fusion_cpu_cores_min = 16 - params.fusion_open_files_min = 1048576 + fusion_azure { + params.fusion = true + process.machineType = 'Standard_E8d_v5' + params.fusion_disk_gb_min = '300,600' // 8-vCPU or 16-vCPU 'd' VM } } diff --git a/main.nf b/main.nf index acdf033..a0d2260 100644 --- a/main.nf +++ b/main.nf @@ -373,30 +373,24 @@ process TEST_GPU { process TEST_FUSION_DOCTOR { /* Runs fusion doctor to validate the Fusion filesystem configuration. - Prints a text diagnostic report to stdout and saves a JSON report - to file. Fails if the fusion binary is not available in the task - environment. + The reference-profile YAML is staged in as a file (built via collectFile + in the workflow), avoiding any shell quoting or indentation issues. */ - publishDir { params.outdir ?: file(workflow.workDir).resolve("outputs/fusion").toUriString() }, mode: 'copy' + tag { meta.run_id } + publishDir { (params.outdir ? file(params.outdir) : file(workflow.workDir).resolve("outputs/fusion")).toUriString() }, mode: 'copy' input: - val(dummy_val) - path(reference_profile) - val(rw_buckets) - val(ro_buckets) - val(cache_path) + tuple val(dummy_val), val(meta), path(reference_profile), val(rw_buckets), val(ro_buckets) output: - path("fusion-doctor-report.json"), emit: report + path("fusion-doctor-report-${meta.run_id}.json"), emit: report script: - def disk_flag = "--check-disk-usage ${cache_path ?: '/tmp'}" - def redact_flag = params.fusion_redact ? "--redact" : "" - - // Build bucket args from lists - def rw_bucket_args = rw_buckets ? rw_buckets.collect { bucket -> "--check-bucket-read-write ${bucket}" }.join(' ') : "" - def ro_bucket_args = ro_buckets ? ro_buckets.collect { bucket -> "--check-bucket-read-only ${bucket}" }.join(' ') : "" + def disk_flag = "--check-disk-usage ${meta.cache_path ?: '/tmp'}" + def redact_flag = params.fusion_redact ? "--redact" : "" + def rw_bucket_args = rw_buckets ? rw_buckets.collect { b -> "--check-bucket-read-write ${b}" }.join(' ') : "" + def ro_bucket_args = ro_buckets ? ro_buckets.collect { b -> "--check-bucket-read-only ${b}" }.join(' ') : "" """ #!/bin/bash @@ -411,11 +405,10 @@ process TEST_FUSION_DOCTOR { fi fi - # Run fusion doctor and capture exit code - # Allow validation failures (exit codes 1 and 3) but abort on other errors + # Run fusion doctor; allow validation failures (exit 1/3) but abort on others set +e fusion doctor \\ - --output fusion-doctor-report.json \\ + --output fusion-doctor-report-${meta.run_id}.json \\ --reference-profile ${reference_profile} \\ ${disk_flag} \\ ${redact_flag} \\ @@ -424,30 +417,30 @@ process TEST_FUSION_DOCTOR { EXIT_CODE=\$? set -e - # Only allow exit codes 0, 1, and 3 (success and validation failures) - # Abort on any other exit code (non-validation errors) if [[ \$EXIT_CODE -ne 0 && \$EXIT_CODE -ne 1 && \$EXIT_CODE -ne 3 ]]; then echo "ERROR: fusion doctor failed with exit code \$EXIT_CODE (non-validation error)" >&2 exit \$EXIT_CODE fi - - # Exit successfully for validation failures to allow report generation exit 0 """ } process FUSION_DOCTOR_GENERATE_REPORT { /* - Aggregates doctor, bench, and objbench JSON reports into a single - consolidated HTML report and combined JSON report using the Python - generate_fusion_report.py script. + Aggregates one or more doctor JSON reports (one per parameter-sweep + combination) into a single consolidated HTML report and combined JSON + report using the Python generate_fusion_report.py script. + + When multiple doctor reports exist (parameter sweep), they are all + staged into the task directory and passed to the script via repeated + --doctor flags so the report covers every combination. */ container 'community.wave.seqera.io/library/jinja2_python_uv:7113b0a0e59d95a6' publishDir { (params.outdir ? file(params.outdir) : file(workflow.workDir).resolve("outputs")).resolve("fusion").toUriString() }, mode: 'copy' input: - path(doctor_report) + path(doctor_reports) // one or more JSON reports from the sweep path(template_file) output: @@ -455,21 +448,119 @@ process FUSION_DOCTOR_GENERATE_REPORT { path("fusion-report.json"), emit: json_report script: + def doctor_args = doctor_reports.collect { f -> "--doctor ${f}" }.join(' \\\n ') """ generate_fusion_report.py \\ - --doctor ${doctor_report} \\ + ${doctor_args} \\ --template ${template_file} \\ --output-html fusion-report.html \\ --output-json fusion-report.json """ } +/* sweepList +*/ +/** + * Split a comma-separated parameter string into a trimmed, non-empty list. + * Returns an empty list when the value is null, empty, or blank. + * Used to normalise all sweep parameters before building the Cartesian product. + * + * Examples: + * sweepList("5.10, 5.15") → ["5.10", "5.15"] + * sweepList("4,8, ,16") → ["4", "8", "16"] + * sweepList(null) → [] + */ +def sweepList(v) { + v ? v.toString().tokenize(',').collect { p -> p.trim() }.findAll { p -> p } : [] +} + +workflow FUSION_DOCTOR { + take: + trigger_ch // val channel — one item fires the whole sweep + kernel_version_min // e.g. "5.10,5.15" + memory_gb_min // e.g. "4,8,16" + disk_gb_min // e.g. "100,200,950" + nvme_required // e.g. "false,true" + cpu_cores_min // e.g. "2,4,16" + open_files_min // e.g. "65535,131072,1048576" + cache_path // e.g. "/tmp" + read_write_buckets // comma-separated bucket URIs + read_only_buckets // comma-separated bucket URIs + + main: + def rw_buckets_list = sweepList(read_write_buckets) + [workflow.workDir.toUriString()] + def ro_buckets_list = sweepList(read_only_buckets) + + def kernel_sweep = sweepList(kernel_version_min) + def memory_sweep = sweepList(memory_gb_min) + def disk_sweep = sweepList(disk_gb_min) + def nvme_sweep = sweepList(nvme_required) + def cpu_sweep = sweepList(cpu_cores_min) + def openf_sweep = sweepList(open_files_min) + def cache_sweep = sweepList(cache_path ?: '/tmp') + + trigger_ch + .combine(channel.fromList(kernel_sweep)) + .combine(channel.fromList(memory_sweep)) + .combine(channel.fromList(disk_sweep)) + .combine(channel.fromList(nvme_sweep)) + .combine(channel.fromList(cpu_sweep)) + .combine(channel.fromList(openf_sweep)) + .combine(channel.fromList(cache_sweep)) + .map { dummy_val, kernel, memory, disk, nvme, cpu, openf, cache -> + def parts = [] + if (kernel) parts << "k${kernel.replaceAll('[^a-zA-Z0-9]', '_')}" + if (memory) parts << "mem${memory}" + if (disk) parts << "disk${disk}" + if (nvme) parts << "nvme${nvme}" + if (cpu) parts << "cpu${cpu}" + if (openf) parts << "of${openf}" + if (cache && cache != '/tmp') parts << "cache${cache.replaceAll('[^a-zA-Z0-9]', '_')}" + + def yaml_lines = [] + if (kernel) yaml_lines << "kernel_version_min: \"${kernel}\"" + if (memory) yaml_lines << "memory_gb_min: ${memory}" + if (disk) yaml_lines << "disk_gb_min: ${disk}" + if (nvme) yaml_lines << "nvme_required: ${nvme}" + if (cpu) yaml_lines << "cpu_cores_min: ${cpu}" + if (openf) yaml_lines << "open_files_min: ${openf}" + + def run_id = parts ? parts.join('_') : 'default' + [run_id, dummy_val, [run_id: run_id, cache_path: cache ?: '/tmp'], yaml_lines.join('\n')] + } + .set { sweep_ch } + + // Materialise each per-combination YAML string as a staged file, + // then rejoin on run_id to rebuild the full process input tuple. + sweep_ch + .map { run_id, dummy_val, meta, yaml_text -> [run_id, yaml_text] } + .collectFile { run_id, yaml_text -> [ "fusion-reference-profile-${run_id}.yaml", yaml_text + '\n' ] } + .map { f -> [f.baseName.replace('fusion-reference-profile-', ''), f] } + .join(sweep_ch.map { run_id, dummy_val, meta, yaml_text -> [run_id, dummy_val, meta] }) + .map { run_id, reference_profile, dummy_val, meta -> + [dummy_val, meta, reference_profile, rw_buckets_list, ro_buckets_list] + } + .set { inputs_ch } + + emit: + inputs = inputs_ch // tuple: [dummy_val, meta, reference_profile, rw_buckets, ro_buckets] +} + workflow NF_CANARY { take: run_tools skip_tools gpu fusion + fusion_kernel_version_min + fusion_memory_gb_min + fusion_disk_gb_min + fusion_nvme_required + fusion_cpu_cores_min + fusion_open_files_min + fusion_cache_path + fusion_read_write_buckets + fusion_read_only_buckets main: def default_run_tools = [ @@ -494,7 +585,7 @@ workflow NF_CANARY { def run = run_tools ? run_tools.tokenize(",")*.toUpperCase() : default_run_tools def skip = skip_tools.tokenize(",")*.toUpperCase() - Channel.fromList(run.findAll { it !in skip }) + channel.fromList(run.findAll { it !in skip }) .flatten() .branch { toolname -> TEST_BIN_SCRIPT: toolname == "TEST_BIN_SCRIPT" @@ -517,27 +608,12 @@ workflow NF_CANARY { } .set { run_ch } - Channel + channel .of("alpha", "beta", "gamma") .collectFile(name: 'sample.txt', newLine: true) .set { test_file } - remote_file = params.remoteFile ? Channel.fromPath(params.remoteFile, glob:false) : Channel.empty() - - // Parse bucket parameters into lists - def rw_buckets_list = (params.fusion_read_write_buckets ? params.fusion_read_write_buckets.tokenize(',').collect { it.trim() } : []) + [workflow.workDir.toUriString()] - def ro_buckets_list = params.fusion_read_only_buckets ? params.fusion_read_only_buckets.tokenize(',').collect { it.trim() } : [] - - // Build fusion-doctor reference profile YAML from fusion parameters - def yaml_lines = [] - if (params.fusion_kernel_version_min) yaml_lines.add("kernel_version_min: \"${params.fusion_kernel_version_min}\"") - if (params.fusion_memory_gb_min) yaml_lines.add("memory_gb_min: ${params.fusion_memory_gb_min}") - if (params.fusion_disk_gb_min) yaml_lines.add("disk_gb_min: ${params.fusion_disk_gb_min}") - if (params.fusion_nvme_required != null) yaml_lines.add("nvme_required: ${params.fusion_nvme_required}") - if (params.fusion_cpu_cores_min) yaml_lines.add("cpu_cores_min: ${params.fusion_cpu_cores_min}") - if (params.fusion_open_files_min) yaml_lines.add("open_files_min: ${params.fusion_open_files_min}") - reference_profile_ch = channel.of(yaml_lines.join('\n')) - .collectFile(name: 'fusion-reference-profile.yaml', newLine: true) + remote_file = params.remoteFile ? channel.fromPath(params.remoteFile, glob:false) : channel.empty() // Run tests TEST_SUCCESS( run_ch.TEST_SUCCESS ) @@ -557,17 +633,27 @@ workflow NF_CANARY { TEST_VAL_INPUT( run_ch.TEST_VAL_INPUT, "Hello World" ) TEST_GPU( run_ch.TEST_GPU, "dummy" ) - TEST_FUSION_DOCTOR(run_ch.TEST_FUSION_DOCTOR, reference_profile_ch, rw_buckets_list, ro_buckets_list, params.fusion_cache_path) + FUSION_DOCTOR( + run_ch.TEST_FUSION_DOCTOR, + fusion_kernel_version_min, + fusion_memory_gb_min, + fusion_disk_gb_min, + fusion_nvme_required, + fusion_cpu_cores_min, + fusion_open_files_min, + fusion_cache_path, + fusion_read_write_buckets, + fusion_read_only_buckets + ) + + TEST_FUSION_DOCTOR(FUSION_DOCTOR.out.inputs) - // Generate consolidated fusion report from doctor output - // Only run FUSION_DOCTOR_GENERATE_REPORT if TEST_FUSION_DOCTOR produced output FUSION_DOCTOR_GENERATE_REPORT( - TEST_FUSION_DOCTOR.out.report, + TEST_FUSION_DOCTOR.out.report.collect(), file("${projectDir}/assets/templates/fusion_report_template.html") ) - // POC of emitting the channel - Channel.empty() + channel.empty() .mix( TEST_SUCCESS.out, TEST_CREATE_FILE.out, @@ -585,7 +671,7 @@ workflow NF_CANARY { TEST_MV_FOLDER_CONTENTS.out, TEST_VAL_INPUT.out, TEST_GPU.out, - TEST_FUSION_DOCTOR.out, + TEST_FUSION_DOCTOR.out.report, FUSION_DOCTOR_GENERATE_REPORT.out.html_report.ifEmpty([]), FUSION_DOCTOR_GENERATE_REPORT.out.json_report.ifEmpty([]) ) @@ -596,5 +682,19 @@ workflow NF_CANARY { } workflow { - NF_CANARY(params.run, params.skip, params.gpu, params.fusion) + NF_CANARY( + params.run, + params.skip, + params.gpu, + params.fusion, + params.fusion_kernel_version_min, + params.fusion_memory_gb_min, + params.fusion_disk_gb_min, + params.fusion_nvme_required, + params.fusion_cpu_cores_min, + params.fusion_open_files_min, + params.fusion_cache_path, + params.fusion_read_write_buckets, + params.fusion_read_only_buckets + ) } diff --git a/nextflow.config b/nextflow.config index 09c1b1b..2bcf412 100644 --- a/nextflow.config +++ b/nextflow.config @@ -6,15 +6,21 @@ params { remoteFile = null container = "quay.io/biocontainers/ubuntu:24.04" fusion = false - fusion_kernel_version_min = null - fusion_memory_gb_min = null - fusion_disk_gb_min = null - fusion_nvme_required = null - fusion_cpu_cores_min = null - fusion_open_files_min = null - fusion_cache_path = '/tmp' - fusion_read_write_buckets = "" - fusion_read_only_buckets = "" + fusion_redact = false + + // Parameter sweep: each value may be a single value OR a comma-separated + // list of values. FUSION_DOCTOR will be executed once for every + // combination (Cartesian product) of all non-null values across these + // parameters. + fusion_kernel_version_min = "5.10" // e.g. "5.10" or "5.10,5.15,6.1" + fusion_memory_gb_min = "4,8,16" // e.g. 4 or "4,8,16" + fusion_disk_gb_min = "50,75,100,200,300,375,600,750,950" // e.g. 100 or "100,200,950" + fusion_nvme_required = "false,true" // e.g. false or "false,true" + fusion_cpu_cores_min = "2,4,16" // e.g. 2 or "2,4,16" + fusion_open_files_min = "65535,131072,1048576" // e.g. 65535 or "65535,131072,1048576" + fusion_cache_path = '/tmp' // e.g. "/tmp" or "/tmp,/scratch" + fusion_read_write_buckets = "" // comma-separated bucket URIs (treated as one value per run) + fusion_read_only_buckets = "" // comma-separated bucket URIs (treated as one value per run) } process { diff --git a/nextflow_schema.json b/nextflow_schema.json index bc63a80..e23fa67 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -13,13 +13,13 @@ "properties": { "run": { "type": "string", - "description": "Selectively run tests as comma delimited values", - "help_text": "Tests to run as comma delimited values. E.g. --skip TEST_SUCCESS,TEST_INPUT. Case insensitive. Note this excludes all other tests." + "description": "Selectively run tests as comma-delimited values.", + "help_text": "Tests to run as comma-delimited values. E.g. --run TEST_SUCCESS,TEST_INPUT. Case insensitive. Note this excludes all other tests." }, "skip": { "type": "string", - "description": "Tests to skip as comma delimited values.", - "help_text": "Tests to skip as comma delimited values. E.g. --skip TEST_SUCCESS,TEST_INPUT. Case insensitive." + "description": "Tests to skip as comma-delimited values.", + "help_text": "Tests to skip as comma-delimited values. E.g. --skip TEST_SUCCESS,TEST_INPUT. Case insensitive." }, "remoteFile": { "type": "string", @@ -29,7 +29,7 @@ }, "container": { "type": "string", - "description": "Container URI for nf-canary", + "description": "Container URI for nf-canary.", "help_text": "Specifies the container URI. By default, this is an Ubuntu container on quay.io with no usage limits.", "default": "quay.io/biocontainers/ubuntu:24.04" }, @@ -64,13 +64,55 @@ "fa_icon": "fas fa-mask", "default": false }, + "fusion_kernel_version_min": { + "type": "string", + "description": "Minimum Linux kernel version(s) required for Fusion.", + "help_text": "Comma-separated list of kernel version thresholds to sweep. FUSION_DOCTOR runs once per value. E.g. '5.10' for a single check, or '5.10,5.15,6.1' to sweep multiple thresholds. AWS Forge images ship 5.10+ (AL2) or 6.1+ (AL2023); GCP and Azure ship 5.15+.", + "default": "5.10,5.15", + "fa_icon": "fas fa-microchip" + }, + "fusion_memory_gb_min": { + "type": "string", + "description": "Minimum memory (GB) required for Fusion, as a comma-separated sweep.", + "help_text": "Comma-separated list of memory thresholds (in GB) to sweep. FUSION_DOCTOR runs once per value. E.g. '8' for a single check, or '4,8,16' to sweep multiple thresholds.", + "default": "4,8,16", + "fa_icon": "fas fa-memory" + }, + "fusion_disk_gb_min": { + "type": "string", + "description": "Minimum local disk size (GB) required for Fusion, as a comma-separated sweep.", + "help_text": "Comma-separated list of disk size thresholds (in GB) to sweep. FUSION_DOCTOR runs once per value. E.g. '200' for a single check, or '100,200,950' to sweep multiple thresholds. Seqera docs require at least 200 GB local temp storage; 400 GB+ for files >100 GB. Cloud-specific profiles override this with values appropriate to that cloud's disk increments.", + "default": "50,75,100,200,300,375,600,750,950", + "fa_icon": "fas fa-hdd" + }, + "fusion_nvme_required": { + "type": "string", + "description": "Whether NVMe local storage is required, as a comma-separated sweep.", + "help_text": "Comma-separated list of boolean values to sweep ('true', 'false', or 'false,true'). FUSION_DOCTOR runs once per value. Fusion performs best on NVMe-backed instance storage. Set to 'true' when using NVMe instance families (e.g. AWS m6id, GCP local SSD, Azure 'd'-suffix VMs).", + "default": "false,true", + "fa_icon": "fas fa-bolt" + }, + "fusion_cpu_cores_min": { + "type": "string", + "description": "Minimum CPU cores required for Fusion, as a comma-separated sweep.", + "help_text": "Comma-separated list of CPU core thresholds to sweep. FUSION_DOCTOR runs once per value. E.g. '4' for a single check, or '2,4,16' to sweep multiple thresholds. Seqera recommends 16 vCPUs for large production pipelines.", + "default": "2,4,16", + "fa_icon": "fas fa-server" + }, + "fusion_open_files_min": { + "type": "string", + "description": "Minimum open file descriptor limit required for Fusion, as a comma-separated sweep.", + "help_text": "Comma-separated list of open file descriptor thresholds to sweep. FUSION_DOCTOR runs once per value. Fusion opens many file handles for FUSE operations and cloud storage connections. E.g. '65535' (Linux default), '131072' (recommended), or '65535,131072,1048576' to sweep all tiers.", + "default": "65535,131072,1048576", + "fa_icon": "fas fa-file" + }, "fusion_cache_path": { "type": "string", - "description": "Filesystem path for Fusion cache directory.", - "help_text": "Specifies the directory where Fusion should cache data locally. Used by the diagnostics process to verify sufficient disk space is available.", + "description": "Filesystem path(s) for Fusion cache directory, as a comma-separated sweep.", + "help_text": "Comma-separated list of cache directory paths to sweep. FUSION_DOCTOR runs once per value. Used to verify sufficient disk space is available at that location. E.g. '/tmp' or '/tmp,/scratch'.", "default": "/tmp", - "pattern": "^/.*$", - "fa_icon": "fas fa-hdd", + "pattern": "^/[^,]*(,/[^,]*)*$", + "fa_icon": "fas fa-folder", "hidden": true }, "fusion_read_only_buckets": { diff --git a/tests/main.fusion_doctor_workflow.nf.test b/tests/main.fusion_doctor_workflow.nf.test new file mode 100644 index 0000000..2075a38 --- /dev/null +++ b/tests/main.fusion_doctor_workflow.nf.test @@ -0,0 +1,385 @@ +nextflow_workflow { + + name "Test workflow FUSION_DOCTOR (channel preparation)" + script "../main.nf" + workflow "FUSION_DOCTOR" + + test("Emits one item for single values across all sweep params") { + + when { + workflow { + """ + input[0] = channel.of('trigger') // trigger_ch + input[1] = "5.10" // kernel_version_min + input[2] = "8" // memory_gb_min + input[3] = "200" // disk_gb_min + input[4] = "true" // nvme_required + input[5] = "4" // cpu_cores_min + input[6] = "131072" // open_files_min + input[7] = "/tmp" // cache_path + input[8] = "" // read_write_buckets + input[9] = "" // read_only_buckets + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert workflow.out.inputs.size() == 1 } + ) + } + + } + + test("Cartesian product: 2 kernels × 2 memory values = 4 combinations") { + + when { + workflow { + """ + input[0] = channel.of('trigger') + input[1] = "5.10,5.15" + input[2] = "4,8" + input[3] = "200" + input[4] = "true" + input[5] = "4" + input[6] = "131072" + input[7] = "/tmp" + input[8] = "" + input[9] = "" + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert workflow.out.inputs.size() == 4 } + ) + } + + } + + test("Cartesian product: 2 kernels × 3 memory × 2 nvme = 12 combinations") { + + when { + workflow { + """ + input[0] = channel.of('trigger') + input[1] = "5.10,5.15" + input[2] = "4,8,16" + input[3] = "200" + input[4] = "false,true" + input[5] = "4" + input[6] = "131072" + input[7] = "/tmp" + input[8] = "" + input[9] = "" + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert workflow.out.inputs.size() == 12 } + ) + } + + } + + test("meta snapshot and YAML content for single combination") { + + when { + workflow { + """ + input[0] = channel.of('trigger') + input[1] = "5.10" + input[2] = "8" + input[3] = "200" + input[4] = "true" + input[5] = "4" + input[6] = "131072" + input[7] = "/tmp" + input[8] = "" + input[9] = "" + """ + } + } + + then { + assertAll( + { assert workflow.success }, + // meta map + { assert snapshot(workflow.out.inputs[0][1]).match("meta") }, + // YAML file — read content directly; path() wrapping avoids collectFile hang + { assert snapshot(path(workflow.out.inputs[0][2]).text).match("yaml_text") }, + { assert path(workflow.out.inputs[0][2]).name == "fusion-reference-profile-k5_10_mem8_disk200_nvmetrue_cpu4_of131072.yaml" } + ) + } + + } + + test("run_id omits cache suffix when cache_path is /tmp (default)") { + + when { + workflow { + """ + input[0] = channel.of('trigger') + input[1] = "5.10" + input[2] = "8" + input[3] = "200" + input[4] = "true" + input[5] = "4" + input[6] = "131072" + input[7] = "/tmp" + input[8] = "" + input[9] = "" + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert !workflow.out.inputs[0][1].run_id.contains("cache") } + ) + } + + } + + test("run_id includes cache suffix when cache_path is non-default") { + + when { + workflow { + """ + input[0] = channel.of('trigger') + input[1] = "5.10" + input[2] = "8" + input[3] = "200" + input[4] = "true" + input[5] = "4" + input[6] = "131072" + input[7] = "/scratch" + input[8] = "" + input[9] = "" + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot(workflow.out.inputs[0][1]).match("meta_scratch") } + ) + } + + } + + test("cache_path sweep produces one item per path") { + + when { + workflow { + """ + input[0] = channel.of('trigger') + input[1] = "5.10" + input[2] = "8" + input[3] = "200" + input[4] = "true" + input[5] = "4" + input[6] = "131072" + input[7] = "/tmp,/scratch" + input[8] = "" + input[9] = "" + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert workflow.out.inputs.size() == 2 }, + { assert snapshot( + workflow.out.inputs.collect { it[1] }.sort { a, b -> a.run_id <=> b.run_id } + ).match("cache_sweep_metas") } + ) + } + + } + + test("workDir is always appended to rw_buckets when none are supplied") { + + when { + workflow { + """ + input[0] = channel.of('trigger') + input[1] = "5.10" + input[2] = "8" + input[3] = "200" + input[4] = "true" + input[5] = "4" + input[6] = "131072" + input[7] = "/tmp" + input[8] = "" + input[9] = "" + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert workflow.out.inputs[0][3].size() == 1 }, + { assert workflow.out.inputs[0][3][0].startsWith("file://") } + ) + } + + } + + test("explicit rw_buckets are prepended before workDir") { + + when { + workflow { + """ + input[0] = channel.of('trigger') + input[1] = "5.10" + input[2] = "8" + input[3] = "200" + input[4] = "true" + input[5] = "4" + input[6] = "131072" + input[7] = "/tmp" + input[8] = "s3://my-results,s3://my-cache" + input[9] = "" + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert workflow.out.inputs[0][3].size() == 3 }, + // snapshot the two stable explicit entries; workDir varies by run + { assert snapshot(workflow.out.inputs[0][3][0..1]).match("rw_buckets_explicit") }, + { assert workflow.out.inputs[0][3][2].startsWith("file://") } + ) + } + + } + + test("ro_buckets are passed through correctly") { + + when { + workflow { + """ + input[0] = channel.of('trigger') + input[1] = "5.10" + input[2] = "8" + input[3] = "200" + input[4] = "true" + input[5] = "4" + input[6] = "131072" + input[7] = "/tmp" + input[8] = "" + input[9] = "s3://reference-data,gs://shared-files" + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot(workflow.out.inputs[0][4]).match("ro_buckets") } + ) + } + + } + + test("empty ro_buckets produces an empty list") { + + when { + workflow { + """ + input[0] = channel.of('trigger') + input[1] = "5.10" + input[2] = "8" + input[3] = "200" + input[4] = "true" + input[5] = "4" + input[6] = "131072" + input[7] = "/tmp" + input[8] = "" + input[9] = "" + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot(workflow.out.inputs[0][4]).match("ro_buckets_empty") } + ) + } + + } + + test("meta.cache_path reflects the cache_path input") { + + when { + workflow { + """ + input[0] = channel.of('trigger') + input[1] = "5.10" + input[2] = "8" + input[3] = "200" + input[4] = "true" + input[5] = "4" + input[6] = "131072" + input[7] = "/scratch" + input[8] = "" + input[9] = "" + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert workflow.out.inputs[0][1].cache_path == "/scratch" } + ) + } + + } + + test("whitespace in comma-separated params is trimmed") { + + when { + workflow { + """ + input[0] = channel.of('trigger') + input[1] = " 5.10 , 5.15 " + input[2] = "4 , 8" + input[3] = "200" + input[4] = "true" + input[5] = "4" + input[6] = "131072" + input[7] = "/tmp" + input[8] = "" + input[9] = "" + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert workflow.out.inputs.size() == 4 }, + { assert snapshot( + workflow.out.inputs.collect { it[1] }.sort { a, b -> a.run_id <=> b.run_id } + ).match("whitespace_trim_metas") } + ) + } + + } + +} diff --git a/tests/main.nf.test b/tests/main.nf.test index 1d549ba..964154f 100644 --- a/tests/main.nf.test +++ b/tests/main.nf.test @@ -15,16 +15,10 @@ nextflow_pipeline{ then { assertAll( { assert workflow.success }, - { assert workflow.trace.tasks().size() == 17 }, - { assert workflow.trace.succeeded().size() == 16 }, + { assert workflow.trace.tasks().size() == 502 }, + { assert workflow.trace.succeeded().size() == 501 }, { assert workflow.trace.failed().size() == 1 }, - // Check fusion reports exist but don't snapshot their content (contains workDir paths) - { assert path(params.outdir).resolve("fusion-doctor-report.json").exists() }, - { assert path(params.outdir).resolve("fusion/fusion-report.json").exists() }, - { assert path(params.outdir).resolve("fusion/fusion-report.html").exists() }, - { assert snapshot(workflow, path(params.outdir).list().findAll { - !it.toString().contains("fusion") - }).match() } + { assert snapshot(workflow.out).match() } ) } @@ -43,22 +37,15 @@ nextflow_pipeline{ then { assertAll( { assert workflow.success }, - { assert workflow.trace.tasks().size() == 16 }, - { assert workflow.trace.succeeded().size() == 15 }, + { assert workflow.trace.tasks().size() == 501 }, + { assert workflow.trace.succeeded().size() == 500 }, { assert workflow.trace.failed().size() == 1 }, - // Check fusion reports exist but don't snapshot their content (contains workDir paths) - { assert path(params.outdir).resolve("fusion-doctor-report.json").exists() }, - { assert path(params.outdir).resolve("fusion/fusion-report.json").exists() }, - { assert path(params.outdir).resolve("fusion/fusion-report.html").exists() }, - { assert snapshot(workflow, path(params.outdir).list().findAll { - !it.toString().contains("fusion") - }).match() } + { assert snapshot(workflow.out).match() } ) } } - test("Should only run one process") { when { @@ -75,7 +62,7 @@ nextflow_pipeline{ { assert workflow.trace.tasks().size() == 1 }, { assert workflow.trace.succeeded().size() == 1 }, { assert workflow.trace.failed().size() == 0 }, - { assert snapshot(workflow, path(params.outdir).list()).match() } + { assert snapshot(workflow.out).match() } ) } @@ -93,16 +80,10 @@ nextflow_pipeline{ then { assertAll( { assert workflow.success }, - { assert workflow.trace.tasks().size() == 17 }, - { assert workflow.trace.succeeded().size() == 16 }, + { assert workflow.trace.tasks().size() == 502 }, + { assert workflow.trace.succeeded().size() == 501 }, { assert workflow.trace.failed().size() == 1 }, - // Check fusion reports exist but don't snapshot their content (contains workDir paths) - { assert path("${launchDir}/output").resolve("fusion-doctor-report.json").exists() }, - { assert path("${launchDir}/output").resolve("fusion/fusion-report.json").exists() }, - { assert path("${launchDir}/output").resolve("fusion/fusion-report.html").exists() }, - { assert snapshot(workflow, path("${launchDir}/output").list().findAll { - !it.toString().contains("fusion") - }).match() } + { assert snapshot(workflow.out).match() } ) } From 1e4a587a38f2f642a837674d3bb6bf4d3c54218b Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Tue, 28 Apr 2026 19:46:16 +0100 Subject: [PATCH 52/53] Apply suggestions from code review Co-authored-by: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index a71ea88..c317eab 100644 --- a/README.md +++ b/README.md @@ -201,7 +201,6 @@ nextflow run seqeralabs/nf-canary -profile fusion_aws_recommended In AWS Batch, the Seqera Platform UI allows selecting instance families (e.g. `m6id`) but not specific sizes. Small instances in an otherwise valid family may not meet the `recommended` disk threshold. For example, on AWS the `m6id` family NVMe ranges from 118 GB (`.large`) to 1,900 GB (`.8xlarge`), with only `.xlarge` and above meeting the 200 GB threshold. -If `fusion doctor` reports a disk requirement failure, request more CPUs/memory to get a larger instance, or use the `low` profile for small tasks. #### Custom Requirements From 996f57656435c133679dc5d0a70928f74c051a23 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 22 Jul 2026 13:09:11 +0000 Subject: [PATCH 53/53] fix: resolve merge conflicts with origin/main - Resolve nextflow.config: add gpu_container, rename fusion params to match updated fusion-doctor schema (memory_capacity_gb_min, disk_capacity_gb_min, vcpus_min), keep sweep defaults - Resolve main.nf: use fusion-doctor:1.0.0 container, add gpu_container to NF_CANARY workflow signature and call, add onComplete handler from main - Update FUSION_DOCTOR workflow YAML keys to new schema names - Update nextflow_schema.json with renamed param keys - Replace simplified fusion profiles with full tiered profiles from main - Update test comment labels for renamed parameters --- conf/fusion.config | 219 +++++++++++++++++++--- main.nf | 57 +++--- nextflow.config | 55 ++---- nextflow_schema.json | 14 +- tests/main.fusion_doctor_workflow.nf.test | 6 +- 5 files changed, 240 insertions(+), 111 deletions(-) diff --git a/conf/fusion.config b/conf/fusion.config index d060321..9dc38b1 100644 --- a/conf/fusion.config +++ b/conf/fusion.config @@ -1,51 +1,214 @@ /* * Fusion Validation Profiles * - * These profiles configure cloud-specific settings for running FUSION_DOCTOR. - * The parameter sweep values (kernel, memory, disk, CPU, open files) are - * defined as comma-separated defaults in nextflow.config and apply to all - * profiles automatically. + * These profiles define recommended thresholds for different cloud + * environments and workload sizes. Parameters are internally used to build + * a reference profile that is passed to `fusion-doctor`. * - * Usage: nextflow run seqeralabs/nf-canary -profile fusion_aws - * nextflow run seqeralabs/nf-canary -profile fusion_gcp - * nextflow run seqeralabs/nf-canary -profile fusion_azure + * Tiers: + * low — small Nextflow workloads + * recommended — typical pipeline workloads + * high — large-scale production workloads + * + * Usage: nextflow run seqeralabs/nf-canary -profile fusion_aws_recommended + * + * --- Threshold review notes (2026-03-04) --- * * Sources: + * - https://docs.seqera.io/platform-cloud/compute-envs/aws-batch * - https://docs.seqera.io/fusion/guide/aws-batch * - https://docs.seqera.io/fusion/guide/gcp-batch * - https://docs.seqera.io/platform-cloud/compute-envs/google-cloud-batch * - https://docs.seqera.io/platform-cloud/compute-envs/azure-batch + * + * Seqera docs requirements for Fusion (all clouds): + * - Local temp storage: at least 200 GB, random read speed 1000 MBps+ + * - For files >100 GB: 400 GB+ temp storage + * + * kernel_version_min: + * Minimum kernel shipped by Seqera Forge images: + * AWS - ECS-Optimized AL2023: 6.1 | legacy AL2: 5.10 (EOL June 2026) + * GCP - Ubuntu 22.04 LTS: 5.15+ | Ubuntu 24.04 LTS: 6.8+ + * Azure - Ubuntu HPC 22.04: 5.15 | legacy Ubuntu 20.04: 5.4 + * + * Decision: Use cloud-specific minimums: + * - AWS: 5.10 (covers legacy AL2 until EOL June 2026) + * - GCP/Azure: 5.15 (current-gen Forge images) + * + * memory_capacity_gb_min: + * No Fusion-specific minimum documented. Thresholds reflect workload size. + * Tiers: 4 GB (low), 8 GB (recommended), 16 GB (high). + * + * disk_capacity_gb_min: + * Thresholds match Seqera's documented minimums for Fusion. Note that + * Platform's UI only allows selecting instance families, not sizes. Small + * instances in valid families (e.g. m6id.large = 118 GB) may not meet the + * recommended threshold — this is expected and signals the instance is + * undersized for Fusion despite being in the right family. + * + * AWS: + * Platform auto-selects NVMe-based instance families when Fusion is + * enabled (e.g. m6id, c6id, r6id). 8xlarge+ recommended for production. + * - Without NVMe: EBS bumped to 100 GB (gp3, 325 MB/s) + * - With NVMe: starts at 118 GB (.large), 1900 GB (.8xlarge) + * Tiers: + * - 100 GB (low — EBS gp3 when Fusion enabled without NVMe) + * - 200 GB (recommended — Seqera docs minimum) + * - 950 GB (high — .4xlarge NVMe, 16 vCPUs, large datasets) + * + * GCP: + * Platform auto-selects families that support local SSDs (e.g. n2, c2, + * n2d). A 375 GB local NVMe SSD is provisioned per job. + * - Persistent disk: min 10 GiB (no local SSD) + * - Local NVMe SSD: 375 GiB increments (attached at creation time) + * - Production: n2-highmem-16 with local SSD, or larger + * Tiers: + * - 50 GB (low — persistent disk, small workloads) + * - 375 GB (recommended — 1x local NVMe SSD) + * - 750 GB (high — 2x local NVMe SSD) + * + * Azure: + * No auto-selection — user must pick VM size. Seqera recommends + * E-series with 'd' suffix (e.g. Standard_E8d_v5, Standard_E16d_v5). + * Standard SSDs only, no network-attached storage. + * - 'd' suffix VMs have local temp SSD (~37.5 GiB per vCPU) + * - Production: Standard_E16d_v5 or larger + * Tiers: + * - 75 GB (low — 2-vCPU 'd' VM, small workloads) + * - 300 GB (recommended — 8-vCPU 'd' VM, e.g. Standard_E8d_v5) + * - 600 GB (high — 16-vCPU 'd' VM, e.g. Standard_E16d_v5) + * + * nvme_required: + * Whether NVMe local storage is required. Fusion performs best with + * NVMe-backed instance storage for temp/scratch. Set to true for + * recommended/high tiers where NVMe families are expected, false for + * low tiers that may use EBS/persistent disk. + * + * vcpus_min: + * Minimum vCPUs. Seqera internal benchmarking recommends instances + * with 16 vCPUs for large, long-lived production pipelines. + * Tiers: 2 (low), 4 (recommended), 16 (high — matches docs). + * + * open_files_min: + * Minimum open file descriptor soft limit. Fusion opens many file handles + * for FUSE operations and cloud storage connections. 65535 is the common + * Linux default; production workloads benefit from higher limits. + * Tiers: 65535 (low), 131072 (recommended), 1048576 (high — matches docs). */ profiles { - // ---- AWS ---- - // Platform auto-selects NVMe-backed instance families (e.g. m6id, c6id, r6id) - // when Fusion is enabled — no machineType override needed. + // ---- AWS profiles ---- - fusion_aws { - params.fusion = true + fusion_aws_low { + // Small workloads — EBS-only, no NVMe + // 100 GB = EBS size Platform sets when Fusion is enabled without NVMe + params.fusion = true + params.fusion_kernel_version_min = "5.10" + params.fusion_memory_capacity_gb_min = 4 + params.fusion_disk_capacity_gb_min = 100 + params.fusion_nvme_required = false + params.fusion_vcpus_min = 2 + params.fusion_open_files_min = 65535 } - // ---- Google Cloud ---- - // Local NVMe SSDs are provisioned in 375 GB increments. - // Families supporting local SSDs: n2, c2, n2d (auto-selected by Platform). - // Recommended instance: n2-standard-8 or larger with local SSD attached. + fusion_aws_recommended { + // Seqera docs: NVMe-based families (e.g. m6id, c6id, r6id), 8xlarge+ for production + // 200 GB = Seqera docs minimum for Fusion local temp storage + params.fusion = true + params.fusion_kernel_version_min = "5.10" + params.fusion_memory_capacity_gb_min = 8 + params.fusion_disk_capacity_gb_min = 200 + params.fusion_nvme_required = true + params.fusion_vcpus_min = 4 + params.fusion_open_files_min = 131072 + } + + fusion_aws_high { + // Large-scale production — NVMe families, .4xlarge+ instances (16 vCPUs) + // 950 GB = NVMe on .4xlarge; Seqera docs: 400 GB+ for files >100 GB + params.fusion = true + params.fusion_kernel_version_min = "5.10" + params.fusion_memory_capacity_gb_min = 16 + params.fusion_disk_capacity_gb_min = 950 + params.fusion_nvme_required = true + params.fusion_vcpus_min = 16 + params.fusion_open_files_min = 1048576 + } + + // ---- Google Cloud profiles ---- + + fusion_google_low { + // Small workloads — persistent disk, no local SSD + // 50 GB = modest persistent disk for small workloads + params.fusion = true + params.fusion_kernel_version_min = "5.15" + params.fusion_memory_capacity_gb_min = 4 + params.fusion_disk_capacity_gb_min = 50 + params.fusion_nvme_required = false + params.fusion_vcpus_min = 2 + params.fusion_open_files_min = 65535 + } - fusion_gcp { - params.fusion = true - process.machineType = 'n2-standard-8' - params.fusion_disk_gb_min = '375,750' // 1x or 2x local NVMe SSD + fusion_google_recommended { + // Seqera docs: families supporting local SSDs (e.g. n2, c2, n2d) + // 375 GB = 1x local NVMe SSD (GCP provisions per job) + params.fusion = true + params.fusion_kernel_version_min = "5.15" + params.fusion_memory_capacity_gb_min = 8 + params.fusion_disk_capacity_gb_min = 375 + params.fusion_nvme_required = true + params.fusion_vcpus_min = 4 + params.fusion_open_files_min = 131072 } - // ---- Azure ---- - // User must select a VM with a 'd' suffix for local temp SSD. - // Seqera recommends E-series 'd' VMs (e.g. Standard_E8d_v5, Standard_E16d_v5). - // Local disk size is ~37.5 GiB per vCPU on these VM families. + fusion_google_high { + // Large-scale production — 2x local NVMe SSDs + // 750 GB = 2 x 375 GB; Seqera docs: 400 GB+ for files >100 GB + params.fusion = true + params.fusion_kernel_version_min = "5.15" + params.fusion_memory_capacity_gb_min = 16 + params.fusion_disk_capacity_gb_min = 750 + params.fusion_nvme_required = true + params.fusion_vcpus_min = 16 + params.fusion_open_files_min = 1048576 + } + + // ---- Azure profiles ---- + + fusion_azure_low { + // Small workloads — smallest 'd' suffix VM (2 vCPU) + // 75 GB = local temp disk on 2-vCPU 'd' VM + params.fusion = true + params.fusion_kernel_version_min = "5.15" + params.fusion_memory_capacity_gb_min = 4 + params.fusion_disk_capacity_gb_min = 75 + params.fusion_nvme_required = false + params.fusion_vcpus_min = 2 + params.fusion_open_files_min = 65535 + } + + fusion_azure_recommended { + // Seqera docs: E-series with 'd' suffix (e.g. Standard_E8d_v5, Standard_E16d_v5) + // 300 GB = 8-vCPU 'd' VM temp disk + params.fusion = true + params.fusion_kernel_version_min = "5.15" + params.fusion_memory_capacity_gb_min = 8 + params.fusion_disk_capacity_gb_min = 300 + params.fusion_nvme_required = true + params.fusion_vcpus_min = 4 + params.fusion_open_files_min = 131072 + } - fusion_azure { - params.fusion = true - process.machineType = 'Standard_E8d_v5' - params.fusion_disk_gb_min = '300,600' // 8-vCPU or 16-vCPU 'd' VM + fusion_azure_high { + // Large-scale production — 16-vCPU 'd' VM (e.g. Standard_E16d_v5) + // 600 GB = Standard_E16d_v5 temp disk (614 GiB) + params.fusion = true + params.fusion_kernel_version_min = "5.15" + params.fusion_memory_capacity_gb_min = 16 + params.fusion_disk_capacity_gb_min = 600 + params.fusion_nvme_required = true + params.fusion_vcpus_min = 16 + params.fusion_open_files_min = 1048576 } } diff --git a/main.nf b/main.nf index 15c49bb..2162e08 100644 --- a/main.nf +++ b/main.nf @@ -409,14 +409,9 @@ process TEST_FUSION_DOCTOR { in the workflow), avoiding any shell quoting or indentation issues. */ -<<<<<<< HEAD - container 'cr.seqera.io/public/fusion/doctor:1.0.0-dev-260420150843' + container 'cr.seqera.io/public/fusion/doctor:1.0.0' tag { meta.run_id } publishDir { (params.outdir ? file(params.outdir) : file(workflow.workDir).resolve("outputs/fusion")).toUriString() }, mode: 'copy' -======= - container 'cr.seqera.io/public/fusion/doctor:1.0.0' - publishDir { params.outdir ?: file(workflow.workDir).resolve("outputs/fusion").toUriString() }, mode: 'copy' ->>>>>>> origin/main input: tuple val(dummy_val), val(meta), path(reference_profile), val(rw_buckets), val(ro_buckets) @@ -507,10 +502,10 @@ workflow FUSION_DOCTOR { take: trigger_ch // val channel — one item fires the whole sweep kernel_version_min // e.g. "5.10,5.15" - memory_gb_min // e.g. "4,8,16" - disk_gb_min // e.g. "100,200,950" + memory_capacity_gb_min // e.g. "4,8,16" + disk_capacity_gb_min // e.g. "100,200,950" nvme_required // e.g. "false,true" - cpu_cores_min // e.g. "2,4,16" + vcpus_min // e.g. "2,4,16" open_files_min // e.g. "65535,131072,1048576" cache_path // e.g. "/tmp" read_write_buckets // comma-separated bucket URIs @@ -521,10 +516,10 @@ workflow FUSION_DOCTOR { def ro_buckets_list = sweepList(read_only_buckets) def kernel_sweep = sweepList(kernel_version_min) - def memory_sweep = sweepList(memory_gb_min) - def disk_sweep = sweepList(disk_gb_min) + def memory_sweep = sweepList(memory_capacity_gb_min) + def disk_sweep = sweepList(disk_capacity_gb_min) def nvme_sweep = sweepList(nvme_required) - def cpu_sweep = sweepList(cpu_cores_min) + def cpu_sweep = sweepList(vcpus_min) def openf_sweep = sweepList(open_files_min) def cache_sweep = sweepList(cache_path ?: '/tmp') @@ -548,10 +543,10 @@ workflow FUSION_DOCTOR { def yaml_lines = [] if (kernel) yaml_lines << "kernel_version_min: \"${kernel}\"" - if (memory) yaml_lines << "memory_gb_min: ${memory}" - if (disk) yaml_lines << "disk_gb_min: ${disk}" + if (memory) yaml_lines << "memory_capacity_gb_min: ${memory}" + if (disk) yaml_lines << "disk_capacity_gb_min: ${disk}" if (nvme) yaml_lines << "nvme_required: ${nvme}" - if (cpu) yaml_lines << "cpu_cores_min: ${cpu}" + if (cpu) yaml_lines << "vcpus_min: ${cpu}" if (openf) yaml_lines << "open_files_min: ${openf}" def run_id = parts ? parts.join('_') : 'default' @@ -577,27 +572,20 @@ workflow FUSION_DOCTOR { workflow NF_CANARY { take: -<<<<<<< HEAD run_tools skip_tools gpu fusion + gpu_container fusion_kernel_version_min - fusion_memory_gb_min - fusion_disk_gb_min + fusion_memory_capacity_gb_min + fusion_disk_capacity_gb_min fusion_nvme_required - fusion_cpu_cores_min + fusion_vcpus_min fusion_open_files_min fusion_cache_path fusion_read_write_buckets fusion_read_only_buckets -======= - run_tools - skip_tools - gpu - fusion - gpu_container ->>>>>>> origin/main main: def default_run_tools = [ @@ -683,10 +671,10 @@ workflow NF_CANARY { FUSION_DOCTOR( run_ch.TEST_FUSION_DOCTOR, fusion_kernel_version_min, - fusion_memory_gb_min, - fusion_disk_gb_min, + fusion_memory_capacity_gb_min, + fusion_disk_capacity_gb_min, fusion_nvme_required, - fusion_cpu_cores_min, + fusion_vcpus_min, fusion_open_files_min, fusion_cache_path, fusion_read_write_buckets, @@ -729,24 +717,22 @@ workflow NF_CANARY { } workflow { -<<<<<<< HEAD NF_CANARY( params.run, params.skip, params.gpu, params.fusion, + params.gpu_container, params.fusion_kernel_version_min, - params.fusion_memory_gb_min, - params.fusion_disk_gb_min, + params.fusion_memory_capacity_gb_min, + params.fusion_disk_capacity_gb_min, params.fusion_nvme_required, - params.fusion_cpu_cores_min, + params.fusion_vcpus_min, params.fusion_open_files_min, params.fusion_cache_path, params.fusion_read_write_buckets, params.fusion_read_only_buckets ) -======= - NF_CANARY(params.run, params.skip, params.gpu, params.fusion, params.gpu_container) workflow.onComplete = { if (workflow.success) { @@ -770,5 +756,4 @@ workflow { ) } } ->>>>>>> origin/main } diff --git a/nextflow.config b/nextflow.config index 8e28540..4bec74e 100644 --- a/nextflow.config +++ b/nextflow.config @@ -3,48 +3,29 @@ manifest { } params { -<<<<<<< HEAD - skip = '' - gpu = false - run = '' - outdir = null - remoteFile = null - container = "quay.io/biocontainers/ubuntu:24.04" - fusion = false - fusion_redact = false + skip = '' + gpu = false + run = '' + outdir = null + remoteFile = null + container = "quay.io/biocontainers/ubuntu:24.04" + gpu_container = "pytorch/pytorch:latest" + fusion = false + fusion_redact = false // Parameter sweep: each value may be a single value OR a comma-separated // list of values. FUSION_DOCTOR will be executed once for every // combination (Cartesian product) of all non-null values across these // parameters. - fusion_kernel_version_min = "5.10" // e.g. "5.10" or "5.10,5.15,6.1" - fusion_memory_gb_min = "4,8,16" // e.g. 4 or "4,8,16" - fusion_disk_gb_min = "50,75,100,200,300,375,600,750,950" // e.g. 100 or "100,200,950" - fusion_nvme_required = "false,true" // e.g. false or "false,true" - fusion_cpu_cores_min = "2,4,16" // e.g. 2 or "2,4,16" - fusion_open_files_min = "65535,131072,1048576" // e.g. 65535 or "65535,131072,1048576" - fusion_cache_path = '/tmp' // e.g. "/tmp" or "/tmp,/scratch" - fusion_read_write_buckets = "" // comma-separated bucket URIs (treated as one value per run) - fusion_read_only_buckets = "" // comma-separated bucket URIs (treated as one value per run) -======= - skip = '' - gpu = false - run = '' - outdir = null - remoteFile = null - container = "quay.io/biocontainers/ubuntu:24.04" - gpu_container = "pytorch/pytorch:latest" - fusion = false - fusion_kernel_version_min = null - fusion_memory_capacity_gb_min = null - fusion_disk_capacity_gb_min = null - fusion_nvme_required = null - fusion_vcpus_min = null - fusion_open_files_min = null - fusion_cache_path = '/tmp' - fusion_read_write_buckets = "" - fusion_read_only_buckets = "" ->>>>>>> origin/main + fusion_kernel_version_min = "5.10" // e.g. "5.10" or "5.10,5.15,6.1" + fusion_memory_capacity_gb_min = "4,8,16" // e.g. 4 or "4,8,16" + fusion_disk_capacity_gb_min = "50,75,100,200,300,375,600,750,950" // e.g. 100 or "100,200,950" + fusion_nvme_required = "false,true" // e.g. false or "false,true" + fusion_vcpus_min = "2,4,16" // e.g. 2 or "2,4,16" + fusion_open_files_min = "65535,131072,1048576" // e.g. 65535 or "65535,131072,1048576" + fusion_cache_path = '/tmp' // e.g. "/tmp" or "/tmp,/scratch" + fusion_read_write_buckets = "" // comma-separated bucket URIs (treated as one value per run) + fusion_read_only_buckets = "" // comma-separated bucket URIs (treated as one value per run) } process { diff --git a/nextflow_schema.json b/nextflow_schema.json index 5cf7461..1054a3b 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -77,16 +77,16 @@ "default": "5.10,5.15", "fa_icon": "fas fa-microchip" }, - "fusion_memory_gb_min": { + "fusion_memory_capacity_gb_min": { "type": "string", - "description": "Minimum memory (GB) required for Fusion, as a comma-separated sweep.", + "description": "Minimum memory capacity (GB) required for Fusion, as a comma-separated sweep.", "help_text": "Comma-separated list of memory thresholds (in GB) to sweep. FUSION_DOCTOR runs once per value. E.g. '8' for a single check, or '4,8,16' to sweep multiple thresholds.", "default": "4,8,16", "fa_icon": "fas fa-memory" }, - "fusion_disk_gb_min": { + "fusion_disk_capacity_gb_min": { "type": "string", - "description": "Minimum local disk size (GB) required for Fusion, as a comma-separated sweep.", + "description": "Minimum local disk capacity (GB) required for Fusion, as a comma-separated sweep.", "help_text": "Comma-separated list of disk size thresholds (in GB) to sweep. FUSION_DOCTOR runs once per value. E.g. '200' for a single check, or '100,200,950' to sweep multiple thresholds. Seqera docs require at least 200 GB local temp storage; 400 GB+ for files >100 GB. Cloud-specific profiles override this with values appropriate to that cloud's disk increments.", "default": "50,75,100,200,300,375,600,750,950", "fa_icon": "fas fa-hdd" @@ -98,10 +98,10 @@ "default": "false,true", "fa_icon": "fas fa-bolt" }, - "fusion_cpu_cores_min": { + "fusion_vcpus_min": { "type": "string", - "description": "Minimum CPU cores required for Fusion, as a comma-separated sweep.", - "help_text": "Comma-separated list of CPU core thresholds to sweep. FUSION_DOCTOR runs once per value. E.g. '4' for a single check, or '2,4,16' to sweep multiple thresholds. Seqera recommends 16 vCPUs for large production pipelines.", + "description": "Minimum vCPUs required for Fusion, as a comma-separated sweep.", + "help_text": "Comma-separated list of vCPU thresholds to sweep. FUSION_DOCTOR runs once per value. E.g. '4' for a single check, or '2,4,16' to sweep multiple thresholds. Seqera recommends 16 vCPUs for large production pipelines.", "default": "2,4,16", "fa_icon": "fas fa-server" }, diff --git a/tests/main.fusion_doctor_workflow.nf.test b/tests/main.fusion_doctor_workflow.nf.test index 2075a38..0602a57 100644 --- a/tests/main.fusion_doctor_workflow.nf.test +++ b/tests/main.fusion_doctor_workflow.nf.test @@ -11,10 +11,10 @@ nextflow_workflow { """ input[0] = channel.of('trigger') // trigger_ch input[1] = "5.10" // kernel_version_min - input[2] = "8" // memory_gb_min - input[3] = "200" // disk_gb_min + input[2] = "8" // memory_capacity_gb_min + input[3] = "200" // disk_capacity_gb_min input[4] = "true" // nvme_required - input[5] = "4" // cpu_cores_min + input[5] = "4" // vcpus_min input[6] = "131072" // open_files_min input[7] = "/tmp" // cache_path input[8] = "" // read_write_buckets