From a5019c328d0a6723013c507e7d0bc59317199328 Mon Sep 17 00:00:00 2001 From: Frank Scholter Peres Date: Wed, 20 May 2026 11:10:22 +0000 Subject: [PATCH 01/11] score_metamodel: autodiscover requirement types for metrics --- docs/conf.py | 4 + src/extensions/score_metamodel/__init__.py | 96 ++++-- ...st_traceability_metrics_json_generation.py | 300 +++++++++++++++++- 3 files changed, 379 insertions(+), 21 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 0255915c2..488b0c86b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -18,3 +18,7 @@ extensions = [ "score_sphinx_bundle", ] + +# Configure traceability metrics explicitly for this repository. +score_metamodel_requirement_types = "tool_req" +score_metamodel_include_external_needs = False diff --git a/src/extensions/score_metamodel/__init__.py b/src/extensions/score_metamodel/__init__.py index 557e7b974..f9fea3a9d 100644 --- a/src/extensions/score_metamodel/__init__.py +++ b/src/extensions/score_metamodel/__init__.py @@ -111,29 +111,36 @@ def _write_metrics_json(app: Sphinx, exception: Exception | None) -> None: return all_needs: list[Any] = list(SphinxNeedsData(app.env).get_needs_view().values()) - - raw = str(getattr(app.config, "score_metamodel_requirement_types", "tool_req")) - requirement_types = {t.strip() for t in raw.split(",") if t.strip()} or {"tool_req"} - include_not_implemented = True include_external: bool = bool( getattr(app.config, "score_metamodel_include_external_needs", False) ) + raw = str(getattr(app.config, "score_metamodel_requirement_types", "")).strip() + requirement_types = {t.strip() for t in raw.split(",") if t.strip()} + if not requirement_types: + requirement_types = _discover_requirement_types(app, all_needs, include_external) + include_not_implemented = True + metrics_by_type: dict[str, Any] = {} - for req_type in sorted(requirement_types): - type_summary = compute_traceability_summary( - all_needs=all_needs, - requirement_types={req_type}, - include_not_implemented=include_not_implemented, - filtered_test_types=set(), - include_external=include_external, + if not requirement_types: + logger.info( + "No requirement types configured or discovered; writing empty metrics.json." ) - metrics_by_type[req_type] = { - "include_not_implemented": type_summary["include_not_implemented"], - "include_external": type_summary["include_external"], - "requirements": type_summary["requirements"], - "tests": type_summary["tests"], - } + else: + for req_type in sorted(requirement_types): + type_summary = compute_traceability_summary( + all_needs=all_needs, + requirement_types={req_type}, + include_not_implemented=include_not_implemented, + filtered_test_types=set(), + include_external=include_external, + ) + metrics_by_type[req_type] = { + "include_not_implemented": type_summary["include_not_implemented"], + "include_external": type_summary["include_external"], + "requirements": type_summary["requirements"], + "tests": type_summary["tests"], + } output: dict[str, Any] = { "schema_version": "1", @@ -147,6 +154,56 @@ def _write_metrics_json(app: Sphinx, exception: Exception | None) -> None: logger.info(f"Traceability metrics written to: {out_path}") +def _get_need_value(need: Any, key: str, default: Any = None) -> Any: + getter = getattr(need, "get", None) + if callable(getter): + return getter(key, default) + try: + return need[key] + except Exception: + return default + + +def _discover_requirement_types( + app: Sphinx, all_needs: list[Any], include_external: bool +) -> set[str]: + """Discover requirement directives that are both tagged and present.""" + tagged_requirements: set[str] = set() + needs_types = getattr(app.config, "needs_types", []) + for need_type in needs_types or []: + if not isinstance(need_type, dict): + continue + directive = need_type.get("directive") + tags = need_type.get("tags", []) + if not isinstance(directive, str): + continue + if not isinstance(tags, list): + continue + normalized = {str(tag).strip() for tag in tags} + if "requirement_excl_process" in normalized or "requirement" in normalized: + tagged_requirements.add(directive) + + present_types: set[str] = set() + for need in all_needs: + is_external = bool(_get_need_value(need, "is_external", False)) + if not include_external and is_external: + continue + need_type: Any = _get_need_value(need, "type", None) + if isinstance(need_type, str): + present_types.add(need_type) + discovered = tagged_requirements.intersection(present_types) + if not discovered: + # Fallback for repositories that use *_req directives but do not tag + # requirement types in needs_types. + discovered = {t for t in present_types if t.endswith("_req")} + if discovered: + logger.info( + "score_metamodel_requirement_types is not configured; " + f"using discovered requirement types: {', '.join(sorted(discovered))}" + ) + return discovered + + def _run_checks(app: Sphinx, exception: Exception | None) -> None: # Do not run checks if an exception occurred during build if exception: @@ -334,11 +391,12 @@ def setup(app: Sphinx) -> dict[str, str | bool]: app.add_config_value( "score_metamodel_requirement_types", - "tool_req", + "", rebuild="env", description=( "Comma-separated list of need types treated as requirements for " - "traceability metrics (default: tool_req)." + "traceability metrics. If empty, requirement types are autodiscovered " + "from needs_types tags (requirement, requirement_excl_process)." ), ) diff --git a/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py b/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py index 764659874..7d5e98885 100644 --- a/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py +++ b/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py @@ -38,6 +38,14 @@ def get_needs_view(self) -> dict[str, dict[str, object]]: "testlink": "", "is_external": False, }, + "LOCAL_FEAT": { + "id": "LOCAL_FEAT", + "type": "feat_req", + "implemented": "YES", + "source_code_link": "", + "testlink": "", + "is_external": False, + }, "EXT_REQ": { "id": "EXT_REQ", "type": "tool_req", @@ -46,16 +54,105 @@ def get_needs_view(self) -> dict[str, dict[str, object]]: "testlink": "", "is_external": True, }, + "EXT_FEAT": { + "id": "EXT_FEAT", + "type": "feat_req", + "implemented": "NO", + "source_code_link": "src/ext_feat.py:1", + "testlink": "", + "is_external": True, + }, + "EXT_GD": { + "id": "EXT_GD", + "type": "gd_req", + "implemented": "NO", + "source_code_link": "src/ext_gd.py:1", + "testlink": "", + "is_external": True, + }, } -def _app(tmp_path: Path, include_external: bool) -> SimpleNamespace: +class _FakeNonReqNeedsData: + def __init__(self, env: object): + self._env = env + + def get_needs_view(self) -> dict[str, dict[str, object]]: + return { + "LOCAL_COMP": { + "id": "LOCAL_COMP", + "type": "comp", + "implemented": "YES", + "source_code_link": "", + "testlink": "", + "is_external": False, + }, + "LOCAL_DOC": { + "id": "LOCAL_DOC", + "type": "document", + "implemented": "YES", + "source_code_link": "", + "testlink": "", + "is_external": False, + }, + } + + +class _NeedObj: + def __init__(self, payload: dict[str, object]): + self._payload = payload + + def get(self, key: str, default: object | None = None) -> object | None: + return self._payload.get(key, default) + + +class _FakeObjectNeedsData: + def __init__(self, env: object): + self._env = env + + def get_needs_view(self) -> dict[str, _NeedObj]: + return { + "LOCAL_REQ": _NeedObj( + { + "id": "LOCAL_REQ", + "type": "tool_req", + "implemented": "YES", + "source_code_link": "", + "testlink": "", + "is_external": False, + } + ), + "LOCAL_FEAT": _NeedObj( + { + "id": "LOCAL_FEAT", + "type": "feat_req", + "implemented": "YES", + "source_code_link": "", + "testlink": "", + "is_external": False, + } + ), + } + + +def _app( + tmp_path: Path, + include_external: bool, + requirement_types: str = "tool_req", + needs_types: list[dict[str, object]] | None = None, +) -> SimpleNamespace: + discovered_types = needs_types or [ + {"directive": "tool_req", "tags": ["requirement_excl_process"]}, + {"directive": "feat_req", "tags": ["requirement"]}, + {"directive": "workflow", "tags": []}, + ] return SimpleNamespace( env=object(), outdir=str(tmp_path), config=SimpleNamespace( - score_metamodel_requirement_types="tool_req", + score_metamodel_requirement_types=requirement_types, score_metamodel_include_external_needs=include_external, + needs_types=discovered_types, ), ) @@ -94,3 +191,202 @@ def test_write_metrics_json_can_include_external( assert metrics["include_external"] is True assert metrics["requirements"]["total"] == 2 + + +def test_explicit_requirement_types_disable_autodiscovery( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) + + metamodel_init._write_metrics_json( + cast( + Sphinx, + _app( + tmp_path, + include_external=False, + requirement_types="tool_req", + ), + ), + None, + ) + + payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) + assert set(payload["metrics_by_type"].keys()) == {"tool_req"} + + +def test_write_metrics_json_autodiscovers_when_types_unset( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) + + metamodel_init._write_metrics_json( + cast( + Sphinx, + _app(tmp_path, include_external=False, requirement_types=""), + ), + None, + ) + + payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) + assert payload["schema_version"] == "1" + assert set(payload["metrics_by_type"].keys()) == {"feat_req", "tool_req"} + + +def test_autodiscovery_excludes_tagged_types_not_present_in_needs( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) + + metamodel_init._write_metrics_json( + cast( + Sphinx, + _app( + tmp_path, + include_external=False, + requirement_types="", + needs_types=[ + {"directive": "tool_req", "tags": ["requirement_excl_process"]}, + {"directive": "feat_req", "tags": ["requirement"]}, + {"directive": "aou_req", "tags": ["requirement"]}, + ], + ), + ), + None, + ) + + payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) + assert set(payload["metrics_by_type"].keys()) == {"feat_req", "tool_req"} + + +def test_write_metrics_json_empty_when_no_types_configured_or_discovered( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNonReqNeedsData) + + metamodel_init._write_metrics_json( + cast( + Sphinx, + _app( + tmp_path, + include_external=False, + requirement_types="", + needs_types=[{"directive": "workflow", "tags": []}], + ), + ), + None, + ) + + payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) + assert payload["schema_version"] == "1" + assert payload["metrics_by_type"] == {} + + +def test_autodiscovery_falls_back_to_present_req_suffix_types( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) + + metamodel_init._write_metrics_json( + cast( + Sphinx, + _app( + tmp_path, + include_external=False, + requirement_types="", + needs_types=[{"directive": "workflow", "tags": []}], + ), + ), + None, + ) + + payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) + assert set(payload["metrics_by_type"].keys()) == {"feat_req", "tool_req"} + + +def test_autodiscovery_respects_include_external_scope( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) + + metamodel_init._write_metrics_json( + cast( + Sphinx, + _app( + tmp_path, + include_external=True, + requirement_types="", + needs_types=[{"directive": "workflow", "tags": []}], + ), + ), + None, + ) + + payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) + assert set(payload["metrics_by_type"].keys()) == {"feat_req", "gd_req", "tool_req"} + + +def test_autodiscovery_handles_needitem_like_objects( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeObjectNeedsData) + + metamodel_init._write_metrics_json( + cast( + Sphinx, + _app( + tmp_path, + include_external=False, + requirement_types="", + needs_types=[{"directive": "workflow", "tags": []}], + ), + ), + None, + ) + + payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) + assert set(payload["metrics_by_type"].keys()) == {"feat_req", "tool_req"} + + +@pytest.mark.parametrize( + ("requirement_types", "include_external", "should_exist", "expected_totals"), + [ + ("tool_req", False, True, {"tool_req": 1}), + ("feat_req,tool_req", False, True, {"feat_req": 1, "tool_req": 1}), + ("", False, True, {"feat_req": 1, "tool_req": 1}), + (" ", False, True, {"feat_req": 1, "tool_req": 1}), + ("tool_req", True, True, {"tool_req": 2}), + ], +) +def test_write_metrics_json_settings_matrix( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + requirement_types: str, + include_external: bool, + should_exist: bool, + expected_totals: dict[str, int], +) -> None: + monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) + + metamodel_init._write_metrics_json( + cast( + Sphinx, + _app( + tmp_path, + include_external=include_external, + requirement_types=requirement_types, + ), + ), + None, + ) + + metrics_file = tmp_path / "metrics.json" + assert metrics_file.exists() is should_exist + if not should_exist: + return + + payload = json.loads(metrics_file.read_text(encoding="utf-8")) + by_type = payload["metrics_by_type"] + assert set(by_type.keys()) == set(expected_totals.keys()) + + for req_type, expected_total in expected_totals.items(): + assert by_type[req_type]["requirements"]["total"] == expected_total From 3ba6ee067e359960d6feca5239ac655cebecc086 Mon Sep 17 00:00:00 2001 From: Frank Scholter Peres Date: Wed, 20 May 2026 11:18:03 +0000 Subject: [PATCH 02/11] docs: clarify requirement type autodiscovery override --- docs/how-to/dashboards_and_quality_gates.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/how-to/dashboards_and_quality_gates.rst b/docs/how-to/dashboards_and_quality_gates.rst index ea0797579..a0aa58fde 100644 --- a/docs/how-to/dashboards_and_quality_gates.rst +++ b/docs/how-to/dashboards_and_quality_gates.rst @@ -50,6 +50,11 @@ In ``docs/conf.py``: score_metamodel_requirement_types = "feat_req,comp_req,aou_req" score_metamodel_include_external_needs = False +By default, ``score_metamodel`` autodiscovers requirement types from the +repository needs that are present in the current build. If +``score_metamodel_requirement_types`` is set, that explicit list overrides +autodiscovery. + Use ``score_metamodel_include_external_needs = True`` only in repositories that intentionally aggregate requirements across module dependencies, such as integration repositories. Use ``False`` for module repositories to gate only on From 2c638a340ae0618feba4867abb7a6f9e44684418 Mon Sep 17 00:00:00 2001 From: Frank Scholter Peres Date: Wed, 20 May 2026 11:30:21 +0000 Subject: [PATCH 03/11] lint fix --- src/extensions/score_metamodel/__init__.py | 29 ++++++++++++++-------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/extensions/score_metamodel/__init__.py b/src/extensions/score_metamodel/__init__.py index f9fea3a9d..f196cc4a6 100644 --- a/src/extensions/score_metamodel/__init__.py +++ b/src/extensions/score_metamodel/__init__.py @@ -118,7 +118,9 @@ def _write_metrics_json(app: Sphinx, exception: Exception | None) -> None: raw = str(getattr(app.config, "score_metamodel_requirement_types", "")).strip() requirement_types = {t.strip() for t in raw.split(",") if t.strip()} if not requirement_types: - requirement_types = _discover_requirement_types(app, all_needs, include_external) + requirement_types = _discover_requirement_types( + app, all_needs, include_external + ) include_not_implemented = True metrics_by_type: dict[str, Any] = {} @@ -164,6 +166,19 @@ def _get_need_value(need: Any, key: str, default: Any = None) -> Any: return default +def _as_requirement_directive(need_type: Any) -> str | None: + if not isinstance(need_type, dict): + return None + directive = need_type.get("directive") + tags = need_type.get("tags", []) + if not isinstance(directive, str) or not isinstance(tags, list): + return None + normalized = {str(tag).strip() for tag in tags} + if "requirement_excl_process" in normalized or "requirement" in normalized: + return directive + return None + + def _discover_requirement_types( app: Sphinx, all_needs: list[Any], include_external: bool ) -> set[str]: @@ -171,16 +186,8 @@ def _discover_requirement_types( tagged_requirements: set[str] = set() needs_types = getattr(app.config, "needs_types", []) for need_type in needs_types or []: - if not isinstance(need_type, dict): - continue - directive = need_type.get("directive") - tags = need_type.get("tags", []) - if not isinstance(directive, str): - continue - if not isinstance(tags, list): - continue - normalized = {str(tag).strip() for tag in tags} - if "requirement_excl_process" in normalized or "requirement" in normalized: + directive = _as_requirement_directive(need_type) + if directive: tagged_requirements.add(directive) present_types: set[str] = set() From d892ca6a0c39479323d6edf980a9eb417be79bf9 Mon Sep 17 00:00:00 2001 From: Frank Scholter Peres Date: Fri, 22 May 2026 08:57:59 +0000 Subject: [PATCH 04/11] Refine requirement type autodiscovery --- docs/conf.py | 4 -- docs/how-to/dashboards_and_quality_gates.rst | 30 +++++--- src/extensions/score_metamodel/__init__.py | 20 +++--- ...st_traceability_metrics_json_generation.py | 71 +++---------------- 4 files changed, 37 insertions(+), 88 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 488b0c86b..0255915c2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -18,7 +18,3 @@ extensions = [ "score_sphinx_bundle", ] - -# Configure traceability metrics explicitly for this repository. -score_metamodel_requirement_types = "tool_req" -score_metamodel_include_external_needs = False diff --git a/docs/how-to/dashboards_and_quality_gates.rst b/docs/how-to/dashboards_and_quality_gates.rst index a0aa58fde..418fc6228 100644 --- a/docs/how-to/dashboards_and_quality_gates.rst +++ b/docs/how-to/dashboards_and_quality_gates.rst @@ -40,25 +40,35 @@ Typical Setup For details, see :ref:`setup`. -Minimal Configuration Example ------------------------------ +Configuration +------------- + +Default Behavior (No Configuration Needed) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +By default, ``score_metamodel`` autodiscovers requirement types from the +repository needs present in the current build. Requirement types are identified +from ``needs_types`` entries tagged with ``requirement`` or +``requirement_excl_process``. -In ``docs/conf.py``: +This is the recommended setup for most repositories. + +Optional Override for Requirement Types +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If a repository needs to force a specific set of requirement types, set an +explicit override in ``docs/conf.py``: .. code-block:: python score_metamodel_requirement_types = "feat_req,comp_req,aou_req" - score_metamodel_include_external_needs = False -By default, ``score_metamodel`` autodiscovers requirement types from the -repository needs that are present in the current build. If -``score_metamodel_requirement_types`` is set, that explicit list overrides +When this setting is provided, the explicit list is used instead of autodiscovery. Use ``score_metamodel_include_external_needs = True`` only in repositories that intentionally aggregate requirements across module dependencies, such as -integration repositories. Use ``False`` for module repositories to gate only on -local traceability. +integration repositories. Building the Dashboard ---------------------- @@ -92,7 +102,7 @@ There are two common modes: **Module repository** -- Set ``score_metamodel_include_external_needs = False``. +- No setting needed. Local-only scope is the default. - Gate only on the needs owned by the repository itself. - Use this for per-module implementation progress and traceability. diff --git a/src/extensions/score_metamodel/__init__.py b/src/extensions/score_metamodel/__init__.py index f196cc4a6..4d4eaa77a 100644 --- a/src/extensions/score_metamodel/__init__.py +++ b/src/extensions/score_metamodel/__init__.py @@ -157,13 +157,7 @@ def _write_metrics_json(app: Sphinx, exception: Exception | None) -> None: def _get_need_value(need: Any, key: str, default: Any = None) -> Any: - getter = getattr(need, "get", None) - if callable(getter): - return getter(key, default) - try: - return need[key] - except Exception: - return default + return need.get(key, default) def _as_requirement_directive(need_type: Any) -> str | None: @@ -198,11 +192,15 @@ def _discover_requirement_types( need_type: Any = _get_need_value(need, "type", None) if isinstance(need_type, str): present_types.add(need_type) + discovered = tagged_requirements.intersection(present_types) - if not discovered: - # Fallback for repositories that use *_req directives but do not tag - # requirement types in needs_types. - discovered = {t for t in present_types if t.endswith("_req")} + + if tagged_requirements and not discovered: + logger.warning( + "No requirement types discovered in current build for tagged " + "needs_types requirement directives." + ) + if discovered: logger.info( "score_metamodel_requirement_types is not configured; " diff --git a/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py b/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py index 7d5e98885..47af209cb 100644 --- a/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py +++ b/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py @@ -98,47 +98,10 @@ def get_needs_view(self) -> dict[str, dict[str, object]]: } -class _NeedObj: - def __init__(self, payload: dict[str, object]): - self._payload = payload - - def get(self, key: str, default: object | None = None) -> object | None: - return self._payload.get(key, default) - - -class _FakeObjectNeedsData: - def __init__(self, env: object): - self._env = env - - def get_needs_view(self) -> dict[str, _NeedObj]: - return { - "LOCAL_REQ": _NeedObj( - { - "id": "LOCAL_REQ", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "", - "testlink": "", - "is_external": False, - } - ), - "LOCAL_FEAT": _NeedObj( - { - "id": "LOCAL_FEAT", - "type": "feat_req", - "implemented": "YES", - "source_code_link": "", - "testlink": "", - "is_external": False, - } - ), - } - - def _app( tmp_path: Path, include_external: bool, - requirement_types: str = "tool_req", + requirement_types: str = "", needs_types: list[dict[str, object]] | None = None, ) -> SimpleNamespace: discovered_types = needs_types or [ @@ -281,7 +244,7 @@ def test_write_metrics_json_empty_when_no_types_configured_or_discovered( assert payload["metrics_by_type"] == {} -def test_autodiscovery_falls_back_to_present_req_suffix_types( +def test_autodiscovery_without_tagged_requirement_types_is_empty( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) @@ -300,7 +263,7 @@ def test_autodiscovery_falls_back_to_present_req_suffix_types( ) payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) - assert set(payload["metrics_by_type"].keys()) == {"feat_req", "tool_req"} + assert payload["metrics_by_type"] == {} def test_autodiscovery_respects_include_external_scope( @@ -315,7 +278,11 @@ def test_autodiscovery_respects_include_external_scope( tmp_path, include_external=True, requirement_types="", - needs_types=[{"directive": "workflow", "tags": []}], + needs_types=[ + {"directive": "tool_req", "tags": ["requirement_excl_process"]}, + {"directive": "feat_req", "tags": ["requirement"]}, + {"directive": "gd_req", "tags": ["requirement"]}, + ], ), ), None, @@ -325,28 +292,6 @@ def test_autodiscovery_respects_include_external_scope( assert set(payload["metrics_by_type"].keys()) == {"feat_req", "gd_req", "tool_req"} -def test_autodiscovery_handles_needitem_like_objects( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeObjectNeedsData) - - metamodel_init._write_metrics_json( - cast( - Sphinx, - _app( - tmp_path, - include_external=False, - requirement_types="", - needs_types=[{"directive": "workflow", "tags": []}], - ), - ), - None, - ) - - payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) - assert set(payload["metrics_by_type"].keys()) == {"feat_req", "tool_req"} - - @pytest.mark.parametrize( ("requirement_types", "include_external", "should_exist", "expected_totals"), [ From c7248983b1b7a162de7c53169b0532d7385c6a88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20S=C3=B6ren=20Pollak?= Date: Fri, 5 Jun 2026 14:23:13 +0200 Subject: [PATCH 05/11] feat: tracability improvements (#578) --- docs/conf.py | 2 + docs/how-to/dashboards_and_quality_gates.rst | 290 +++++++++++++++ .../requirements/implementation_state.rst | 201 ++++++----- docs/internals/requirements/requirements.rst | 12 + src/BUILD | 1 + src/extensions/score_metamodel/BUILD | 27 +- src/extensions/score_metamodel/__init__.py | 171 +-------- .../score_metamodel/checks/standards.py | 5 +- .../checks/traceability_dashboard.py | 181 ---------- .../tests/test_sphinx_filters.py | 74 ---- .../tests/test_traceability_dashboard.py | 218 ----------- .../tests/test_traceability_metrics.py | 206 ----------- ...st_traceability_metrics_json_generation.py | 337 ------------------ .../score_metamodel/traceability_metrics.py | 258 -------------- src/extensions/score_metrics/BUILD | 56 +++ src/extensions/score_metrics/__init__.py | 85 +++++ .../sphinx_filters.py | 81 ++++- .../tests/test_sphinx_filters.py | 205 +++++++++++ .../tests/test_traceability_metrics.py | 255 +++++++++++++ .../score_metrics/traceability_metrics.py | 195 ++++++++++ .../tests/test_xml_parser.py | 8 +- .../score_source_code_linker/xml_parser.py | 43 +-- src/extensions/score_sphinx_bundle/BUILD | 1 + .../score_sphinx_bundle/__init__.py | 1 + 24 files changed, 1343 insertions(+), 1570 deletions(-) delete mode 100644 src/extensions/score_metamodel/checks/traceability_dashboard.py delete mode 100644 src/extensions/score_metamodel/tests/test_sphinx_filters.py delete mode 100644 src/extensions/score_metamodel/tests/test_traceability_dashboard.py delete mode 100644 src/extensions/score_metamodel/tests/test_traceability_metrics.py delete mode 100644 src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py delete mode 100644 src/extensions/score_metamodel/traceability_metrics.py create mode 100644 src/extensions/score_metrics/BUILD create mode 100644 src/extensions/score_metrics/__init__.py rename src/extensions/{score_metamodel => score_metrics}/sphinx_filters.py (66%) create mode 100644 src/extensions/score_metrics/tests/test_sphinx_filters.py create mode 100644 src/extensions/score_metrics/tests/test_traceability_metrics.py create mode 100644 src/extensions/score_metrics/traceability_metrics.py diff --git a/docs/conf.py b/docs/conf.py index 0255915c2..56031d451 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,6 +10,7 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* +import matplotlib project = "Score Docs-as-Code" project_url = "https://eclipse-score.github.io/docs-as-code/" @@ -18,3 +19,4 @@ extensions = [ "score_sphinx_bundle", ] +matplotlib.rcParamsDefault["savefig.bbox"] = "tight" diff --git a/docs/how-to/dashboards_and_quality_gates.rst b/docs/how-to/dashboards_and_quality_gates.rst index 418fc6228..0bdaf9465 100644 --- a/docs/how-to/dashboards_and_quality_gates.rst +++ b/docs/how-to/dashboards_and_quality_gates.rst @@ -151,6 +151,296 @@ For a new consumer repository: 4. Introduce modest thresholds in CI. 5. Raise thresholds over time as the repository matures. +.. + ╓ ╖ + ║ .. The Following part has been generated by Copilot ║ + ╙ ╜ + +Metrics Needpie functions +========================= + +Overview +-------- + +These helpers read values from a nested dictionary named ``CALCULATED_METRICS``. +A metric is selected by a **colon-separated path**, for example: + +- ``overall_metrics:total`` +- ``overall_metrics:with_test_link`` +- ``types:bug:total`` + +All resolved values are converted to ``int`` and appended to a mutable +``results`` list passed by the caller. + +The ``needs`` parameter is accepted for integration compatibility, but it is not +used by the current implementations. + +Path format +----------- + +Paths are split using ``:`` and then resolved step by step. + +Example: + +.. code-block:: text + + path = "overall_metrics:total" + +is resolved like: + +.. code-block:: python + + current = CALCULATED_METRICS + current = current["overall_metrics"] + current = current["total"] + +Function reference +------------------ + +_get_key_values(results, argument_paths) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Purpose +^^^^^^^ + +Internal helper that appends metric values for a list of paths. + +Behavior +^^^^^^^^ + +1. Iterate over each item in ``argument_paths``. +2. Strip whitespace. +3. Skip empty paths. +4. Resolve the path inside ``CALCULATED_METRICS``. +5. Convert to integer and append to ``results``. + +Notes +^^^^^ + +- Modifies ``results`` in place. +- Does not return a new list. +- Raises ``KeyError`` if a path key does not exist. +- Raises ``ValueError`` if a resolved value cannot be converted to ``int``. + +get_metrics_with_overall_total_considered(...) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Purpose +^^^^^^^ + +Compute a remainder against global overall total. + +Behavior +^^^^^^^^ + +1. Append ``CALCULATED_METRICS["overall_metrics"]["total"]`` as first value. +2. Append each metric referenced by ``kwargs.values()``. +3. Replace the first value with: + + ``overall_total - sum(all other appended values)`` + +Typical use case +^^^^^^^^^^^^^^^^ + +Use this when your pie/chart needs an "Other" bucket based on the global total. + +Example +^^^^^^^ + +Assume: + +.. code-block:: python + + CALCULATED_METRICS = { + "overall_metrics": { + "total": 100, + "with_test_link": 30, + "with_review": 20, + } + } + +Call: + +.. code-block:: python + + results = [] + get_metrics_with_overall_total_considered( + needs=[], + results=results, + a="overall_metrics:with_test_link", + b="overall_metrics:with_review", + ) + +Result: + +.. code-block:: python + + # Step 1 append total: [100] + # Step 2 append selected: [100, 30, 20] + # Step 3 remainder: [50, 30, 20] + results == [50, 30, 20] + +get_metrics_with_custom_type_total_considered(...) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Purpose +^^^^^^^ + +Support custom total path if provided as the last kwarg value. + +Behavior +^^^^^^^^ + +If the **last** provided path ends with ``:total``: + +1. Append that last path as baseline total. +2. Append all preceding paths as components. +3. Replace the first value with: + + ``baseline_total - sum(components)`` + +Otherwise: + +- Append all provided paths directly (no subtraction). + +Important ordering rule +^^^^^^^^^^^^^^^^^^^^^^^ + +The special total path must be the **last** kwarg value, for example: + +.. code-block:: python + + get_metrics_with_custom_type_total_considered( + needs=[], + results=results, + part_a="types:feature:done", + part_b="types:feature:in_progress", + total="types:feature:total", # must be last + ) + +Example with custom total +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Assume: + +.. code-block:: python + + CALCULATED_METRICS = { + "types": { + "feature": { + "total": 40, + "done": 10, + "in_progress": 5, + } + } + } + +Call: + +.. code-block:: python + + results = [] + get_metrics_with_custom_type_total_considered( + needs=[], + results=results, + done="types:feature:done", + in_progress="types:feature:in_progress", + total="types:feature:total", + ) + +Result: + +.. code-block:: python + + # baseline total: 40 + # components: 10, 5 + # remainder: 40 - (10 + 5) = 25 + results == [25, 10, 5] + +Example without custom total +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Call: + +.. code-block:: python + + results = [] + get_metrics_with_custom_type_total_considered( + needs=[], + results=results, + done="types:feature:done", + in_progress="types:feature:in_progress", + ) + +Result: + +.. code-block:: python + + # no subtraction mode + results == [10, 5] + +get_just_metrics(...) +~~~~~~~~~~~~~~~~~~~~~ + +Purpose +^^^^^^^ + +Append only the selected metric values. + +Behavior +^^^^^^^^ + +- Interprets each kwarg value as a path. +- Resolves and appends each value. +- No total baseline and no remainder calculation. + +Example +^^^^^^^ + +.. code-block:: python + + results = [] + get_just_metrics( + needs=[], + results=results, + a="overall_metrics:with_test_link", + b="overall_metrics:with_review", + ) + # results might become [30, 20] + +How to use these functions correctly +------------------------------------ + +1. Pass a mutable list in ``results`` (usually ``[]`` initially). +2. Provide metric paths through keyword argument values. +3. Use colon-separated keys (``a:b:c``), not dot notation. +4. For custom-total mode, ensure the total path is the last kwarg value. +5. Expect in-place mutation of ``results``. + +Common pitfalls +--------------- + +- Empty kwargs in custom-total function can lead to index errors + (because ``values[-1]`` is accessed). +- Invalid paths raise ``KeyError``. +- Non-integer values raise ``ValueError`` during ``int(...)`` conversion. +- ``print(results)`` currently causes side effects during execution; remove if + quiet behavior is preferred in production. + +Testing recommendations +----------------------- + +Add unit tests for: + +- Basic path resolution with one and multiple paths. +- Whitespace and empty-path handling. +- Overall total remainder logic. +- Custom total behavior when last path ends with ``:total``. +- Behavior when no custom total is provided. +- Error handling for missing keys and non-integer values. +- Empty ``kwargs`` behavior in custom-total function. + + Related Guides -------------- diff --git a/docs/internals/requirements/implementation_state.rst b/docs/internals/requirements/implementation_state.rst index e46df74fa..32dc974d7 100644 --- a/docs/internals/requirements/implementation_state.rst +++ b/docs/internals/requirements/implementation_state.rst @@ -23,105 +23,114 @@ requirements. It focuses on tooling capabilities offered to downstream repositories rather than on product-specific traceability inside those repositories. -Overview --------- - -.. needpie:: Tool Requirements Status - :labels: not implemented, implemented but incomplete traceability, fully linked - :colors: red,yellow, green - :filter-func: src.extensions.score_metamodel.checks.traceability_dashboard.pie_requirements_status(tool_req) - -Jump to evidence tables: - -- :ref:`Tool Requirement Implementation and Links table ` -- :ref:`Process Requirement to Tool Requirement mapping table ` - -How To Read These Levels ------------------------- - -The overview pie combines implementation state and traceability evidence: - -- ``not implemented``: - requirement has ``implemented == NO``. -- ``implemented but incomplete traceability``: - requirement has ``implemented == YES`` or ``implemented == PARTIAL``, - but is missing at least one traceability link (code link and/or test link). -- ``fully linked``: - requirement is implemented and has both ``source_code_link`` and ``testlink``. - -Implementation labels used on this page: - -- ``NO``: requirement is not implemented. -- ``PARTIAL``: requirement is partly implemented. -- ``YES``: requirement is implemented. - -Why multiple pies are shown: - -- ``Requirements with Codelinks`` shows requirement-to-implementation traceability. -- ``Requirements with linked tests`` shows requirement-to-verification traceability. -- ``Requirements fully linked`` is the strict roll-up (both links present). - -These are intentionally separate because they answer different diagnostics: -missing code links, missing test links, or both. - -In Detail ---------- - -.. grid:: 2 - :class-container: score-grid - - .. grid-item-card:: - - .. needpie:: Requirements marked as Implemented - :labels: not implemented, partial, implemented - :colors: red, orange, green - - type == 'tool_req' and implemented == 'NO' - type == 'tool_req' and implemented == 'PARTIAL' - type == 'tool_req' and implemented == 'YES' - - .. grid-item-card:: - - .. needpie:: Requirements with Codelinks - :labels: no codelink, with codelink - :colors: red, green - :filter-func: src.extensions.score_metamodel.checks.traceability_dashboard.pie_requirements_with_code_links(tool_req) - - .. grid-item-card:: - - .. needpie:: Requirements with linked tests - :labels: no test link, with test link - :colors: red, green - :filter-func: src.extensions.score_metamodel.checks.traceability_dashboard.pie_requirements_with_test_links(tool_req) - - .. grid-item-card:: - - .. needpie:: Requirements fully linked (code + tests) - :labels: not fully linked, fully linked - :colors: orange, green - :filter-func: src.extensions.score_metamodel.checks.traceability_dashboard.pie_requirements_fully_linked(tool_req) - - .. grid-item-card:: - - .. needpie:: Process requirements linked by tool requirements - :labels: not linked, linked - :colors: red, green - :filter-func: src.extensions.score_metamodel.checks.traceability_dashboard.pie_process_requirements_linked(tool_req,true) +.. Overview +.. -------- +.. +.. needpie:: Overall Metrics with Total incooperated + :labels: without any link, overall with test link, overall with code, overall fully linked + :colors: red, yellow, blue, green + :filter-func: score_metrics.sphinx_filters.get_metrics_with_overall_total_considered(overall_metrics:with_test_link,overall_metrics:with_code_link,overall_metrics:fully_linked) -Process-to-Tool Mapping ------------------------ -.. _tooling_coverage_table_process_mapping: -.. needtable:: Process requirement -> tool requirement mapping - :types: tool_req - :columns: satisfies as "Process Requirement";id as "Tool Requirement" - :style: table +.. needpie:: Metrics without any total incooperated + :labels: tool req with test link, tool req with code, overall fully linked + :colors: yellow, blue, green + :filter-func: score_metrics.sphinx_filters.get_just_metrics(metrics_by_type:tool_req:with_test_link,metrics_by_type:tool_req:with_code_link,overall_metrics:fully_linked) -.. _tooling_coverage_table_impl_links: -.. needtable:: Tool requirement implementation and links - :types: tool_req - :columns: id as "Tool Requirement";implemented;source_code_link;testlink - :style: table +.. Jump to evidence tables: +.. +.. - :ref:`Tool Requirement Implementation and Links table ` +.. - :ref:`Process Requirement to Tool Requirement mapping table ` +.. +.. How To Read These Levels +.. ------------------------ +.. +.. The overview pie combines implementation state and traceability evidence: +.. +.. - ``not implemented``: +.. requirement has ``implemented == NO``. +.. - ``implemented but incomplete traceability``: +.. requirement has ``implemented == YES`` or ``implemented == PARTIAL``, +.. but is missing at least one traceability link (code link and/or test link). +.. - ``fully linked``: +.. requirement is implemented and has both ``source_code_link`` and ``testlink``. +.. +.. Implementation labels used on this page: +.. +.. - ``NO``: requirement is not implemented. +.. - ``PARTIAL``: requirement is partly implemented. +.. - ``YES``: requirement is implemented. +.. +.. Why multiple pies are shown: +.. +.. - ``Requirements with Codelinks`` shows requirement-to-implementation traceability. +.. - ``Requirements with linked tests`` shows requirement-to-verification traceability. +.. - ``Requirements fully linked`` is the strict roll-up (both links present). +.. +.. These are intentionally separate because they answer different diagnostics: +.. missing code links, missing test links, or both. +.. +.. In Detail +.. --------- +.. +.. .. grid:: 2 +.. :class-container: score-grid +.. +.. .. grid-item-card:: +.. +.. .. needpie:: Requirements marked as Implemented +.. :labels: not implemented, partial, implemented +.. :colors: red, orange, green +.. +.. type == 'tool_req' and implemented == 'NO' +.. type == 'tool_req' and implemented == 'PARTIAL' +.. type == 'tool_req' and implemented == 'YES' +.. +.. .. grid-item-card:: +.. +.. .. needpie:: Requirements with Codelinks +.. :labels: no codelink, with codelink +.. :colors: red, green +.. :filter-func: src.extensions.score_metrics.traceability_dashboard.pie_requirements_with_code_links(tool_req) +.. +.. .. grid-item-card:: +.. +.. .. needpie:: Requirements with linked tests +.. :labels: no test link, with test link +.. :colors: red, green +.. :filter-func: src.extensions.score_metrics.traceability_dashboard.pie_requirements_with_test_links(tool_req) +.. +.. .. grid-item-card:: +.. +.. .. needpie:: Requirements fully linked (code + tests) +.. :labels: not fully linked, fully linked +.. :colors: orange, green +.. :filter-func: src.extensions.score_metrics.traceability_dashboard.pie_requirements_fully_linked(tool_req) +.. +.. .. grid-item-card:: +.. +.. .. needpie:: Process requirements linked by tool requirements +.. :labels: not linked, linked +.. :colors: red, green +.. :filter-func: src.extensions.score_metrics.traceability_dashboard.pie_process_requirements_linked(tool_req,true) +.. +.. +.. Process-to-Tool Mapping +.. ----------------------- +.. +.. .. _tooling_coverage_table_process_mapping: +.. +.. .. needtable:: Process requirement -> tool requirement mapping +.. :types: tool_req +.. :columns: satisfies as "Process Requirement";id as "Tool Requirement" +.. :style: table +.. +.. .. _tooling_coverage_table_impl_links: +.. +.. .. needtable:: Tool requirement implementation and links +.. :types: tool_req +.. :columns: id as "Tool Requirement";implemented;source_code_link;testlink +.. :style: table diff --git a/docs/internals/requirements/requirements.rst b/docs/internals/requirements/requirements.rst index 45df88e71..220d94dae 100644 --- a/docs/internals/requirements/requirements.rst +++ b/docs/internals/requirements/requirements.rst @@ -852,6 +852,18 @@ Testing - If Partially/FullyVerifies are set in Unit Test these shall link to Component Requirements +.. tool_req:: Provide Metrics for linked requirements + :id: tool_req__docs_test_linkage_metrics + :tags: Testing + :version: 1 + :implemented: YES + :parent_covered: NO: Placeholder process requirement + :satisfies: gd_req__verification_reporting + :status: invalid + + Docs-AS-Code shall provide a way to gather statistics on linkages to implementation(source_code_links) & tests(testlink) for all needs. + It shall also be possible to filter these by type and use the provided statistics in the documentation (via diagrams drawn from it etc.) + 🧪 Tool Verification Reports ############################ diff --git a/src/BUILD b/src/BUILD index b4bf27c14..5f8dd1bef 100644 --- a/src/BUILD +++ b/src/BUILD @@ -53,6 +53,7 @@ filegroup( "//src/extensions/score_source_code_linker:all_sources", "//src/extensions/score_sphinx_bundle:all_sources", "//src/extensions/score_sync_toml:all_sources", + "//src/extensions/score_metrics:all_sources", "//src/helper_lib:all_sources", ], visibility = ["//visibility:public"], diff --git a/src/extensions/score_metamodel/BUILD b/src/extensions/score_metamodel/BUILD index 64c0ebeb5..9fdb011e2 100644 --- a/src/extensions/score_metamodel/BUILD +++ b/src/extensions/score_metamodel/BUILD @@ -55,6 +55,7 @@ py_library( visibility = ["//visibility:public"], # TODO: Figure out if all requirements are needed or if we can break it down a bit deps = all_requirements + [ + "@score_docs_as_code//src/extensions/score_metrics", "@score_docs_as_code//src/helper_lib", ], ) @@ -70,13 +71,17 @@ score_pytest( ), data = ["tests/model/simple_model.yaml"], pytest_config = "//:pyproject.toml", + visibility = ["//visibility:public"], deps = [":score_metamodel"], ) score_pytest( name = "file_based_tests_architecture", srcs = ["tests/test_rules_file_based.py"], - data = glob(["tests/rst/architecture/*.rst"]) + ["tests/rst/conf.py", "tests/rst/needs.json"], + data = glob(["tests/rst/architecture/*.rst"]) + [ + "tests/rst/conf.py", + "tests/rst/needs.json", + ], pytest_config = "//:pyproject.toml", deps = [":score_metamodel"], ) @@ -84,7 +89,10 @@ score_pytest( score_pytest( name = "file_based_tests_attributes", srcs = ["tests/test_rules_file_based.py"], - data = glob(["tests/rst/attributes/*.rst"]) + ["tests/rst/conf.py", "tests/rst/needs.json"], + data = glob(["tests/rst/attributes/*.rst"]) + [ + "tests/rst/conf.py", + "tests/rst/needs.json", + ], pytest_config = "//:pyproject.toml", deps = [":score_metamodel"], ) @@ -92,7 +100,10 @@ score_pytest( score_pytest( name = "file_based_tests_graph", srcs = ["tests/test_rules_file_based.py"], - data = glob(["tests/rst/graph/*.rst"]) + ["tests/rst/conf.py", "tests/rst/needs.json"], + data = glob(["tests/rst/graph/*.rst"]) + [ + "tests/rst/conf.py", + "tests/rst/needs.json", + ], pytest_config = "//:pyproject.toml", deps = [":score_metamodel"], ) @@ -100,7 +111,10 @@ score_pytest( score_pytest( name = "file_based_tests_id_contains_feature", srcs = ["tests/test_rules_file_based.py"], - data = glob(["tests/rst/id_contains_feature/*.rst"]) + ["tests/rst/conf.py", "tests/rst/needs.json"], + data = glob(["tests/rst/id_contains_feature/*.rst"]) + [ + "tests/rst/conf.py", + "tests/rst/needs.json", + ], pytest_config = "//:pyproject.toml", deps = [":score_metamodel"], ) @@ -108,7 +122,10 @@ score_pytest( score_pytest( name = "file_based_tests_options", srcs = ["tests/test_rules_file_based.py"], - data = glob(["tests/rst/options/*.rst"]) + ["tests/rst/conf.py", "tests/rst/needs.json"], + data = glob(["tests/rst/options/*.rst"]) + [ + "tests/rst/conf.py", + "tests/rst/needs.json", + ], pytest_config = "//:pyproject.toml", deps = [":score_metamodel"], ) diff --git a/src/extensions/score_metamodel/__init__.py b/src/extensions/score_metamodel/__init__.py index 471394c6e..8adc43702 100644 --- a/src/extensions/score_metamodel/__init__.py +++ b/src/extensions/score_metamodel/__init__.py @@ -11,12 +11,10 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* import importlib -import json import os import pkgutil from collections.abc import Callable from pathlib import Path -from typing import Any from sphinx.application import Sphinx from sphinx_needs import logging @@ -31,9 +29,6 @@ ProhibitedWordCheck as ProhibitedWordCheck, ScoreNeedType as ScoreNeedType, ) -from src.extensions.score_metamodel.traceability_metrics import ( - compute_traceability_summary, -) from src.extensions.score_metamodel.yaml_parser import ( default_options as default_options, load_metamodel_data as load_metamodel_data, @@ -99,116 +94,6 @@ def graph_check(func: graph_check_function): return func -def _write_metrics_json(app: Sphinx, exception: Exception | None) -> None: - """Write a schema-v1 metrics.json alongside needs.json in the build output. - - This is the single source of truth for traceability metrics. It runs - inside the Sphinx build so it has access to all needs (local + external) - and produces the same metrics the dashboard pie charts display. - The traceability_gate reads this file to enforce CI thresholds. - """ - if exception: - return - - all_needs: list[Any] = list(SphinxNeedsData(app.env).get_needs_view().values()) - include_external: bool = bool( - getattr(app.config, "score_metamodel_include_external_needs", False) - ) - - raw = str(getattr(app.config, "score_metamodel_requirement_types", "")).strip() - requirement_types = {t.strip() for t in raw.split(",") if t.strip()} - if not requirement_types: - requirement_types = _discover_requirement_types( - app, all_needs, include_external - ) - include_not_implemented = True - - metrics_by_type: dict[str, Any] = {} - if not requirement_types: - logger.info( - "No requirement types configured or discovered; writing empty metrics.json." - ) - else: - for req_type in sorted(requirement_types): - type_summary = compute_traceability_summary( - all_needs=all_needs, - requirement_types={req_type}, - include_not_implemented=include_not_implemented, - filtered_test_types=set(), - include_external=include_external, - ) - metrics_by_type[req_type] = { - "include_not_implemented": type_summary["include_not_implemented"], - "include_external": type_summary["include_external"], - "requirements": type_summary["requirements"], - "tests": type_summary["tests"], - } - - output: dict[str, Any] = { - "schema_version": "1", - "generated_by": "sphinx_build", - "metrics_by_type": metrics_by_type, - } - - out_path = Path(app.outdir) / "metrics.json" - out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_text(json.dumps(output, indent=2), encoding="utf-8") - logger.info(f"Traceability metrics written to: {out_path}") - - -def _get_need_value(need: Any, key: str, default: Any = None) -> Any: - return need.get(key, default) - - -def _as_requirement_directive(need_type: Any) -> str | None: - if not isinstance(need_type, dict): - return None - directive = need_type.get("directive") - tags = need_type.get("tags", []) - if not isinstance(directive, str) or not isinstance(tags, list): - return None - normalized = {str(tag).strip() for tag in tags} - if "requirement_excl_process" in normalized or "requirement" in normalized: - return directive - return None - - -def _discover_requirement_types( - app: Sphinx, all_needs: list[Any], include_external: bool -) -> set[str]: - """Discover requirement directives that are both tagged and present.""" - tagged_requirements: set[str] = set() - needs_types = getattr(app.config, "needs_types", []) - for need_type in needs_types or []: - directive = _as_requirement_directive(need_type) - if directive: - tagged_requirements.add(directive) - - present_types: set[str] = set() - for need in all_needs: - is_external = bool(_get_need_value(need, "is_external", False)) - if not include_external and is_external: - continue - need_type: Any = _get_need_value(need, "type", None) - if isinstance(need_type, str): - present_types.add(need_type) - - discovered = tagged_requirements.intersection(present_types) - - if tagged_requirements and not discovered: - logger.warning( - "No requirement types discovered in current build for tagged " - "needs_types requirement directives." - ) - - if discovered: - logger.info( - "score_metamodel_requirement_types is not configured; " - f"using discovered requirement types: {', '.join(sorted(discovered))}" - ) - return discovered - - def _run_checks(app: Sphinx, exception: Exception | None) -> None: # Do not run checks if an exception occurred during build if exception: @@ -276,32 +161,6 @@ def is_check_enabled(check: local_check_function | graph_check_function): ) -def _configure_traceability_dashboard(app: Sphinx, config: object) -> None: - """Propagate repo-level traceability settings to dashboard filters.""" - from src.extensions.score_metamodel.checks.traceability_dashboard import ( - set_default_include_external, - ) - - include_external = bool( - getattr(config, "score_metamodel_include_external_needs", False) - ) - set_default_include_external(include_external) - - -def _remove_prefix(word: str, prefixes: list[str]) -> str: - for prefix in prefixes or []: - if isinstance(word, str) and word.startswith(prefix): - return word.removeprefix(prefix) - return word - - -def _get_need_type_for_need(app: Sphinx, need: NeedItem) -> ScoreNeedType: - for nt in app.config.needs_types: - if nt["directive"] == need["type"]: - return nt - raise ValueError(f"Need type {need['type']} not found in needs_types") - - def _resolve_linkable_types( link_name: str, link_value: str, @@ -410,8 +269,10 @@ def setup(app: Sphinx) -> dict[str, str | bool]: # sphinx-collections runs on default prio 500. # We need to populate the sphinx-collections config before that happens. - # --> 499 - _ = app.connect("config-inited", connect_external_needs, priority=499) + # If we put it anywhere higher it seems that other things already lock the needs + # To ensure that this runs first before locking happens priot is => 450 + # The lower the number the higher priority it has (runs earlier) + _ = app.connect("config-inited", connect_external_needs, priority=450) discover_checks() @@ -424,30 +285,6 @@ def setup(app: Sphinx) -> dict[str, str | bool]: ), ) - app.add_config_value( - "score_metamodel_requirement_types", - "", - rebuild="env", - description=( - "Comma-separated list of need types treated as requirements for " - "traceability metrics. If empty, requirement types are autodiscovered " - "from needs_types tags (requirement, requirement_excl_process)." - ), - ) - - app.add_config_value( - "score_metamodel_include_external_needs", - False, - rebuild="env", - description=( - "When True, include external requirements in dashboard and CI metrics. " - "Default is False so each repo gates only its own needs." - ), - ) - - _ = app.connect("config-inited", _configure_traceability_dashboard, priority=498) - - _ = app.connect("build-finished", _write_metrics_json) _ = app.connect("build-finished", _run_checks) return { diff --git a/src/extensions/score_metamodel/checks/standards.py b/src/extensions/score_metamodel/checks/standards.py index a04d176ba..da32a3ce2 100644 --- a/src/extensions/score_metamodel/checks/standards.py +++ b/src/extensions/score_metamodel/checks/standards.py @@ -16,13 +16,12 @@ # ╙ ╜ # from sphinx.application import Sphinx -from sphinx_needs.need_item import NeedItem - -from ..sphinx_filters import ( +from score_metrics.sphinx_filters import ( generic_pie_items_by_tag, generic_pie_items_in_relationships, generic_pie_linked_items, ) +from sphinx_needs.need_item import NeedItem # from score_metamodel import ( # CheckLogger, diff --git a/src/extensions/score_metamodel/checks/traceability_dashboard.py b/src/extensions/score_metamodel/checks/traceability_dashboard.py deleted file mode 100644 index a59087928..000000000 --- a/src/extensions/score_metamodel/checks/traceability_dashboard.py +++ /dev/null @@ -1,181 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -# ╓ ╖ -# ║ Some portions generated by Github Copilot ║ -# ╙ ╜ - -"""Needpie filter functions backed by shared traceability metric calculations.""" - -from __future__ import annotations - -from collections.abc import Sequence -from typing import Any - -from sphinx_needs.need_item import NeedItem - -from ..traceability_metrics import compute_traceability_summary, filter_requirements - -_DEFAULT_INCLUDE_EXTERNAL = False - - -def set_default_include_external(include_external: bool) -> None: - """Configure default behaviour for including external requirements.""" - global _DEFAULT_INCLUDE_EXTERNAL - _DEFAULT_INCLUDE_EXTERNAL = bool(include_external) - - -def _include_external(kwargs: dict[str, str | int | float]) -> bool: - """Read include_external override from filter args, else use configured default.""" - raw = kwargs.get("arg2") - if raw is None: - return _DEFAULT_INCLUDE_EXTERNAL - text = str(raw).strip().lower() - return text in {"1", "true", "yes", "on"} - - -def _requirement_types(kwargs: dict[str, str | int | float]) -> set[str]: - raw = str(kwargs.get("arg1", "tool_req")).strip() - values = {value.strip() for value in raw.split(",") if value.strip()} - return values or {"tool_req"} - - -def pie_requirements_status( - needs: Sequence[NeedItem | dict[str, Any]], - results: list[int], - **kwargs: str | int | float, -) -> None: - """Dashboard status split: not implemented, implemented/incomplete, fully linked.""" - req_types = _requirement_types(kwargs) - include_external = _include_external(kwargs) - - all_requirements = filter_requirements( - needs, - requirement_types=req_types, - include_not_implemented=True, - include_external=include_external, - ) - implemented_requirements = filter_requirements( - needs, - requirement_types=req_types, - include_not_implemented=False, - include_external=include_external, - ) - summary = compute_traceability_summary( - all_needs=needs, - requirement_types=req_types, - include_not_implemented=False, - filtered_test_types=set(), - include_external=include_external, - ) - - not_implemented = len(all_requirements) - len(implemented_requirements) - fully_linked = int(summary["requirements"]["fully_linked"]) - implemented_incomplete = len(implemented_requirements) - fully_linked - - results.append(not_implemented) - results.append(implemented_incomplete) - results.append(fully_linked) - - -def pie_requirements_with_code_links( - needs: Sequence[NeedItem | dict[str, Any]], - results: list[int], - **kwargs: str | int | float, -) -> None: - """Dashboard split: requirements with and without source code links.""" - req_types = _requirement_types(kwargs) - include_external = _include_external(kwargs) - summary = compute_traceability_summary( - all_needs=needs, - requirement_types=req_types, - include_not_implemented=True, - filtered_test_types=set(), - include_external=include_external, - ) - - total = int(summary["requirements"]["total"]) - with_code = int(summary["requirements"]["with_code_link"]) - - results.append(total - with_code) - results.append(with_code) - - -def pie_requirements_with_test_links( - needs: Sequence[NeedItem | dict[str, Any]], - results: list[int], - **kwargs: str | int | float, -) -> None: - """Dashboard split: requirements with and without testcase links.""" - req_types = _requirement_types(kwargs) - include_external = _include_external(kwargs) - summary = compute_traceability_summary( - all_needs=needs, - requirement_types=req_types, - include_not_implemented=True, - filtered_test_types=set(), - include_external=include_external, - ) - - total = int(summary["requirements"]["total"]) - with_test = int(summary["requirements"]["with_test_link"]) - - results.append(total - with_test) - results.append(with_test) - - -def pie_requirements_fully_linked( - needs: Sequence[NeedItem | dict[str, Any]], - results: list[int], - **kwargs: str | int | float, -) -> None: - """Dashboard split: requirements fully linked vs incomplete.""" - req_types = _requirement_types(kwargs) - include_external = _include_external(kwargs) - summary = compute_traceability_summary( - all_needs=needs, - requirement_types=req_types, - include_not_implemented=True, - filtered_test_types=set(), - include_external=include_external, - ) - - total = int(summary["requirements"]["total"]) - fully_linked = int(summary["requirements"]["fully_linked"]) - - results.append(total - fully_linked) - results.append(fully_linked) - - -def pie_process_requirements_linked( - needs: Sequence[NeedItem | dict[str, Any]], - results: list[int], - **kwargs: str | int | float, -) -> None: - """Dashboard split: process requirements linked vs not linked.""" - req_types = _requirement_types(kwargs) - include_external = _include_external(kwargs) - summary = compute_traceability_summary( - all_needs=needs, - requirement_types=req_types, - include_not_implemented=True, - filtered_test_types=set(), - include_external=include_external, - ) - - process_requirements = summary["process_requirements"] - total = int(process_requirements["total"]) - linked = int(process_requirements["linked"]) - - results.append(total - linked) - results.append(linked) diff --git a/src/extensions/score_metamodel/tests/test_sphinx_filters.py b/src/extensions/score_metamodel/tests/test_sphinx_filters.py deleted file mode 100644 index c065a01b7..000000000 --- a/src/extensions/score_metamodel/tests/test_sphinx_filters.py +++ /dev/null @@ -1,74 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -from typing import cast - -from sphinx_needs.need_item import NeedItem - -from src.extensions.score_metamodel.sphinx_filters import ( - generic_pie_items_by_tag, - generic_pie_linked_items, -) - - -def test_generic_pie_linked_items_matches_source_by_id_prefix() -> None: - needs = cast( - list[NeedItem], - [ - {"id": "std_req__iso26262__001", "type": "std_req"}, - # Type intentionally does not match selector prefix, id does. - { - "id": "gd_guidl__xyz", - "type": "guideline", - "complies": ["std_req__iso26262__001"], - }, - ], - ) - - results: list[int] = [] - generic_pie_linked_items( - needs, - results, - arg1="std_req__iso26262__", - arg2="gd_", - arg3="complies", - ) - - assert results == [1, 0] - - -def test_generic_pie_items_by_tag_matches_source_by_id_prefix() -> None: - needs = cast( - list[NeedItem], - [ - {"id": "REQ_A", "type": "tool_req", "tags": ["aspice40_man5"]}, - {"id": "REQ_B", "type": "tool_req", "tags": ["aspice40_man5"]}, - # Type intentionally does not match selector prefix, id does. - { - "id": "gd_req__abc", - "type": "process_requirement", - "complies": ["REQ_A"], - }, - ], - ) - - results: list[int] = [] - generic_pie_items_by_tag( - needs, - results, - arg1="aspice40_man5", - arg2="gd_", - arg3="complies", - ) - - assert results == [1, 1] diff --git a/src/extensions/score_metamodel/tests/test_traceability_dashboard.py b/src/extensions/score_metamodel/tests/test_traceability_dashboard.py deleted file mode 100644 index 2e06b3b7d..000000000 --- a/src/extensions/score_metamodel/tests/test_traceability_dashboard.py +++ /dev/null @@ -1,218 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -"""Tests that dashboard filters follow local/external settings.""" - -from collections.abc import Sequence -from typing import Any - -import pytest - -from src.extensions.score_metamodel.checks import traceability_dashboard -from src.extensions.score_metamodel.checks.traceability_dashboard import ( - pie_process_requirements_linked, - pie_requirements_fully_linked, - pie_requirements_with_code_links, - pie_requirements_with_test_links, - set_default_include_external, -) -from src.extensions.score_metamodel.traceability_metrics import ( - compute_traceability_summary, -) - - -def _needs() -> list[dict[str, object]]: - return [ - { - "id": "LOCAL_REQ", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "", - "testlink": "", - "is_external": False, - }, - { - "id": "LOCAL_SYS_REQ", - "type": "sys_req", - "implemented": "YES", - "source_code_link": "", - "testlink": "T_LOCAL", - "is_external": False, - }, - { - "id": "EXT_REQ", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "src/ext.py:10", - "testlink": "T_EXT", - "is_external": True, - }, - ] - - -def test_dashboard_defaults_to_local_only() -> None: - set_default_include_external(False) - - results: list[int] = [] - pie_requirements_with_code_links(_needs(), results, arg1="tool_req") - - summary = compute_traceability_summary( - all_needs=_needs(), - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=False, - ) - - assert results == [1, 0] - assert results == [ - summary["requirements"]["total"] - summary["requirements"]["with_code_link"], - summary["requirements"]["with_code_link"], - ] - - -def test_dashboard_can_include_external_via_default_flag() -> None: - set_default_include_external(True) - - results: list[int] = [] - pie_requirements_with_code_links(_needs(), results, arg1="tool_req") - - summary = compute_traceability_summary( - all_needs=_needs(), - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=True, - ) - - assert results == [1, 1] - assert results == [ - summary["requirements"]["total"] - summary["requirements"]["with_code_link"], - summary["requirements"]["with_code_link"], - ] - - -def test_dashboard_filter_arg_can_override_default() -> None: - set_default_include_external(True) - - results: list[int] = [] - pie_requirements_with_code_links(_needs(), results, arg1="tool_req", arg2="false") - - assert results == [1, 0] - - -def test_requirements_with_test_links_default_local_only() -> None: - set_default_include_external(False) - - results: list[int] = [] - pie_requirements_with_test_links(_needs(), results, arg1="tool_req") - - summary = compute_traceability_summary( - all_needs=_needs(), - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=False, - ) - - assert results == [1, 0] - assert results == [ - summary["requirements"]["total"] - summary["requirements"]["with_test_link"], - summary["requirements"]["with_test_link"], - ] - - -def test_requirements_with_test_links_can_override_include_external() -> None: - set_default_include_external(False) - - results: list[int] = [] - pie_requirements_with_test_links(_needs(), results, arg1="tool_req", arg2="true") - - assert results == [1, 1] - - -def test_requirements_with_test_links_parses_multiple_types() -> None: - set_default_include_external(False) - - results: list[int] = [] - pie_requirements_with_test_links(_needs(), results, arg1="tool_req,sys_req") - - assert results == [1, 1] - - -def test_requirements_fully_linked_uses_shared_summary() -> None: - set_default_include_external(False) - - results: list[int] = [] - pie_requirements_fully_linked(_needs(), results, arg1="tool_req") - - summary = compute_traceability_summary( - all_needs=_needs(), - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=False, - ) - - assert results == [1, 0] - assert results == [ - summary["requirements"]["total"] - summary["requirements"]["fully_linked"], - summary["requirements"]["fully_linked"], - ] - - -def test_requirements_fully_linked_can_include_external() -> None: - set_default_include_external(True) - - results: list[int] = [] - pie_requirements_fully_linked(_needs(), results, arg1="tool_req") - - assert results == [1, 1] - - -def test_process_requirements_linked_uses_stream_a_process_requirement_totals( - monkeypatch: pytest.MonkeyPatch, -) -> None: - captured: dict[str, object] = {} - - def _fake_summary( - all_needs: Sequence[dict[str, Any]], - requirement_types: set[str], - include_not_implemented: bool, - filtered_test_types: set[str], - include_external: bool, - ) -> dict[str, dict[str, int]]: - captured["all_needs"] = all_needs - captured["requirement_types"] = requirement_types - captured["include_not_implemented"] = include_not_implemented - captured["filtered_test_types"] = filtered_test_types - captured["include_external"] = include_external - return { - "requirements": {"total": 99, "linked": 0}, - "process_requirements": {"total": 4, "linked": 3}, - } - - monkeypatch.setattr( - traceability_dashboard, "compute_traceability_summary", _fake_summary - ) - - results: list[int] = [] - pie_process_requirements_linked( - _needs(), results, arg1="tool_req,sys_req", arg2="true" - ) - - assert results == [1, 3] - assert captured["requirement_types"] == {"tool_req", "sys_req"} - assert captured["include_not_implemented"] is True - assert captured["filtered_test_types"] == set() - assert captured["include_external"] is True diff --git a/src/extensions/score_metamodel/tests/test_traceability_metrics.py b/src/extensions/score_metamodel/tests/test_traceability_metrics.py deleted file mode 100644 index 850f064f6..000000000 --- a/src/extensions/score_metamodel/tests/test_traceability_metrics.py +++ /dev/null @@ -1,206 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -"""Unit tests for traceability_metrics include_external handling.""" - -from src.extensions.score_metamodel.traceability_metrics import ( - compute_traceability_summary, - filter_requirements, -) - - -def _needs() -> list[dict[str, object]]: - return [ - { - "id": "LOCAL_REQ", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "src/local.py:1", - "testlink": "tests/test_local.py::test_ok", - "is_external": False, - }, - { - "id": "EXT_REQ", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "src/external.py:9", - "testlink": "tests/test_external.py::test_ok", - "is_external": True, - }, - { - "id": "TC_1", - "type": "testcase", - "partially_verifies": "LOCAL_REQ", - "fully_verifies": "", - "is_external": False, - }, - ] - - -def test_filter_requirements_defaults_to_local_only() -> None: - filtered = filter_requirements( - _needs(), - requirement_types={"tool_req"}, - include_not_implemented=True, - ) - - assert [need["id"] for need in filtered] == ["LOCAL_REQ"] - - -def test_filter_requirements_can_include_external_needs() -> None: - filtered = filter_requirements( - _needs(), - requirement_types={"tool_req"}, - include_not_implemented=True, - include_external=True, - ) - - assert sorted(need["id"] for need in filtered) == ["EXT_REQ", "LOCAL_REQ"] - - -def test_compute_traceability_summary_propagates_include_external() -> None: - summary_local = compute_traceability_summary( - all_needs=_needs(), - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=False, - ) - summary_all = compute_traceability_summary( - all_needs=_needs(), - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=True, - ) - - assert summary_local["include_external"] is False - assert summary_local["requirements"]["total"] == 1 - assert summary_all["include_external"] is True - assert summary_all["requirements"]["total"] == 2 - - -def test_compute_traceability_summary_process_requirements_summary() -> None: - summary = compute_traceability_summary( - all_needs=[ - { - "id": "TOOL_REQ_1", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "src/req.py:10", - "testlink": "tests/test_req.py::test_ok", - "satisfies": "PR_LOCAL_1,OTHER_REQ", - "is_external": False, - }, - { - "id": "TOOL_REQ_2", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "src/req.py:20", - "testlink": "tests/test_req.py::test_ok_2", - "satisfies": ["PR_LOCAL_1", "PR_LOCAL_2"], - "is_external": False, - }, - { - "id": "PR_LOCAL_1", - "type": "process_req", - "is_external": False, - }, - { - "id": "PR_LOCAL_2", - "type": "gd_req", - "is_external": False, - }, - { - "id": "PR_LOCAL_3", - "type": "gd_req", - "is_external": False, - }, - ], - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=False, - ) - - process_requirements = summary["process_requirements"] - - assert process_requirements["total"] == 3 - assert process_requirements["linked_by_tool_requirements"] == 2 - assert process_requirements["linked_by_tool_requirements_pct"] == (2 / 3) * 100 - assert process_requirements["unlinked_ids"] == ["PR_LOCAL_3"] - - -def test_compute_traceability_summary_process_requirements_respects_include_external() -> ( - None -): - all_needs = [ - { - "id": "TOOL_REQ_LOCAL", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "src/local.py:1", - "testlink": "tests/test_local.py::test_ok", - "satisfies": "PR_LOCAL", - "is_external": False, - }, - { - "id": "TOOL_REQ_EXTERNAL", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "src/external.py:1", - "testlink": "tests/test_external.py::test_ok", - "satisfies": "PR_EXTERNAL", - "is_external": True, - }, - { - "id": "PR_LOCAL", - "type": "gd_req", - "is_external": False, - }, - { - "id": "PR_EXTERNAL", - "type": "gd_req", - "is_external": True, - }, - ] - - summary_local = compute_traceability_summary( - all_needs=all_needs, - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=False, - ) - summary_all = compute_traceability_summary( - all_needs=all_needs, - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=True, - ) - - assert summary_local["process_requirements"] == { - "total": 1, - "linked": 1, - "linked_by_tool_requirements": 1, - "linked_by_tool_requirements_pct": 100.0, - "unlinked_ids": [], - } - assert summary_all["process_requirements"] == { - "total": 2, - "linked": 2, - "linked_by_tool_requirements": 2, - "linked_by_tool_requirements_pct": 100.0, - "unlinked_ids": [], - } diff --git a/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py b/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py deleted file mode 100644 index 47af209cb..000000000 --- a/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py +++ /dev/null @@ -1,337 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -"""Tests for Sphinx-side metrics.json generation defaults.""" - -import json -from pathlib import Path -from types import SimpleNamespace -from typing import cast - -import pytest -from sphinx.application import Sphinx - -import src.extensions.score_metamodel.__init__ as metamodel_init - - -class _FakeNeedsData: - def __init__(self, env: object): - self._env = env - - def get_needs_view(self) -> dict[str, dict[str, object]]: - return { - "LOCAL_REQ": { - "id": "LOCAL_REQ", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "", - "testlink": "", - "is_external": False, - }, - "LOCAL_FEAT": { - "id": "LOCAL_FEAT", - "type": "feat_req", - "implemented": "YES", - "source_code_link": "", - "testlink": "", - "is_external": False, - }, - "EXT_REQ": { - "id": "EXT_REQ", - "type": "tool_req", - "implemented": "NO", - "source_code_link": "src/ext.py:1", - "testlink": "", - "is_external": True, - }, - "EXT_FEAT": { - "id": "EXT_FEAT", - "type": "feat_req", - "implemented": "NO", - "source_code_link": "src/ext_feat.py:1", - "testlink": "", - "is_external": True, - }, - "EXT_GD": { - "id": "EXT_GD", - "type": "gd_req", - "implemented": "NO", - "source_code_link": "src/ext_gd.py:1", - "testlink": "", - "is_external": True, - }, - } - - -class _FakeNonReqNeedsData: - def __init__(self, env: object): - self._env = env - - def get_needs_view(self) -> dict[str, dict[str, object]]: - return { - "LOCAL_COMP": { - "id": "LOCAL_COMP", - "type": "comp", - "implemented": "YES", - "source_code_link": "", - "testlink": "", - "is_external": False, - }, - "LOCAL_DOC": { - "id": "LOCAL_DOC", - "type": "document", - "implemented": "YES", - "source_code_link": "", - "testlink": "", - "is_external": False, - }, - } - - -def _app( - tmp_path: Path, - include_external: bool, - requirement_types: str = "", - needs_types: list[dict[str, object]] | None = None, -) -> SimpleNamespace: - discovered_types = needs_types or [ - {"directive": "tool_req", "tags": ["requirement_excl_process"]}, - {"directive": "feat_req", "tags": ["requirement"]}, - {"directive": "workflow", "tags": []}, - ] - return SimpleNamespace( - env=object(), - outdir=str(tmp_path), - config=SimpleNamespace( - score_metamodel_requirement_types=requirement_types, - score_metamodel_include_external_needs=include_external, - needs_types=discovered_types, - ), - ) - - -def test_write_metrics_json_defaults_to_local_only( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) - - metamodel_init._write_metrics_json( - cast(Sphinx, _app(tmp_path, include_external=False)), - None, - ) - - payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) - metrics = payload["metrics_by_type"]["tool_req"] - - assert payload["schema_version"] == "1" - assert metrics["include_not_implemented"] is True - assert metrics["include_external"] is False - assert metrics["requirements"]["total"] == 1 - - -def test_write_metrics_json_can_include_external( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) - - metamodel_init._write_metrics_json( - cast(Sphinx, _app(tmp_path, include_external=True)), - None, - ) - - payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) - metrics = payload["metrics_by_type"]["tool_req"] - - assert metrics["include_external"] is True - assert metrics["requirements"]["total"] == 2 - - -def test_explicit_requirement_types_disable_autodiscovery( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) - - metamodel_init._write_metrics_json( - cast( - Sphinx, - _app( - tmp_path, - include_external=False, - requirement_types="tool_req", - ), - ), - None, - ) - - payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) - assert set(payload["metrics_by_type"].keys()) == {"tool_req"} - - -def test_write_metrics_json_autodiscovers_when_types_unset( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) - - metamodel_init._write_metrics_json( - cast( - Sphinx, - _app(tmp_path, include_external=False, requirement_types=""), - ), - None, - ) - - payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) - assert payload["schema_version"] == "1" - assert set(payload["metrics_by_type"].keys()) == {"feat_req", "tool_req"} - - -def test_autodiscovery_excludes_tagged_types_not_present_in_needs( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) - - metamodel_init._write_metrics_json( - cast( - Sphinx, - _app( - tmp_path, - include_external=False, - requirement_types="", - needs_types=[ - {"directive": "tool_req", "tags": ["requirement_excl_process"]}, - {"directive": "feat_req", "tags": ["requirement"]}, - {"directive": "aou_req", "tags": ["requirement"]}, - ], - ), - ), - None, - ) - - payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) - assert set(payload["metrics_by_type"].keys()) == {"feat_req", "tool_req"} - - -def test_write_metrics_json_empty_when_no_types_configured_or_discovered( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNonReqNeedsData) - - metamodel_init._write_metrics_json( - cast( - Sphinx, - _app( - tmp_path, - include_external=False, - requirement_types="", - needs_types=[{"directive": "workflow", "tags": []}], - ), - ), - None, - ) - - payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) - assert payload["schema_version"] == "1" - assert payload["metrics_by_type"] == {} - - -def test_autodiscovery_without_tagged_requirement_types_is_empty( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) - - metamodel_init._write_metrics_json( - cast( - Sphinx, - _app( - tmp_path, - include_external=False, - requirement_types="", - needs_types=[{"directive": "workflow", "tags": []}], - ), - ), - None, - ) - - payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) - assert payload["metrics_by_type"] == {} - - -def test_autodiscovery_respects_include_external_scope( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) - - metamodel_init._write_metrics_json( - cast( - Sphinx, - _app( - tmp_path, - include_external=True, - requirement_types="", - needs_types=[ - {"directive": "tool_req", "tags": ["requirement_excl_process"]}, - {"directive": "feat_req", "tags": ["requirement"]}, - {"directive": "gd_req", "tags": ["requirement"]}, - ], - ), - ), - None, - ) - - payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) - assert set(payload["metrics_by_type"].keys()) == {"feat_req", "gd_req", "tool_req"} - - -@pytest.mark.parametrize( - ("requirement_types", "include_external", "should_exist", "expected_totals"), - [ - ("tool_req", False, True, {"tool_req": 1}), - ("feat_req,tool_req", False, True, {"feat_req": 1, "tool_req": 1}), - ("", False, True, {"feat_req": 1, "tool_req": 1}), - (" ", False, True, {"feat_req": 1, "tool_req": 1}), - ("tool_req", True, True, {"tool_req": 2}), - ], -) -def test_write_metrics_json_settings_matrix( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, - requirement_types: str, - include_external: bool, - should_exist: bool, - expected_totals: dict[str, int], -) -> None: - monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) - - metamodel_init._write_metrics_json( - cast( - Sphinx, - _app( - tmp_path, - include_external=include_external, - requirement_types=requirement_types, - ), - ), - None, - ) - - metrics_file = tmp_path / "metrics.json" - assert metrics_file.exists() is should_exist - if not should_exist: - return - - payload = json.loads(metrics_file.read_text(encoding="utf-8")) - by_type = payload["metrics_by_type"] - assert set(by_type.keys()) == set(expected_totals.keys()) - - for req_type, expected_total in expected_totals.items(): - assert by_type[req_type]["requirements"]["total"] == expected_total diff --git a/src/extensions/score_metamodel/traceability_metrics.py b/src/extensions/score_metamodel/traceability_metrics.py deleted file mode 100644 index 08e1723e9..000000000 --- a/src/extensions/score_metamodel/traceability_metrics.py +++ /dev/null @@ -1,258 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -# ╓ ╖ -# ║ Some portions generated by Github Copilot ║ -# ╙ ╜ - -"""Shared traceability metric calculations for CI checks and dashboards.""" - -from __future__ import annotations - -from collections.abc import Sequence -from typing import Any - - -def is_non_empty(value: Any) -> bool: - """Return True if value should be treated as present for traceability checks.""" - if isinstance(value, str): - return bool(value.strip()) - return bool(value) - - -def parse_need_id_list(value: Any) -> list[str]: - """Normalize need-id lists encoded as CSV strings or string lists.""" - if value is None: - return [] - if isinstance(value, str): - return [item.strip() for item in value.split(",") if item.strip()] - if isinstance(value, list): - out: list[str] = [] - for item in value: - if isinstance(item, str) and item.strip(): - out.append(item.strip()) - return out - return [] - - -def safe_percent(numerator: int, denominator: int) -> float: - """Return percentage in range [0, 100], treating empty denominator as 100%.""" - if denominator == 0: - return 100.0 - return (numerator / denominator) * 100.0 - - -def filter_requirements( - all_needs: Sequence[Any], - requirement_types: set[str], - include_not_implemented: bool, - include_external: bool = False, -) -> list[Any]: - """Extract requirements by type, implementation state, and origin.""" - requirements: list[dict[str, Any]] = [] - for need in all_needs: - need_type = str(need.get("type", "")).strip() - if need_type not in requirement_types: - continue - if not include_external and need.get("is_external", False): - continue - if not include_not_implemented: - implemented = str(need.get("implemented", "")).upper().strip() - if implemented not in {"YES", "PARTIAL"}: - continue - requirements.append(need) - return requirements - - -def calculate_requirement_metrics( - requirements: Sequence[Any], -) -> dict[str, Any]: - """Calculate requirement traceability statistics for links and completeness.""" - total = len(requirements) - with_code = sum( - 1 for need in requirements if is_non_empty(need.get("source_code_link")) - ) - with_test = sum(1 for need in requirements if is_non_empty(need.get("testlink"))) - fully_linked = sum( - 1 - for need in requirements - if is_non_empty(need.get("source_code_link")) - and is_non_empty(need.get("testlink")) - ) - - missing_code_ids = [ - str(need.get("id", "")) - for need in requirements - if not is_non_empty(need.get("source_code_link")) and need.get("id") - ] - missing_test_ids = [ - str(need.get("id", "")) - for need in requirements - if not is_non_empty(need.get("testlink")) and need.get("id") - ] - not_fully_linked_ids = [ - str(need.get("id", "")) - for need in requirements - if ( - ( - not is_non_empty(need.get("source_code_link")) - or not is_non_empty(need.get("testlink")) - ) - and need.get("id") - ) - ] - - return { - "total": total, - "with_code_link": with_code, - "with_test_link": with_test, - "fully_linked": fully_linked, - "with_code_link_pct": safe_percent(with_code, total), - "with_test_link_pct": safe_percent(with_test, total), - "fully_linked_pct": safe_percent(fully_linked, total), - "missing_code_link_ids": sorted(missing_code_ids), - "missing_test_link_ids": sorted(missing_test_ids), - "not_fully_linked_ids": sorted(not_fully_linked_ids), - } - - -def calculate_test_metrics( - all_needs: Sequence[Any], - requirement_ids: set[str], - filtered_test_types: set[str], -) -> dict[str, Any]: - """Calculate testcase linkage and broken testcase-reference statistics.""" - testcases = [ - need for need in all_needs if str(need.get("type", "")).strip() == "testcase" - ] - if filtered_test_types: - testcases = [ - need - for need in testcases - if str(need.get("test_type", need.get("TestType", ""))).strip() - in filtered_test_types - ] - - tests_total = len(testcases) - tests_linked = 0 - broken_references: list[dict[str, str]] = [] - - for test_need in testcases: - test_id = str(test_need.get("id", "")) - partially = parse_need_id_list( - test_need.get("partially_verifies", test_need.get("PartiallyVerifies")) - ) - fully = parse_need_id_list( - test_need.get("fully_verifies", test_need.get("FullyVerifies")) - ) - refs = partially + fully - if refs: - tests_linked += 1 - for ref in refs: - if ref not in requirement_ids: - broken_references.append({"testcase": test_id, "missing_need": ref}) - - return { - "total": tests_total, - "filtered_test_types": sorted(filtered_test_types), - "linked_to_requirements": tests_linked, - "linked_to_requirements_pct": safe_percent(tests_linked, tests_total), - "broken_references": broken_references, - } - - -def calculate_process_requirement_metrics( - all_needs: Sequence[Any], - include_not_implemented: bool, - include_external: bool, -) -> dict[str, Any]: - """Calculate process-requirement coverage via tool_req ``satisfies`` links.""" - process_requirements = [ - need - for need in all_needs - if str(need.get("type", "")).strip() in {"gd_req", "process_req"} - and (include_external or not need.get("is_external", False)) - ] - process_requirement_ids = { - str(need.get("id", "")).strip() - for need in process_requirements - if need.get("id") - } - - tool_requirements = filter_requirements( - all_needs, - requirement_types={"tool_req"}, - include_not_implemented=include_not_implemented, - include_external=include_external, - ) - - linked_process_requirement_ids: set[str] = set() - for need in tool_requirements: - satisfies_ids = parse_need_id_list(need.get("satisfies", need.get("Satisfies"))) - for ref_id in satisfies_ids: - if ref_id in process_requirement_ids: - linked_process_requirement_ids.add(ref_id) - - total = len(process_requirement_ids) - linked_by_tool_requirements = len(linked_process_requirement_ids) - unlinked_ids = sorted(process_requirement_ids - linked_process_requirement_ids) - - return { - "total": total, - "linked": linked_by_tool_requirements, - "linked_by_tool_requirements": linked_by_tool_requirements, - "linked_by_tool_requirements_pct": safe_percent( - linked_by_tool_requirements, total - ), - "unlinked_ids": unlinked_ids, - } - - -def compute_traceability_summary( - all_needs: Sequence[Any], - requirement_types: set[str], - include_not_implemented: bool, - filtered_test_types: set[str], - include_external: bool = False, -) -> dict[str, Any]: - """Return full CI/dashboard summary using one shared metric implementation.""" - requirements = filter_requirements( - all_needs, - requirement_types=requirement_types, - include_not_implemented=include_not_implemented, - include_external=include_external, - ) - requirement_ids = { - str(need.get("id", "")).strip() for need in requirements if need.get("id") - } - - req_metrics = calculate_requirement_metrics(requirements) - test_metrics = calculate_test_metrics( - all_needs, - requirement_ids=requirement_ids, - filtered_test_types=filtered_test_types, - ) - process_requirement_metrics = calculate_process_requirement_metrics( - all_needs, - include_not_implemented=include_not_implemented, - include_external=include_external, - ) - - return { - "requirement_types": sorted(requirement_types), - "include_not_implemented": include_not_implemented, - "include_external": include_external, - "requirements": req_metrics, - "tests": test_metrics, - "process_requirements": process_requirement_metrics, - } diff --git a/src/extensions/score_metrics/BUILD b/src/extensions/score_metrics/BUILD new file mode 100644 index 000000000..1b9943168 --- /dev/null +++ b/src/extensions/score_metrics/BUILD @@ -0,0 +1,56 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +load("@aspect_rules_py//py:defs.bzl", "py_library") +load("@docs_as_code_hub_env//:requirements.bzl", "all_requirements") +load("//:score_pytest.bzl", "score_pytest") + +filegroup( + name = "all_sources", + srcs = glob( + [ + "*.py", + ], + ), + visibility = ["//visibility:public"], +) + +filegroup( + name = "tests", + srcs = glob( + ["tests/*.py"], + ), +) + +py_library( + name = "score_metrics", + srcs = [":all_sources"], + imports = ["."], + visibility = ["//visibility:public"], + # TODO: Figure out if all requirements are needed or if we can break it down a bit + deps = all_requirements + [ + "@score_docs_as_code//src/helper_lib", + ], +) + +score_pytest( + name = "score_metrics_test", + size = "small", + srcs = glob(["tests/*.py"]), + pytest_config = "//:pyproject.toml", + # All requirements already in the library so no need to have it double + deps = [ + ":score_metrics", + "@score_docs_as_code//src/extensions/score_metamodel", + "@score_docs_as_code//src/extensions/score_metamodel:unit_tests", + ], +) diff --git a/src/extensions/score_metrics/__init__.py b/src/extensions/score_metrics/__init__.py new file mode 100644 index 000000000..b9ef2a1c0 --- /dev/null +++ b/src/extensions/score_metrics/__init__.py @@ -0,0 +1,85 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +import json +from pathlib import Path +from typing import Any + +from score_metrics.traceability_metrics import ( + CALCULATED_METRICS, + calculate_full_need_metrics, +) +from sphinx.application import Sphinx +from sphinx.environment import BuildEnvironment +from sphinx_needs import logging + +logger = logging.get_logger(__name__) + + +def calculate_need_metrics(app: Sphinx, env: BuildEnvironment) -> None: + """ + This is the single source of truth for traceability metrics. It runs + inside the Sphinx build so it has access to all needs (local + external) + and produces the same metrics the dashboard pie charts display. + The traceability_gate reads this file to enforce CI thresholds. + """ + include_external: bool = bool( + getattr(app.config, "score_metamodel_include_external_needs", False) + ) + calculate_full_need_metrics(app=app, include_external=include_external) + + +def _write_metrics_json(app: Sphinx, exception: Any | None) -> None: + """ + Write a schema-v1 metrics.json alongside needs.json in the build output. + """ + if exception: + logger.error(f"Sphinx-Exception at end of build: {exception}") + out_path = Path(app.outdir) / "metrics.json" + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(CALCULATED_METRICS, indent=2), encoding="utf-8") + print(f"Traceability metrics written to: {out_path}") + + +def setup(app: Sphinx) -> dict[str, str | bool]: + app.add_config_value( + "score_metamodel_requirement_types", + "", + rebuild="env", + description=( + "Comma-separated list of need types treated as requirements for " + "traceability metrics. If empty, requirement types are autodiscovered " + "from needs_types tags (requirement, requirement_excl_process)." + ), + ) + + app.add_config_value( + "score_metamodel_include_external_needs", + False, + rebuild="env", + description=( + "When True, include external requirements in dashboard and CI metrics. " + "Default is False so each repo gates only its own needs." + ), + ) + + # Calculates the metrics & sets global var for access + _ = app.connect("env-updated", calculate_need_metrics, priority=600) + + # Writes the metrics to a json in '_build' + _ = app.connect("build-finished", _write_metrics_json, priority=550) + + return { + "version": "0.1", + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/src/extensions/score_metamodel/sphinx_filters.py b/src/extensions/score_metrics/sphinx_filters.py similarity index 66% rename from src/extensions/score_metamodel/sphinx_filters.py rename to src/extensions/score_metrics/sphinx_filters.py index 8fd5fca19..b82ac955c 100644 --- a/src/extensions/score_metamodel/sphinx_filters.py +++ b/src/extensions/score_metrics/sphinx_filters.py @@ -39,6 +39,9 @@ def func(needs: list[NeedItem], results: list[int], **kwargs) -> None: ... from __future__ import annotations +from typing import Any + +from score_metrics.traceability_metrics import CALCULATED_METRICS from sphinx_needs.need_item import NeedItem @@ -55,7 +58,7 @@ def _matches_source_selector(need: NeedItem, selector: str) -> bool: def generic_pie_linked_items( - needs: list[NeedItem], results: list[int], **kwargs: str | int | float + needs: list[NeedItem], results: list[int], **kwargs: Any ) -> None: """Count target IDs by whether they are linked by selected source needs. @@ -63,6 +66,7 @@ def generic_pie_linked_items( selector prefix, matched against source ``type`` and ``id``), and ``arg3`` (link field name, default ``complies``). """ + results.clear() id_prefix = str(kwargs.get("arg1", "")) source_selector = str(kwargs.get("arg2", "")) link_field = str(kwargs.get("arg3", "complies")) @@ -89,7 +93,7 @@ def generic_pie_linked_items( def generic_pie_items_by_tag( - needs: list[NeedItem], results: list[int], **kwargs: str | int | float + needs: list[NeedItem], results: list[int], **kwargs: Any ) -> None: """Count tagged items split by whether selected source needs link them. @@ -97,6 +101,7 @@ def generic_pie_items_by_tag( matched against source ``type`` and ``id``), and ``arg3`` (link field name, default ``complies``). """ + results.clear() tag = str(kwargs.get("arg1", "")) source_selector = str(kwargs.get("arg2", "")) link_field = str(kwargs.get("arg3", "complies")) @@ -123,7 +128,7 @@ def generic_pie_items_by_tag( def generic_pie_items_in_relationships( - needs: list[NeedItem], results: list[int], **kwargs: str | int | float + needs: list[NeedItem], results: list[int], **kwargs: Any ) -> None: """Count items of a given type by how many container items reference them. @@ -142,6 +147,7 @@ def generic_pie_items_in_relationships( Appends to *results*: ``[not_referenced_count, referenced_once_count, referenced_multiple_count]`` """ + results.clear() container_type = str(kwargs.get("arg1", "")) field = str(kwargs.get("arg2", "")) item_type = str(kwargs.get("arg3", "")) @@ -164,3 +170,72 @@ def generic_pie_items_in_relationships( results.append(not_referenced) results.append(referenced_once) results.append(referenced_multiple) + + +def _get_key_values(results: list[int], argument_paths: list[str]): + """Append integer metric values selected by colon-separated paths. + + Each path is resolved against ``CALCULATED_METRICS`` (for example: + ``"overall_metrics:total"``), converted to ``int``, and appended to + ``results``. + """ + metrics_json = CALCULATED_METRICS + for raw_path in argument_paths: + path = raw_path.strip() + if not path: + continue + + current: Any = metrics_json + for key in path.split(":"): + current = current[key] + + results.append(int(current)) + + +def get_metrics_with_overall_total_considered( + needs: list[Any], results: list[int], **kwargs: Any +) -> None: + """Append selected metrics and compute remainder from overall total. + + This function appends ``overall_metrics:total`` first, then appends all + metrics referenced by ``kwargs`` values. Finally, it replaces the first + appended value with the remainder after subtracting all other appended + values. + """ + results.clear() + metrics_json = CALCULATED_METRICS + results.append(int(metrics_json["overall_metrics"]["total"])) + _get_key_values(results, [str(value) for value in kwargs.values()]) + results[0] -= sum(results[1:]) + + +def get_metrics_with_custom_type_total_considered( + needs: list[Any], results: list[int], **kwargs: Any +) -> None: + """Append selected metrics, optionally using a custom total path. + + If the last kwarg value ends with ``":total"``, that path is used as + baseline total and all preceding paths are treated as components; the first + result becomes ``total - sum(components)``. Otherwise, all paths are simply + appended as-is. + """ + # Get the 'total' that was specified as the first value + + results.clear() + values = [str(value) for value in kwargs.values()] + if values[-1].endswith(":total"): + _get_key_values(results, [values[-1]]) # baseline total + _get_key_values(results, values[:-1]) # components + results[0] -= sum(results[1:]) + else: + _get_key_values(results, values) + + +def get_just_metrics(needs: list[Any], results: list[int], **kwargs: Any) -> None: + """Append selected metric values without any total/remainder calculation. + + All kwarg values are interpreted as colon-separated metric paths and + appended to ``results`` in insertion order. + """ + results.clear() + _get_key_values(results, [str(value) for value in kwargs.values()]) diff --git a/src/extensions/score_metrics/tests/test_sphinx_filters.py b/src/extensions/score_metrics/tests/test_sphinx_filters.py new file mode 100644 index 000000000..79ba4f61d --- /dev/null +++ b/src/extensions/score_metrics/tests/test_sphinx_filters.py @@ -0,0 +1,205 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +from typing import cast + +import pytest + +# noqa +import score_metrics.sphinx_filters as sphinx_filters +from score_metrics.sphinx_filters import ( + generic_pie_items_by_tag, + generic_pie_linked_items, +) +from sphinx_needs.need_item import NeedItem + + +def test_generic_pie_linked_items_matches_source_by_id_prefix() -> None: + needs = cast( + list[NeedItem], + [ + {"id": "std_req__iso26262__001", "type": "std_req"}, + # Type intentionally does not match selector prefix, id does. + { + "id": "gd_guidl__xyz", + "type": "guideline", + "complies": ["std_req__iso26262__001"], + }, + ], + ) + + results: list[int] = [] + generic_pie_linked_items( + needs, results, arg1="std_req__iso26262__", arg2="gd_", arg3="complies" + ) + + assert results == [1, 0] + + +def test_generic_pie_items_by_tag_matches_source_by_id_prefix() -> None: + needs = cast( + list[NeedItem], + [ + {"id": "REQ_A", "type": "tool_req", "tags": ["aspice40_man5"]}, + {"id": "REQ_B", "type": "tool_req", "tags": ["aspice40_man5"]}, + # Type intentionally does not match selector prefix, id does. + { + "id": "gd_req__abc", + "type": "process_requirement", + "complies": ["REQ_A"], + }, + ], + ) + + results: list[int] = [] + generic_pie_items_by_tag( + needs, results, arg1="aspice40_man5", arg2="gd_", arg3="complies" + ) + + assert results == [1, 1] + + +EXAMPLE_METRICS = { + "schema_version": "1", + "generated_by": "sphinx_build", + "overall_metrics": { + "total": 61, + "with_code_link": 46, + "with_test_link": 3, + "fully_linked": 2, + "with_code_link_pct": 75.40983606557377, + "with_test_link_pct": 4.918032786885246, + "fully_linked_pct": 3.278688524590164, + }, + "metrics_by_type": { + "tool_req": { + "total": 61, + "with_code_link": 46, + "with_test_link": 3, + "fully_linked": 2, + } + }, + "tests": { + "total": 208, + "linked_to_requirements": 16, + "linked_to_requirements_pct": 7.6923076923076925, + "broken_references": [], + }, +} + + +@pytest.fixture(autouse=True) +def reset_global_metrics(): + """Reset global CALCULATED_METRICS before and after each test.""" + sphinx_filters.CALCULATED_METRICS = {} + yield + sphinx_filters.CALCULATED_METRICS = {} + + +def test_get_key_values_raises_key_error_when_global_is_empty() -> None: + """It raises KeyError if CALCULATED_METRICS is still empty.""" + results: list[int] = [] + with pytest.raises(KeyError): + sphinx_filters._get_key_values(results, ["overall_metrics:total"]) + + +def test_get_key_values_appends_values_when_metrics_loaded() -> None: + """It appends resolved integer values once metrics data is loaded.""" + sphinx_filters.CALCULATED_METRICS = EXAMPLE_METRICS + results: list[int] = [] + + sphinx_filters._get_key_values( + results, + [ + "overall_metrics:total", + "overall_metrics:with_code_link", + "tests:linked_to_requirements", + ], + ) + + assert results == [61, 46, 16] + + +def test_get_metrics_with_overall_total_considered_when_metrics_loaded() -> None: + """It computes remainder from overall total and selected metrics.""" + sphinx_filters.CALCULATED_METRICS = EXAMPLE_METRICS + results: list[int] = [] + + sphinx_filters.get_metrics_with_overall_total_considered( + needs=[], + results=results, + code="overall_metrics:with_code_link", + test="overall_metrics:with_test_link", + fully="overall_metrics:fully_linked", + ) + + assert results == [10, 46, 3, 2] + + +def test_get_metrics_with_custom_type_total_considered_with_total_suffix() -> None: + """It uses trailing ':total' as baseline and computes remainder.""" + sphinx_filters.CALCULATED_METRICS = EXAMPLE_METRICS + results: list[int] = [] + + sphinx_filters.get_metrics_with_custom_type_total_considered( + needs=[], + results=results, + code="metrics_by_type:tool_req:with_code_link", + test="metrics_by_type:tool_req:with_test_link", + total="metrics_by_type:tool_req:total", + ) + + assert results == [12, 46, 3] + + +def test_get_metrics_with_custom_type_total_considered_without_total_suffix() -> None: + """It appends values directly when trailing ':total' is not provided.""" + sphinx_filters.CALCULATED_METRICS = EXAMPLE_METRICS + results: list[int] = [] + + sphinx_filters.get_metrics_with_custom_type_total_considered( + needs=[], + results=results, + code="metrics_by_type:tool_req:with_code_link", + test="metrics_by_type:tool_req:with_test_link", + ) + + assert results == [46, 3] + + +def test_get_just_metrics_appends_values_when_metrics_loaded() -> None: + """It appends selected values without remainder logic.""" + sphinx_filters.CALCULATED_METRICS = EXAMPLE_METRICS + results: list[int] = [] + + sphinx_filters.get_just_metrics( + needs=[], + results=results, + total="overall_metrics:total", + linked_tests="tests:linked_to_requirements", + ) + + assert results == [61, 16] + + +def test_get_metrics_with_custom_type_total_considered_empty_kwargs_raises_index_error() -> ( + None +): + """Current behavior: empty kwargs raises IndexError.""" + sphinx_filters.CALCULATED_METRICS = EXAMPLE_METRICS + results: list[int] = [] + + with pytest.raises(IndexError): + sphinx_filters.get_metrics_with_custom_type_total_considered( + needs=[], results=results + ) diff --git a/src/extensions/score_metrics/tests/test_traceability_metrics.py b/src/extensions/score_metrics/tests/test_traceability_metrics.py new file mode 100644 index 000000000..59f2e1399 --- /dev/null +++ b/src/extensions/score_metrics/tests/test_traceability_metrics.py @@ -0,0 +1,255 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +from typing import Any, cast + +import pytest +import score_metrics.traceability_metrics as metrics +from score_metamodel import ScoreNeedType +from score_metamodel.tests import need as test_need +from sphinx_needs.data import NeedsView +from sphinx_needs.need_item import NeedItem + +from score_pytest.attribute_plugin import add_test_properties + + +@add_test_properties( + partially_verifies=["tool_req__docs_test_linkage_metrics"], + test_type="requirements-based", + derivation_technique="equivalence-classes", +) +def test_get_need_types_by_tags_returns_matching_directives_only() -> None: + """Return directives for need types sharing at least one selected tag.""" + needs: list[ScoreNeedType] = [ + { + "title": "Test Type 1", + "prefix": "TR", + "tags": ["requirement"], + "parts": 1, + "directive": "tool_req", + "mandatory_options": { + "id": "^tool_req__.*$", + "some_required_option": "^some_value__.*$", + }, + "optional_options": {}, + "mandatory_links": {}, + "optional_links": {}, + }, + { + "title": "Test Type Verification", + "prefix": "TRV", + "tags": ["verification"], + "parts": 1, + "directive": "tool_req_ver", + "mandatory_options": { + "id": "^tool_req__.*$", + "some_required_option": "^some_value__.*$", + }, + "optional_options": {}, + "mandatory_links": {}, + "optional_links": {}, + }, + { + "title": "Test Type Extra", + "prefix": "TRE", + "tags": ["extra"], + "parts": 1, + "directive": "tool_req_ext", + "mandatory_options": { + "id": "^tool_req__.*$", + "some_required_option": "^some_value__.*$", + }, + "optional_options": {}, + "mandatory_links": {}, + "optional_links": {}, + }, + ] + result = metrics.get_need_types_by_tags(needs, {"verification", "requirement"}) + assert result == ["tool_req", "tool_req_ver"] + + +@add_test_properties( + partially_verifies=["tool_req__docs_test_linkage_metrics"], + test_type="requirements-based", + derivation_technique="equivalence-classes", +) +def test_get_need_types_by_tags_returns_empty_on_non_match() -> None: + """Test if function correctly returns empty list if none of the things match""" + needs: list[ScoreNeedType] = [ + { + "title": "Test Type 1", + "prefix": "TR", + "tags": ["requirement"], + "parts": 1, + "directive": "tool_req", + "mandatory_options": { + "id": "^tool_req__.*$", + "some_required_option": "^some_value__.*$", + }, + "optional_options": {}, + "mandatory_links": {}, + "optional_links": {}, + }, + ] + result = metrics.get_need_types_by_tags(needs, {"requirements_without_proccess"}) + assert result == [] + + +@add_test_properties( + partially_verifies=["tool_req__docs_test_linkage_metrics"], + test_type="interface-test", + derivation_technique="boundary-values", +) +@pytest.mark.parametrize( + ("value", "expected"), + [ + (" ", False), + ("text", True), + ([], False), + ([1], True), + (0, False), + (1, True), + (None, False), + ], +) +def test_is_non_empty_string_and_non_string_behavior( + value: Any, expected: bool +) -> None: + """Treat blank strings as empty and all other values by truthiness.""" + # Unsure if we should test this as this is python behaviour, but might as well + assert metrics.is_non_empty(value) is expected + + +@add_test_properties( + partially_verifies=["tool_req__docs_test_linkage_metrics"], + test_type="requirements-based", + derivation_technique="boundary-values", +) +@pytest.mark.parametrize( + ("value1", "value2", "expected"), + [(3, 0, 100.0), (1, 4, 25.0)], +) +def test_safe_percent_zero(value1: int, value2: int, expected: float) -> None: + """Check if 100 is returned for empty denominator & normal behaviour""" + assert metrics.safe_percent(value1, value2) == expected + + +@add_test_properties( + partially_verifies=["tool_req__docs_test_linkage_metrics"], + test_type="requirements-based", + derivation_technique="requirements-analysis", +) +def test_calculate_requirement_metrics_counts_links_and_missing_ids() -> None: + """Count code/test links and derive missing identifier lists.""" + current_requirement_needs: list[NeedItem] = [ + test_need(id="REQ_1", source_code_link="src/main.c", testlink="TC_1"), + test_need(id="REQ_2", source_code_link=" ", testlink="TC_2"), + test_need(id="REQ_3", source_code_link="src/lib.rs", testlink=""), + test_need(id="REQ_4", source_code_link="", testlink=""), + ] + + result = metrics.calculate_requirement_metrics(current_requirement_needs) + + assert result["total"] == 4 + assert result["with_code_link"] == 2 + assert result["with_test_link"] == 2 + assert result["fully_linked"] == 1 + + assert result["with_code_link_pct"] == 50.0 + assert result["with_test_link_pct"] == 50.0 + assert result["fully_linked_pct"] == 25.0 + + +@add_test_properties( + partially_verifies=["tool_req__docs_test_linkage_metrics"], + test_type="requirements-based", + derivation_technique="equivalence-classes", +) +def test_calculate_requirement_metrics_non_fully_linked() -> None: + """Count code/test links and derive missing identifier lists.""" + current_requirement_needs: list[NeedItem] = [ + test_need(id="REQ_1", source_code_link="src/main.c", testlink=""), + test_need(id="REQ_2", source_code_link=" ", testlink="TC_2"), + test_need(id="REQ_3", source_code_link="src/lib.rs", testlink=""), + test_need(id="REQ_4", source_code_link="", testlink=""), + ] + + result = metrics.calculate_requirement_metrics(current_requirement_needs) + + assert result["total"] == 4 + assert result["with_code_link"] == 2 + assert result["with_test_link"] == 1 + assert result["fully_linked"] == 0 + + assert result["with_code_link_pct"] == 50.0 + assert result["with_test_link_pct"] == 25.0 + assert result["fully_linked_pct"] == 0.0 + + +@add_test_properties( + partially_verifies=["tool_req__docs_test_linkage_metrics"], + test_type="requirements-based", + derivation_technique="equivalence-classes", +) +def test_calculate_requirement_metrics_non_fully_linked_2() -> None: + """Count code/test links and derive missing identifier lists.""" + current_requirement_needs: list[NeedItem] = [ + test_need(id="REQ_1", source_code_link="", testlink=""), + test_need(id="REQ_2", source_code_link="", testlink="TC_2"), + test_need(id="REQ_3", source_code_link="src/main.c", testlink=""), + test_need(id="REQ_4", source_code_link=" ", testlink="TC_4"), + ] + + result = metrics.calculate_requirement_metrics(current_requirement_needs) + + assert result["total"] == 4 + assert result["with_code_link"] == 1 + assert result["with_test_link"] == 2 + assert result["fully_linked"] == 0 + + assert result["with_code_link_pct"] == 25.0 + assert result["with_test_link_pct"] == 50.0 + assert result["fully_linked_pct"] == 0.0 + + +@add_test_properties( + partially_verifies=["tool_req__docs_test_linkage_metrics"], + test_type="interface-test", + derivation_technique="design-analysis", +) +def test_calculate_test_metrics_counts_linked_tests_and_broken_refs() -> None: + """Count linked testcases and list references to missing needs.""" + test_needs: list[NeedItem] = [ + test_need(id="TC_1", partially_verifies=["REQ_1"], fully_verifies=[]), + test_need( + id="TC_2", partially_verifies=[], fully_verifies=["REQ_2", "REQ_404"] + ), + test_need(id="TC_3", partially_verifies=[], fully_verifies=[]), + ] + + all_needs = cast( + NeedsView, + { + "REQ_1": test_need(id="REQ_1"), + "REQ_2": test_need(id="REQ_2"), + }, + ) + + result = metrics.calculate_test_metrics(test_needs, all_needs) + + assert result["total"] == 3 + assert result["linked_to_requirements"] == 2 + assert result["linked_to_requirements_pct"] == pytest.approx(66.6666666667) + assert result["broken_references"] == [ + {"testcase": "TC_2", "missing_need": "REQ_404"} + ] diff --git a/src/extensions/score_metrics/traceability_metrics.py b/src/extensions/score_metrics/traceability_metrics.py new file mode 100644 index 000000000..2ce09db8b --- /dev/null +++ b/src/extensions/score_metrics/traceability_metrics.py @@ -0,0 +1,195 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +# ╓ ╖ +# ║ Some portions generated by Github Copilot ║ +# ╙ ╜ + +"""Shared traceability metric calculations for CI checks and dashboards.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from score_metamodel import ScoreNeedType +from score_metamodel.yaml_parser import load_metamodel_data +from sphinx.application import Sphinx +from sphinx_needs.data import NeedsView, SphinxNeedsData +from sphinx_needs.need_item import NeedItem + +CALCULATED_METRICS = {} + + +def get_need_types_by_tags(needs: list[ScoreNeedType], tags: set[str]) -> list[str]: + """ + Takes a list of 'ScoreNeedTypes' and filters out any that have one or more of the + specified tags given. + """ + found_need_types: list[str] = [] + for need_type in needs: + found_tag_set = set(need_type["tags"]) + if tags.intersection(found_tag_set): + found_need_types.append(need_type["directive"]) + return found_need_types + + +def is_non_empty(value: Any) -> bool: + """Return True if value should be treated as present for traceability checks.""" + if isinstance(value, str): + return bool(value.strip()) + return bool(value) + + +def safe_percent(numerator: int, denominator: int) -> float: + """Return percentage in range [0, 100], treating empty denominator as 100%.""" + if denominator == 0: + return 100.0 + return (numerator / denominator) * 100.0 + + +def calculate_requirement_metrics( + current_requirement_needs: list[NeedItem], +) -> dict[str, Any]: + """Calculate requirement traceability statistics for links and completeness.""" + total = len(current_requirement_needs) + reqs_with_code_link = 0 + reqs_with_test_link = 0 + reqs_fully_linked = 0 + + for need in current_requirement_needs: + # Sourcecode link check + if is_non_empty(need.get("source_code_link")): + reqs_with_code_link += 1 + + # Testlink check + if is_non_empty(need.get("testlink")): + reqs_with_test_link += 1 + # Negative check (both missing) + if is_non_empty(need.get("testlink")) and is_non_empty( + need.get("source_code_link") + ): + reqs_fully_linked += 1 + + return { + "total": total, + "with_code_link": reqs_with_code_link, + "with_test_link": reqs_with_test_link, + "fully_linked": reqs_fully_linked, + "with_code_link_pct": safe_percent(reqs_with_code_link, total), + "with_test_link_pct": safe_percent(reqs_with_test_link, total), + "fully_linked_pct": safe_percent(reqs_fully_linked, total), + } + + +def calculate_test_metrics( + test_needs: list[NeedItem], all_needs: NeedsView +) -> dict[str, Any]: + """Calculate testcase linkage and broken testcase-reference statistics.""" + tests_total = len(test_needs) + tests_linked = 0 + broken_references: list[dict[str, str]] = [] + + for test_need in test_needs: + test_id = str(test_need.get("id")) + partially: list[str] = test_need.get("partially_verifies", []) + fully: list[str] = test_need.get("fully_verifies", []) + refs = set(partially + fully) + if refs: + tests_linked += 1 + for ref in refs: + if ref not in all_needs: + broken_references.append({"testcase": test_id, "missing_need": ref}) + + return { + "total": tests_total, + "linked_to_requirements": tests_linked, + "linked_to_requirements_pct": safe_percent(tests_linked, tests_total), + "broken_references": broken_references, + } + + +def calculate_full_need_metrics(app: Sphinx, include_external: bool): + """ + Calculate all tracked metrics for requirements and tests. + Will save the result in a global variable 'CALCULATED_METRICS' + """ + # ───────────────[ Getting configuration values ]─────────────── + global CALCULATED_METRICS + all_needs: NeedsView = SphinxNeedsData(app.env).get_needs_view() + + raw_metamodel_path = app.config.score_metamodel_yaml + override_path = Path(raw_metamodel_path) if raw_metamodel_path else None + metamodel = load_metamodel_data(override_path) + + # We either get the types that should be considered from the configuration + # If none are specified the need types declared in the Metamodel with + # the tags 'requirement' are taken + raw = getattr(app.config, "score_metamodel_requirement_types", "").strip() + filter_reqs = [t.strip() for t in raw.split(",") if t.strip()] + if not filter_reqs: + filter_reqs = get_need_types_by_tags(metamodel.needs_types, {"requirement"}) + # ──────────────────[ Calculate Test Metrics ]────────────────── + + test_needs = list(all_needs.filter_types(["testcase"]).values()) + test_metrics = calculate_test_metrics(test_needs, all_needs) + + metrics_by_type: dict[str, Any] = {} + + # Metrics accumulated over all requirements types + overall_metrics: dict[str, Any] = { + "total": 0, + "with_code_link": 0, + "with_test_link": 0, + "fully_linked": 0, + } + + # ─────[ Calculating Metrics for each requirement_type ]─── + for req_type in sorted(filter_reqs): + if include_external: + needs_of_req_type = all_needs.filter_types([req_type]) + else: + needs_of_req_type = all_needs.filter_types([req_type]).filter_is_external( + False + ) + # We do not care if there is no requirements of this type. + if not list(needs_of_req_type.values()): + continue + req_metrics = calculate_requirement_metrics(list(needs_of_req_type.values())) + overall_metrics["total"] += req_metrics["total"] + overall_metrics["with_code_link"] += req_metrics["with_code_link"] + overall_metrics["with_test_link"] += req_metrics["with_test_link"] + overall_metrics["fully_linked"] += req_metrics["fully_linked"] + metrics_by_type[req_type] = req_metrics + # Calculating % of each category for the overall metrics + overall_metrics["with_code_link_pct"] = safe_percent( + overall_metrics["with_code_link"], overall_metrics["total"] + ) + overall_metrics["with_test_link_pct"] = safe_percent( + overall_metrics["with_test_link"], overall_metrics["total"] + ) + overall_metrics["fully_linked_pct"] = safe_percent( + overall_metrics["fully_linked"], overall_metrics["total"] + ) + + output: dict[str, Any] = { + "schema_version": "1", + "generated_by": "sphinx_build", + "overall_metrics": overall_metrics, + "metrics_by_type": metrics_by_type, + "tests": test_metrics, + } + # Save the metrics in a Global Variable to enable access from other parts. + # Not a great solution but it is needed, as needpie filter functions for example + # can not access 'app'. + CALCULATED_METRICS.update(output) diff --git a/src/extensions/score_source_code_linker/tests/test_xml_parser.py b/src/extensions/score_source_code_linker/tests/test_xml_parser.py index 1bb770f2c..83163a557 100644 --- a/src/extensions/score_source_code_linker/tests/test_xml_parser.py +++ b/src/extensions/score_source_code_linker/tests/test_xml_parser.py @@ -301,7 +301,9 @@ def test_read_test_xml_file( # No properties at all => Should not be a 'valid' testlink needs2, no_props2, missing_props2 = xml_parser.read_test_xml_file(dir2 / "test.xml") - assert needs2 == [] + assert isinstance(needs2, list) and len(needs2) == 1 + tcneed2 = needs2[0] + assert isinstance(tcneed2, DataOfTestCase) assert no_props2 == ["tc_no_props"] assert missing_props2 == [] @@ -315,7 +317,9 @@ def test_read_test_xml_file( # Missing some properties => Should not be a 'valid' testlink needs4, no_props4, missing_props4 = xml_parser.read_test_xml_file(dir4 / "test.xml") - assert needs4 == [] + assert isinstance(needs4, list) and len(needs4) == 1 + tcneed4 = needs4[0] + assert isinstance(tcneed4, DataOfTestCase) assert no_props4 == [] assert missing_props4 == ["tc_with_missing_props"] diff --git a/src/extensions/score_source_code_linker/xml_parser.py b/src/extensions/score_source_code_linker/xml_parser.py index 302d87469..9a8bcac63 100644 --- a/src/extensions/score_source_code_linker/xml_parser.py +++ b/src/extensions/score_source_code_linker/xml_parser.py @@ -219,9 +219,10 @@ def read_test_xml_file(file: Path) -> tuple[list[DataOfTestCase], list[str], lis properties_element = testcase.find("properties") # HINT: This list is hard coded here, might not be ideal to have that in the # long run. - if properties_element is None: - non_prop_tests.append(testname) - continue + # Even if we have no properties we still want to create test_case needs + # if properties_element is None: + # non_prop_tests.append(testname) + # continue # ╓ ╖ # ║ Disabled Temporarily ║ @@ -236,12 +237,16 @@ def read_test_xml_file(file: Path) -> tuple[list[DataOfTestCase], list[str], lis # I think it should be possible to save the 'from_dict' operation # If the is_valid method would return 'False' anyway. # I just can't think of it right now, leaving this for future me - case_properties = parse_properties(case_properties, properties_element) - case_properties.update(md) - test_case = DataOfTestCase.from_dict(case_properties) - if not test_case.is_valid(): - missing_prop_tests.append(testname) - continue + if properties_element is not None: + case_properties = parse_properties(case_properties, properties_element) + case_properties.update(md) + + test_case = DataOfTestCase.from_dict(case_properties) + if not test_case.is_valid(): + missing_prop_tests.append(testname) + else: + non_prop_tests.append(testname) + test_case = DataOfTestCase.from_dict(case_properties) test_case_needs.append(test_case) return test_case_needs, non_prop_tests, missing_prop_tests @@ -332,23 +337,21 @@ def short_hash(input_str: str, length: int = 5) -> str: def construct_and_add_need(app: Sphinx, tn: DataOfTestCase): - # Asserting worldview to a peace Language Server - # And ensure non crashing due to non string concatenation - # Everything but 'result_text', - # and either 'Fully' or 'PartiallyVerifies' should not be None here - assert tn.file is not None + # We will now allow file to be empty in case of non fully fleshed out testcases + file = tn.file if tn.file is not None else "" assert tn.name is not None + name = tn.name external_url = "" if tn.repo_name is None or tn.hash is None or tn.url is None: logger.info( "Creating testcase need with fallback URL due to incomplete repo metadata: " - f"name={tn.name}, file={tn.file}, repo_name={tn.repo_name}, " + f"name={name}, file={file}, repo_name={tn.repo_name}, " f"hash={tn.hash}, url={tn.url}", type="score_source_code_linker", ) line = tn.line if tn.line is not None else 1 external_url = ( - f"https://github.com/placeholder/placeholder/blob/unknown/{tn.file}#L{line}" + f"https://github.com/placeholder/placeholder/blob/unknown/{file}#L{line}" ) else: # Have to build metadata here for the gh link func @@ -359,10 +362,10 @@ def construct_and_add_need(app: Sphinx, tn: DataOfTestCase): _ = add_external_need( app=app, need_type="testcase", - title=tn.name, + title=name, tags="TEST", - id=f"testcase__{tn.name}_{short_hash(tn.file + tn.name)}", - name=tn.name, + id=f"testcase__{name}_{short_hash(file + name)}", + name=name, external_url=external_url, fully_verifies=tn.FullyVerifies if tn.FullyVerifies is not None else "", partially_verifies=tn.PartiallyVerifies @@ -370,7 +373,7 @@ def construct_and_add_need(app: Sphinx, tn: DataOfTestCase): else "", test_type=tn.TestType, derivation_technique=tn.DerivationTechnique, - file=tn.file, + file=file, line=tn.line, result=tn.result, # We just want the 'failed' or whatever result_text=tn.result_text if tn.result_text else "", diff --git a/src/extensions/score_sphinx_bundle/BUILD b/src/extensions/score_sphinx_bundle/BUILD index 832e35a61..c34d2907e 100644 --- a/src/extensions/score_sphinx_bundle/BUILD +++ b/src/extensions/score_sphinx_bundle/BUILD @@ -29,6 +29,7 @@ py_library( "@score_docs_as_code//src/extensions/score_layout", "@score_docs_as_code//src/extensions/score_metamodel", "@score_docs_as_code//src/extensions/score_source_code_linker", + "@score_docs_as_code//src/extensions/score_metrics", "@score_docs_as_code//src/extensions/score_sync_toml", "@score_docs_as_code//src/helper_lib", ], diff --git a/src/extensions/score_sphinx_bundle/__init__.py b/src/extensions/score_sphinx_bundle/__init__.py index 02dc06669..ded7ea095 100644 --- a/src/extensions/score_sphinx_bundle/__init__.py +++ b/src/extensions/score_sphinx_bundle/__init__.py @@ -31,6 +31,7 @@ "sphinxcontrib.mermaid", "needs_config_writer", "score_sync_toml", + "score_metrics", ] From 734f1692b15e090a9ec145b426f9af24c77a9698 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Fri, 5 Jun 2026 15:57:29 +0200 Subject: [PATCH 06/11] Fix: Upgrade of tests & schema --- scripts_bazel/tests/traceability_gate_test.py | 463 ++++++++++++------ scripts_bazel/traceability_gate.py | 105 +++- .../traceability_metrics_schema.json | 145 ++---- src/extensions/score_metamodel/__init__.py | 4 +- src/extensions/score_metrics/__init__.py | 2 +- .../score_metrics/sphinx_filters.py | 41 +- .../score_metrics/traceability_metrics.py | 2 +- 7 files changed, 466 insertions(+), 296 deletions(-) diff --git a/scripts_bazel/tests/traceability_gate_test.py b/scripts_bazel/tests/traceability_gate_test.py index 306cafc52..20d7e607e 100644 --- a/scripts_bazel/tests/traceability_gate_test.py +++ b/scripts_bazel/tests/traceability_gate_test.py @@ -15,220 +15,397 @@ # ║ Some portions generated by Github Copilot ║ # ╙ ╜ -"""Tests for traceability_gate.py.""" +""" +Tests for the traceability gate script using metrics schema v2, +including overall statistics coverage. +""" + +from __future__ import annotations import json -import subprocess -import sys from pathlib import Path - -_MY_PATH = Path(__file__).parent - -_GATE_SCRIPT = _MY_PATH.parent / "traceability_gate.py" +from typing import Any +import pytest + +import scripts_bazel.traceability_gate as gate + + +def _derive_overall_metrics(metrics_by_type: dict[str, Any]) -> dict[str, Any]: + """ + Derive overall requirement metrics from all need types. + + Args: + metrics_by_type: Per-type requirement metrics map. + + Returns: + Aggregated requirement totals and percentages. + """ + total = 0 + with_code_link = 0 + with_test_link = 0 + fully_linked = 0 + + for item in metrics_by_type.values(): + total += int(item["total"]) + with_code_link += int(item["with_code_link"]) + with_test_link += int(item["with_test_link"]) + fully_linked += int(item["fully_linked"]) + + if total == 0: + with_code_link_pct = 0.0 + with_test_link_pct = 0.0 + fully_linked_pct = 0.0 + else: + with_code_link_pct = (with_code_link / total) * 100.0 + with_test_link_pct = (with_test_link / total) * 100.0 + fully_linked_pct = (fully_linked / total) * 100.0 + + return { + "total": total, + "with_code_link": with_code_link, + "with_test_link": with_test_link, + "fully_linked": fully_linked, + "with_code_link_pct": with_code_link_pct, + "with_test_link_pct": with_test_link_pct, + "fully_linked_pct": fully_linked_pct, + } -def _write_metrics_json(tmp_path: Path, metrics_by_type: dict | None = None) -> Path: - """Write a schema-v1 metrics JSON and return its path.""" +def _write_metrics_json( + tmp_path: Path, + metrics_by_type: dict[str, Any] | None = None, + tests: dict[str, Any] | None = None, + schema_version: str = "2", +) -> Path: + """ + Write a metrics.json file in schema v2 format. + + Args: + tmp_path: Temporary directory for test artifacts. + metrics_by_type: Optional per-type requirement metrics. + tests: Optional global test linkage metrics. + schema_version: Schema version string. + + Returns: + Path to the written JSON file. + """ if metrics_by_type is None: metrics_by_type = { "tool_req": { "include_not_implemented": False, "include_external": False, - "requirements": { - "total": 4, - "with_code_link": 3, - "with_test_link": 2, - "fully_linked": 2, - "with_code_link_pct": 75.0, - "with_test_link_pct": 50.0, - "fully_linked_pct": 50.0, - "missing_code_link_ids": ["REQ_4"], - "missing_test_link_ids": ["REQ_3", "REQ_4"], - "not_fully_linked_ids": ["REQ_3", "REQ_4"], - }, - "tests": { - "total": 3, - "filtered_test_types": [], - "linked_to_requirements": 2, - "linked_to_requirements_pct": 66.67, - "broken_references": [], - }, + "total": 4, + "with_code_link": 3, + "with_test_link": 2, + "fully_linked": 2, + "with_code_link_pct": 75.0, + "with_test_link_pct": 50.0, + "fully_linked_pct": 50.0, } } - payload = { - "schema_version": "1", + + if tests is None: + tests = { + "total": 3, + "linked_to_requirements": 2, + "linked_to_requirements_pct": 66.67, + "broken_references": [], + } + + payload: dict[str, Any] = { + "schema_version": schema_version, "generated_by": "sphinx_build", + "overall_metrics": _derive_overall_metrics(metrics_by_type), "metrics_by_type": metrics_by_type, + "tests": tests, } + out = tmp_path / "metrics.json" out.write_text(json.dumps(payload), encoding="utf-8") return out -def _run_gate(metrics_json: Path, extra_args: list[str]) -> subprocess.CompletedProcess: - return subprocess.run( - [sys.executable, _GATE_SCRIPT, "--metrics-json", str(metrics_json)] - + extra_args, - capture_output=True, - text=True, - ) +def _run_main( + monkeypatch: pytest.MonkeyPatch, + args: list[str], +) -> int: + """ + Run ``gate.main()`` with patched ``sys.argv``. + Args: + monkeypatch: Pytest monkeypatch fixture. + args: CLI arguments excluding program name. -def test_gate_passes_when_thresholds_met(tmp_path: Path) -> None: + Returns: + Exit code returned by ``gate.main()``. + """ + monkeypatch.setattr("sys.argv", ["traceability_gate.py", *args]) + return gate.main() + + +def test_gate_passes_when_thresholds_met( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: metrics_json = _write_metrics_json(tmp_path) - result = _run_gate( - metrics_json, - ["--min-req-code", "70", "--min-req-test", "50", "--min-tests-linked", "60"], + rc = _run_main( + monkeypatch, + [ + "--metrics-json", + str(metrics_json), + "--min-req-code", + "70", + "--min-req-test", + "50", + "--min-tests-linked", + "60", + ], ) + captured = capsys.readouterr() - assert result.returncode == 0 - assert "Threshold check passed." in result.stdout + assert rc == 0 + assert "Threshold check passed." in captured.out -def test_gate_fails_when_threshold_not_met(tmp_path: Path) -> None: +def test_gate_fails_when_threshold_not_met( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: metrics_json = _write_metrics_json(tmp_path) - result = _run_gate( - metrics_json, - ["--min-req-code", "100"], + rc = _run_main( + monkeypatch, + ["--metrics-json", str(metrics_json), "--min-req-code", "100"], ) + captured = capsys.readouterr() - assert result.returncode == 2 - assert "Threshold check failed:" in result.stdout - assert "[tool_req] requirements with code links" in result.stdout + assert rc == 2 + assert "Threshold check failed:" in captured.out + assert "[tool_req] requirements with code links" in captured.out -def test_gate_require_all_links_fails(tmp_path: Path) -> None: +def test_gate_require_all_links_fails( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: metrics_json = _write_metrics_json(tmp_path) - result = _run_gate(metrics_json, ["--require-all-links"]) + rc = _run_main( + monkeypatch, + ["--metrics-json", str(metrics_json), "--require-all-links"], + ) + captured = capsys.readouterr() - assert result.returncode == 2 - assert "Threshold check failed:" in result.stdout + assert rc == 2 + assert "Threshold check failed:" in captured.out -def test_gate_fail_on_broken_refs(tmp_path: Path) -> None: +def test_gate_fail_on_broken_test_refs( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: metrics_by_type = { "tool_req": { "include_not_implemented": False, "include_external": False, - "requirements": { - "total": 1, - "with_code_link": 1, - "with_test_link": 1, - "fully_linked": 1, - "with_code_link_pct": 100.0, - "with_test_link_pct": 100.0, - "fully_linked_pct": 100.0, - "missing_code_link_ids": [], - "missing_test_link_ids": [], - "not_fully_linked_ids": [], - }, - "tests": { - "total": 2, - "filtered_test_types": [], - "linked_to_requirements": 2, - "linked_to_requirements_pct": 100.0, - "broken_references": [ - {"testcase": "TC_X", "missing_need": "REQ_UNKNOWN"} - ], - }, + "total": 1, + "with_code_link": 1, + "with_test_link": 1, + "fully_linked": 1, + "with_code_link_pct": 100.0, + "with_test_link_pct": 100.0, + "fully_linked_pct": 100.0, } } - metrics_json = _write_metrics_json(tmp_path, metrics_by_type) + tests = { + "total": 2, + "linked_to_requirements": 2, + "linked_to_requirements_pct": 100.0, + "broken_references": [{"testcase": "TC_X", "missing_need": "REQ_UNKNOWN"}], + } + metrics_json = _write_metrics_json( + tmp_path, metrics_by_type=metrics_by_type, tests=tests + ) - result = _run_gate(metrics_json, ["--fail-on-broken-test-refs"]) + rc = _run_main( + monkeypatch, + ["--metrics-json", str(metrics_json), "--fail-on-broken-test-refs"], + ) + captured = capsys.readouterr() - assert result.returncode == 2 - assert "broken testcase references found:" in result.stdout + assert rc == 2 + assert "broken testcase references found: 1" in captured.out -def test_gate_specific_need_type(tmp_path: Path) -> None: +def test_gate_specific_need_type_only( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: metrics_by_type = { "tool_req": { "include_not_implemented": False, "include_external": False, - "requirements": { - "total": 2, - "with_code_link": 2, - "with_test_link": 2, - "fully_linked": 2, - "with_code_link_pct": 100.0, - "with_test_link_pct": 100.0, - "fully_linked_pct": 100.0, - "missing_code_link_ids": [], - "missing_test_link_ids": [], - "not_fully_linked_ids": [], - }, - "tests": { - "total": 1, - "filtered_test_types": [], - "linked_to_requirements": 1, - "linked_to_requirements_pct": 100.0, - "broken_references": [], - }, + "total": 2, + "with_code_link": 2, + "with_test_link": 2, + "fully_linked": 2, + "with_code_link_pct": 100.0, + "with_test_link_pct": 100.0, + "fully_linked_pct": 100.0, }, "comp_req": { "include_not_implemented": False, "include_external": False, - "requirements": { - "total": 5, - "with_code_link": 0, - "with_test_link": 0, - "fully_linked": 0, - "with_code_link_pct": 0.0, - "with_test_link_pct": 0.0, - "fully_linked_pct": 0.0, - "missing_code_link_ids": ["C1", "C2", "C3", "C4", "C5"], - "missing_test_link_ids": ["C1", "C2", "C3", "C4", "C5"], - "not_fully_linked_ids": ["C1", "C2", "C3", "C4", "C5"], - }, - "tests": { - "total": 0, - "filtered_test_types": [], - "linked_to_requirements": 0, - "linked_to_requirements_pct": 100.0, - "broken_references": [], - }, + "total": 5, + "with_code_link": 0, + "with_test_link": 0, + "fully_linked": 0, + "with_code_link_pct": 0.0, + "with_test_link_pct": 0.0, + "fully_linked_pct": 0.0, }, } - metrics_json = _write_metrics_json(tmp_path, metrics_by_type) + tests = { + "total": 1, + "linked_to_requirements": 1, + "linked_to_requirements_pct": 100.0, + "broken_references": [], + } + metrics_json = _write_metrics_json( + tmp_path, metrics_by_type=metrics_by_type, tests=tests + ) - # Gate only on tool_req (which is fully linked) — comp_req failures are ignored - result = _run_gate( - metrics_json, - ["--need-type", "tool_req", "--require-all-links"], + rc = _run_main( + monkeypatch, + [ + "--metrics-json", + str(metrics_json), + "--need-type", + "tool_req", + "--require-all-links", + ], ) + captured = capsys.readouterr() - assert result.returncode == 0 - assert "[tool_req]" in result.stdout - assert "[comp_req]" not in result.stdout + assert rc == 0 + assert "[tool_req]" in captured.out + assert "[comp_req]" not in captured.out -def test_gate_unknown_need_type_fails(tmp_path: Path) -> None: +def test_gate_unknown_need_type_fails( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: metrics_json = _write_metrics_json(tmp_path) - result = _run_gate(metrics_json, ["--need-type", "nonexistent_req"]) + rc = _run_main( + monkeypatch, + ["--metrics-json", str(metrics_json), "--need-type", "nonexistent_req"], + ) + captured = capsys.readouterr() - assert result.returncode == 2 - assert "not found in metrics JSON" in result.stdout + assert rc == 2 + assert "not found in metrics JSON" in captured.out -def test_gate_unsupported_schema_version(tmp_path: Path) -> None: - bad = tmp_path / "bad.json" - bad.write_text( - json.dumps({"schema_version": "99", "metrics_by_type": {}}), encoding="utf-8" - ) +def test_gate_unsupported_schema_version( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + metrics_json = _write_metrics_json(tmp_path, schema_version="99") - result = _run_gate(bad, []) + rc = _run_main(monkeypatch, ["--metrics-json", str(metrics_json)]) + captured = capsys.readouterr() - assert result.returncode == 1 - assert "unsupported schema_version" in result.stderr + assert rc == 1 + assert "unsupported schema_version" in captured.err -def test_gate_missing_file_returns_error(tmp_path: Path) -> None: - result = _run_gate(tmp_path / "does_not_exist.json", []) +def test_gate_missing_file_returns_error( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + missing = tmp_path / "does_not_exist.json" + + rc = _run_main(monkeypatch, ["--metrics-json", str(missing)]) + captured = capsys.readouterr() + + assert rc == 1 + assert "metrics JSON not found" in captured.err + + +def test_gate_missing_metrics_by_type_returns_error( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + payload = { + "schema_version": "2", + "generated_by": "sphinx_build", + "overall_metrics": { + "total": 0, + "with_code_link": 0, + "with_test_link": 0, + "fully_linked": 0, + "with_code_link_pct": 0.0, + "with_test_link_pct": 0.0, + "fully_linked_pct": 0.0, + }, + "tests": { + "total": 0, + "linked_to_requirements": 0, + "linked_to_requirements_pct": 0.0, + "broken_references": [], + }, + } + metrics_json = tmp_path / "metrics.json" + metrics_json.write_text(json.dumps(payload), encoding="utf-8") + + rc = _run_main(monkeypatch, ["--metrics-json", str(metrics_json)]) + captured = capsys.readouterr() + + assert rc == 1 + assert "missing or empty 'metrics_by_type'" in captured.err + + +def test_gate_missing_tests_section_returns_error( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + metrics_by_type = { + "tool_req": { + "include_not_implemented": False, + "include_external": False, + "total": 1, + "with_code_link": 1, + "with_test_link": 1, + "fully_linked": 1, + "with_code_link_pct": 100.0, + "with_test_link_pct": 100.0, + "fully_linked_pct": 100.0, + } + } + payload = { + "schema_version": "2", + "generated_by": "sphinx_build", + "overall_metrics": _derive_overall_metrics(metrics_by_type), + "metrics_by_type": metrics_by_type, + } + metrics_json = tmp_path / "metrics.json" + metrics_json.write_text(json.dumps(payload), encoding="utf-8") + + rc = _run_main(monkeypatch, ["--metrics-json", str(metrics_json)]) + captured = capsys.readouterr() - assert result.returncode == 1 - assert "not found" in result.stderr + assert rc == 1 + assert "missing 'tests' section" in captured.err diff --git a/scripts_bazel/traceability_gate.py b/scripts_bazel/traceability_gate.py index 399f43f80..44c61172e 100644 --- a/scripts_bazel/traceability_gate.py +++ b/scripts_bazel/traceability_gate.py @@ -22,7 +22,7 @@ CI gate → traceability_gate --metrics-json metrics.json [--min-* ...] The gate never parses needs.json itself; it only reads the pre-computed -schema-v1 metrics file produced by the docs build. +schema-v2 metrics file produced by the docs build, which is saved in '_build/'. """ from __future__ import annotations @@ -34,23 +34,34 @@ from pathlib import Path from typing import Any -_SUPPORTED_SCHEMA_VERSION = "1" +_SUPPORTED_SCHEMA_VERSION = "2" -def _print_type_summary(need_type: str, metrics: dict[str, Any]) -> None: - req = metrics["requirements"] - tst = metrics["tests"] - req_total = req["total"] - req_with_code_link = req["with_code_link"] - req_with_code_link_pct = req["with_code_link_pct"] - req_with_test_link = req["with_test_link"] - req_with_test_link_pct = req["with_test_link_pct"] - req_fully_linked = req["fully_linked"] - req_fully_linked_pct = req["fully_linked_pct"] - tst_total = tst["total"] - tst_linked_to_requirements = tst["linked_to_requirements"] - tst_linked_to_requirements_pct = tst["linked_to_requirements_pct"] - tst_broken_references = tst["broken_references"] +def _print_type_summary( + need_type: str, + req_metrics: dict[str, Any], + tst_metrics: dict[str, Any], +) -> None: + """ + Print a human-readable traceability summary for one requirement type. + + Args: + need_type: Requirement type key from ``metrics_by_type``. + req_metrics: Requirement coverage metrics for this type. + tst_metrics: Global testcase linkage metrics. + """ + req_total = req_metrics["total"] + req_with_code_link = req_metrics["with_code_link"] + req_with_code_link_pct = req_metrics["with_code_link_pct"] + req_with_test_link = req_metrics["with_test_link"] + req_with_test_link_pct = req_metrics["with_test_link_pct"] + req_fully_linked = req_metrics["fully_linked"] + req_fully_linked_pct = req_metrics["fully_linked_pct"] + + tst_total = tst_metrics["total"] + tst_linked_to_requirements = tst_metrics["linked_to_requirements"] + tst_linked_to_requirements_pct = tst_metrics["linked_to_requirements_pct"] + tst_broken_references = tst_metrics["broken_references"] print(f"[{need_type}]") print( @@ -78,21 +89,39 @@ def _print_type_summary(need_type: str, metrics: dict[str, Any]) -> None: def _check_type_thresholds( need_type: str, - metrics: dict[str, Any], + req_metrics: dict[str, Any], + tst_metrics: dict[str, Any], min_req_code: float, min_req_test: float, min_req_fully_linked: float, min_tests_linked: float, fail_on_broken_test_refs: bool, ) -> list[str]: + """ + Evaluate threshold checks for one requirement type. + + Args: + need_type: Requirement type key from ``metrics_by_type``. + req_metrics: Requirement coverage metrics for this type. + tst_metrics: Global testcase linkage metrics. + min_req_code: Minimum percent of requirements with source code links. + min_req_test: Minimum percent of requirements with test links. + min_req_fully_linked: Minimum percent of fully linked requirements. + min_tests_linked: Minimum percent of tests linked to requirements. + fail_on_broken_test_refs: If True, fail when broken test references exist. + + Returns: + A list of failure messages. Empty list means all thresholds passed. + """ failures: list[str] = [] - req = metrics["requirements"] - tst = metrics["tests"] - req_with_code_link_pct = req["with_code_link_pct"] - req_with_test_link_pct = req["with_test_link_pct"] - req_fully_linked_pct = req["fully_linked_pct"] - tst_linked_to_requirements_pct = tst["linked_to_requirements_pct"] - tst_broken_references = tst["broken_references"] + + req_with_code_link_pct = req_metrics["with_code_link_pct"] + req_with_test_link_pct = req_metrics["with_test_link_pct"] + req_fully_linked_pct = req_metrics["fully_linked_pct"] + + tst_linked_to_requirements_pct = tst_metrics["linked_to_requirements_pct"] + tst_broken_references = tst_metrics["broken_references"] + prefix = f"[{need_type}] " if req_with_code_link_pct < min_req_code: @@ -119,10 +148,20 @@ def _check_type_thresholds( failures.append( f"{prefix}broken testcase references found: {len(tst_broken_references)}" ) + return failures def main() -> int: + """ + Run the traceability threshold gate. + + Returns: + Process exit code: + - 0 on success + - 1 on input or schema errors + - 2 on threshold failures + """ parser = argparse.ArgumentParser( description=( "Read a traceability metrics JSON (schema v1) and enforce coverage " @@ -205,7 +244,16 @@ def main() -> int: ) return 1 - metrics_by_type: dict[str, Any] = data["metrics_by_type"] + metrics_by_type: dict[str, Any] = data.get("metrics_by_type", {}) + tests_metrics: dict[str, Any] = data.get("tests", {}) + + if not metrics_by_type: + print("Error: missing or empty 'metrics_by_type' section.", file=sys.stderr) + return 1 + if not tests_metrics: + print("Error: missing 'tests' section.", file=sys.stderr) + return 1 + types_to_check = ( [args.need_type] if args.need_type else list(metrics_by_type.keys()) ) @@ -222,11 +270,14 @@ def main() -> int: f"(available: {available})" ) continue - _print_type_summary(need_type, metrics_by_type[need_type]) + + req_metrics = metrics_by_type[need_type] + _print_type_summary(need_type, req_metrics, tests_metrics) failures.extend( _check_type_thresholds( need_type, - metrics_by_type[need_type], + req_metrics, + tests_metrics, args.min_req_code, args.min_req_test, args.min_req_fully_linked, diff --git a/scripts_bazel/traceability_metrics_schema.json b/scripts_bazel/traceability_metrics_schema.json index c89bc7182..dea866a8c 100644 --- a/scripts_bazel/traceability_metrics_schema.json +++ b/scripts_bazel/traceability_metrics_schema.json @@ -1,54 +1,51 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://eclipse-score.github.io/docs-as-code/traceability-metrics-schema.json", - "title": "Traceability Metrics", - "description": "Per-need-type traceability coverage metrics produced by the docs build (score_metamodel Sphinx extension). Consumed by traceability_gate to enforce coverage thresholds without re-parsing needs.json.", + "title": "Traceability Metrics Schema", + "description": "Schema for traceability metrics generated by the documentation build.", "type": "object", - "required": ["schema_version", "generated_by", "metrics_by_type"], "additionalProperties": false, + "required": [ + "schema_version", + "generated_by", + "overall_metrics", + "metrics_by_type", + "tests" + ], "properties": { "schema_version": { "type": "string", - "const": "1", - "description": "Schema version. Bump when the shape changes incompatibly." + "const": "2", + "description": "Version of this metrics schema." }, "generated_by": { "type": "string", - "description": "Name of the producer that generated this file (e.g. 'sphinx_build')." + "minLength": 1, + "description": "Identifier of the producer that generated this metrics file." + }, + "overall_metrics": { + "$ref": "#/$defs/requirementMetrics" }, "metrics_by_type": { "type": "object", - "description": "Metrics keyed by need type (e.g. 'tool_req', 'comp_req'). Each key is one call-scope: the requirement type string passed via --requirement-types.", + "description": "Mapping from need type (for example, tool_req) to requirement metrics.", "minProperties": 1, "additionalProperties": { - "$ref": "#/$defs/TypeMetrics" + "$ref": "#/$defs/requirementMetrics" + }, + "propertyNames": { + "type": "string", + "minLength": 1 } + }, + "tests": { + "$ref": "#/$defs/testMetrics" } }, "$defs": { - "TypeMetrics": { + "requirementMetrics": { "type": "object", - "required": ["include_not_implemented", "include_external", "requirements", "tests"], "additionalProperties": false, - "properties": { - "include_not_implemented": { - "type": "boolean", - "description": "Whether requirements with implemented==NO were counted in the denominator." - }, - "include_external": { - "type": "boolean", - "description": "Whether external (imported) requirements were included in the metrics." - }, - "requirements": { - "$ref": "#/$defs/RequirementMetrics" - }, - "tests": { - "$ref": "#/$defs/TestMetrics" - } - } - }, - "RequirementMetrics": { - "type": "object", "required": [ "total", "with_code_link", @@ -56,121 +53,87 @@ "fully_linked", "with_code_link_pct", "with_test_link_pct", - "fully_linked_pct", - "missing_code_link_ids", - "missing_test_link_ids", - "not_fully_linked_ids" + "fully_linked_pct" ], - "additionalProperties": false, "properties": { "total": { "type": "integer", - "minimum": 0, - "description": "Total number of requirements in scope." + "minimum": 0 }, "with_code_link": { "type": "integer", - "minimum": 0, - "description": "Requirements that have a non-empty source_code_link." + "minimum": 0 }, "with_test_link": { "type": "integer", - "minimum": 0, - "description": "Requirements that have a non-empty testlink." + "minimum": 0 }, "fully_linked": { "type": "integer", - "minimum": 0, - "description": "Requirements that have both source_code_link and testlink." + "minimum": 0 }, "with_code_link_pct": { - "type": "number", - "minimum": 0, - "maximum": 100, - "description": "with_code_link / total * 100, or 100 when total == 0." + "$ref": "#/$defs/percentage" }, "with_test_link_pct": { - "type": "number", - "minimum": 0, - "maximum": 100, - "description": "with_test_link / total * 100, or 100 when total == 0." + "$ref": "#/$defs/percentage" }, "fully_linked_pct": { - "type": "number", - "minimum": 0, - "maximum": 100, - "description": "fully_linked / total * 100, or 100 when total == 0." - }, - "missing_code_link_ids": { - "type": "array", - "items": { "type": "string" }, - "description": "Sorted IDs of requirements missing source_code_link." - }, - "missing_test_link_ids": { - "type": "array", - "items": { "type": "string" }, - "description": "Sorted IDs of requirements missing testlink." - }, - "not_fully_linked_ids": { - "type": "array", - "items": { "type": "string" }, - "description": "Sorted IDs of requirements missing either source_code_link or testlink." + "$ref": "#/$defs/percentage" } } }, - "TestMetrics": { + "testMetrics": { "type": "object", + "additionalProperties": false, "required": [ "total", - "filtered_test_types", "linked_to_requirements", "linked_to_requirements_pct", "broken_references" ], - "additionalProperties": false, "properties": { "total": { "type": "integer", - "minimum": 0, - "description": "Total testcase needs considered (after optional test_type filter)." - }, - "filtered_test_types": { - "type": "array", - "items": { "type": "string" }, - "description": "The test_type values used to filter testcases. Empty means all types." + "minimum": 0 }, "linked_to_requirements": { "type": "integer", - "minimum": 0, - "description": "Testcases that reference at least one requirement via partially_verifies or fully_verifies." + "minimum": 0 }, "linked_to_requirements_pct": { - "type": "number", - "minimum": 0, - "maximum": 100, - "description": "linked_to_requirements / total * 100, or 100 when total == 0." + "$ref": "#/$defs/percentage" }, "broken_references": { "type": "array", - "items": { "$ref": "#/$defs/BrokenReference" }, - "description": "Testcase references that point to unknown requirement IDs." + "items": { + "$ref": "#/$defs/brokenReference" + } } } }, - "BrokenReference": { + "brokenReference": { "type": "object", - "required": ["testcase", "missing_need"], "additionalProperties": false, + "required": [ + "testcase", + "missing_need" + ], "properties": { "testcase": { "type": "string", - "description": "ID of the testcase containing the broken reference." + "minLength": 1 }, "missing_need": { "type": "string", - "description": "Requirement ID referenced by the testcase that does not exist in needs.json." + "minLength": 1 } } + }, + "percentage": { + "type": "number", + "minimum": 0, + "maximum": 100 } } } diff --git a/src/extensions/score_metamodel/__init__.py b/src/extensions/score_metamodel/__init__.py index 8adc43702..97e4738e3 100644 --- a/src/extensions/score_metamodel/__init__.py +++ b/src/extensions/score_metamodel/__init__.py @@ -270,9 +270,9 @@ def setup(app: Sphinx) -> dict[str, str | bool]: # sphinx-collections runs on default prio 500. # We need to populate the sphinx-collections config before that happens. # If we put it anywhere higher it seems that other things already lock the needs - # To ensure that this runs first before locking happens priot is => 450 + # To ensure that this runs first before locking happens priot is => 425 # The lower the number the higher priority it has (runs earlier) - _ = app.connect("config-inited", connect_external_needs, priority=450) + _ = app.connect("config-inited", connect_external_needs, priority=425) discover_checks() diff --git a/src/extensions/score_metrics/__init__.py b/src/extensions/score_metrics/__init__.py index b9ef2a1c0..4cbfbfa20 100644 --- a/src/extensions/score_metrics/__init__.py +++ b/src/extensions/score_metrics/__init__.py @@ -40,7 +40,7 @@ def calculate_need_metrics(app: Sphinx, env: BuildEnvironment) -> None: def _write_metrics_json(app: Sphinx, exception: Any | None) -> None: """ - Write a schema-v1 metrics.json alongside needs.json in the build output. + Write a schema-v2 metrics.json alongside needs.json in the build output. """ if exception: logger.error(f"Sphinx-Exception at end of build: {exception}") diff --git a/src/extensions/score_metrics/sphinx_filters.py b/src/extensions/score_metrics/sphinx_filters.py index b82ac955c..5bc21a5be 100644 --- a/src/extensions/score_metrics/sphinx_filters.py +++ b/src/extensions/score_metrics/sphinx_filters.py @@ -192,45 +192,24 @@ def _get_key_values(results: list[int], argument_paths: list[str]): results.append(int(current)) -def get_metrics_with_overall_total_considered( +def get_metrics_with_first_value_total( needs: list[Any], results: list[int], **kwargs: Any ) -> None: - """Append selected metrics and compute remainder from overall total. - - This function appends ``overall_metrics:total`` first, then appends all - metrics referenced by ``kwargs`` values. Finally, it replaces the first - appended value with the remainder after subtracting all other appended - values. + """ + This function calculates the first parameter you give it as the 'total'. + E.g. + - (metrics_by_type:tool_req:total, metrics_by_type:tool_req:with_test_link) + It will then treat the first arguemnt so 'tool_req-total' as the 'total number', and will + subtract all following numbers from it. + So in this case it would subtract the 'tool_req-with test link' number from the first. + That way you can have a 'missing' value as the first. """ results.clear() - metrics_json = CALCULATED_METRICS - results.append(int(metrics_json["overall_metrics"]["total"])) + # As kwargs ordering is deterministic this will always put the first total into results[0] _get_key_values(results, [str(value) for value in kwargs.values()]) results[0] -= sum(results[1:]) -def get_metrics_with_custom_type_total_considered( - needs: list[Any], results: list[int], **kwargs: Any -) -> None: - """Append selected metrics, optionally using a custom total path. - - If the last kwarg value ends with ``":total"``, that path is used as - baseline total and all preceding paths are treated as components; the first - result becomes ``total - sum(components)``. Otherwise, all paths are simply - appended as-is. - """ - # Get the 'total' that was specified as the first value - - results.clear() - values = [str(value) for value in kwargs.values()] - if values[-1].endswith(":total"): - _get_key_values(results, [values[-1]]) # baseline total - _get_key_values(results, values[:-1]) # components - results[0] -= sum(results[1:]) - else: - _get_key_values(results, values) - - def get_just_metrics(needs: list[Any], results: list[int], **kwargs: Any) -> None: """Append selected metric values without any total/remainder calculation. diff --git a/src/extensions/score_metrics/traceability_metrics.py b/src/extensions/score_metrics/traceability_metrics.py index 2ce09db8b..c14a944b0 100644 --- a/src/extensions/score_metrics/traceability_metrics.py +++ b/src/extensions/score_metrics/traceability_metrics.py @@ -183,7 +183,7 @@ def calculate_full_need_metrics(app: Sphinx, include_external: bool): ) output: dict[str, Any] = { - "schema_version": "1", + "schema_version": "2", "generated_by": "sphinx_build", "overall_metrics": overall_metrics, "metrics_by_type": metrics_by_type, From 2a71dbe64819c028049ec01c76c697d3fcf2937d Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Fri, 5 Jun 2026 15:57:43 +0200 Subject: [PATCH 07/11] Fix: Upgrading of docs for new way extensions works --- docs/how-to/dashboards_and_quality_gates.rst | 321 +++--------------- .../requirements/implementation_state.rst | 136 -------- .../requirements/tooling_verification.rst | 11 + 3 files changed, 59 insertions(+), 409 deletions(-) delete mode 100644 docs/internals/requirements/implementation_state.rst diff --git a/docs/how-to/dashboards_and_quality_gates.rst b/docs/how-to/dashboards_and_quality_gates.rst index 0bdaf9465..987147bd9 100644 --- a/docs/how-to/dashboards_and_quality_gates.rst +++ b/docs/how-to/dashboards_and_quality_gates.rst @@ -156,295 +156,70 @@ For a new consumer repository: ║ .. The Following part has been generated by Copilot ║ ╙ ╜ -Metrics Needpie functions -========================= +Needpie Usage Guide (Quick and Practical) +========================================= -Overview --------- +Use these examples as ready-to-copy templates for dashboard-style pie charts. -These helpers read values from a nested dictionary named ``CALCULATED_METRICS``. -A metric is selected by a **colon-separated path**, for example: +Tips for readable charts +------------------------ -- ``overall_metrics:total`` -- ``overall_metrics:with_test_link`` -- ``types:bug:total`` +- Keep labels short and audience-friendly. +- Put the most important category first. +- Use high-contrast colors that are easy to distinguish. +- Avoid red/green-only combinations when possible. -All resolved values are converted to ``int`` and appended to a mutable -``results`` list passed by the caller. +Recommended color palette +~~~~~~~~~~~~~~~~~~~~~~~~~ -The ``needs`` parameter is accepted for integration compatibility, but it is not -used by the current implementations. +This palette is generally easier to read: -Path format ------------ +- ``#4E79A7`` (blue) +- ``#F28E2B`` (orange) +- ``#59A14F`` (green) +- ``#B07AA1`` (purple) -Paths are split using ``:`` and then resolved step by step. +Example 1: Overall view (including remaining/unlinked) +------------------------------------------------------ -Example: +.. code-block:: rst -.. code-block:: text + .. needpie:: Overall Requirement Coverage + :labels: Remaining (no selected links), With Test Link, With Code Link, Fully Linked + :colors: #4E79A7, #F28E2B, #59A14F, #B07AA1 + :filter-func: score_metrics.sphinx_filters.get_metrics_with_first_value_total(overall_metrics:total,overall_metrics:with_test_link,overall_metrics:with_code_link,overall_metrics:fully_linked) - path = "overall_metrics:total" +This chart shows: -is resolved like: +- remaining requirements (calculated from total), +- requirements with test link, +- requirements with code link, +- fully linked requirements. -.. code-block:: python - - current = CALCULATED_METRICS - current = current["overall_metrics"] - current = current["total"] - -Function reference ------------------- - -_get_key_values(results, argument_paths) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Purpose -^^^^^^^ - -Internal helper that appends metric values for a list of paths. - -Behavior -^^^^^^^^ - -1. Iterate over each item in ``argument_paths``. -2. Strip whitespace. -3. Skip empty paths. -4. Resolve the path inside ``CALCULATED_METRICS``. -5. Convert to integer and append to ``results``. - -Notes -^^^^^ - -- Modifies ``results`` in place. -- Does not return a new list. -- Raises ``KeyError`` if a path key does not exist. -- Raises ``ValueError`` if a resolved value cannot be converted to ``int``. - -get_metrics_with_overall_total_considered(...) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Purpose -^^^^^^^ - -Compute a remainder against global overall total. - -Behavior -^^^^^^^^ - -1. Append ``CALCULATED_METRICS["overall_metrics"]["total"]`` as first value. -2. Append each metric referenced by ``kwargs.values()``. -3. Replace the first value with: - - ``overall_total - sum(all other appended values)`` - -Typical use case -^^^^^^^^^^^^^^^^ - -Use this when your pie/chart needs an "Other" bucket based on the global total. - -Example -^^^^^^^ - -Assume: - -.. code-block:: python - - CALCULATED_METRICS = { - "overall_metrics": { - "total": 100, - "with_test_link": 30, - "with_review": 20, - } - } - -Call: - -.. code-block:: python - - results = [] - get_metrics_with_overall_total_considered( - needs=[], - results=results, - a="overall_metrics:with_test_link", - b="overall_metrics:with_review", - ) - -Result: - -.. code-block:: python - - # Step 1 append total: [100] - # Step 2 append selected: [100, 30, 20] - # Step 3 remainder: [50, 30, 20] - results == [50, 30, 20] - -get_metrics_with_custom_type_total_considered(...) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Purpose -^^^^^^^ - -Support custom total path if provided as the last kwarg value. - -Behavior -^^^^^^^^ - -If the **last** provided path ends with ``:total``: - -1. Append that last path as baseline total. -2. Append all preceding paths as components. -3. Replace the first value with: - - ``baseline_total - sum(components)`` - -Otherwise: +.. needpie:: Overall Requirement Coverage + :labels: Remaining (no selected links), With Test Link, With Code Link, Fully Linked + :colors: #4E79A7, #F28E2B, #59A14F, #B07AA1 + :filter-func: score_metrics.sphinx_filters.get_metrics_with_first_value_total(overall_metrics:total,overall_metrics:with_test_link,overall_metrics:with_code_link,overall_metrics:fully_linked) -- Append all provided paths directly (no subtraction). +Example 2: Focused view (no total-based remainder) +-------------------------------------------------- -Important ordering rule -^^^^^^^^^^^^^^^^^^^^^^^ +This chart shows direct values only (no remaining slice): -The special total path must be the **last** kwarg value, for example: +- tool requirements with test link, +- tool requirements with code link, +- overall fully linked requirements. -.. code-block:: python - - get_metrics_with_custom_type_total_considered( - needs=[], - results=results, - part_a="types:feature:done", - part_b="types:feature:in_progress", - total="types:feature:total", # must be last - ) - -Example with custom total -^^^^^^^^^^^^^^^^^^^^^^^^^ - -Assume: - -.. code-block:: python - - CALCULATED_METRICS = { - "types": { - "feature": { - "total": 40, - "done": 10, - "in_progress": 5, - } - } - } - -Call: - -.. code-block:: python - - results = [] - get_metrics_with_custom_type_total_considered( - needs=[], - results=results, - done="types:feature:done", - in_progress="types:feature:in_progress", - total="types:feature:total", - ) - -Result: - -.. code-block:: python - - # baseline total: 40 - # components: 10, 5 - # remainder: 40 - (10 + 5) = 25 - results == [25, 10, 5] - -Example without custom total -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Call: - -.. code-block:: python - - results = [] - get_metrics_with_custom_type_total_considered( - needs=[], - results=results, - done="types:feature:done", - in_progress="types:feature:in_progress", - ) - -Result: - -.. code-block:: python - - # no subtraction mode - results == [10, 5] - -get_just_metrics(...) -~~~~~~~~~~~~~~~~~~~~~ - -Purpose -^^^^^^^ - -Append only the selected metric values. - -Behavior -^^^^^^^^ - -- Interprets each kwarg value as a path. -- Resolves and appends each value. -- No total baseline and no remainder calculation. - -Example -^^^^^^^ - -.. code-block:: python - - results = [] - get_just_metrics( - needs=[], - results=results, - a="overall_metrics:with_test_link", - b="overall_metrics:with_review", - ) - # results might become [30, 20] - -How to use these functions correctly ------------------------------------- - -1. Pass a mutable list in ``results`` (usually ``[]`` initially). -2. Provide metric paths through keyword argument values. -3. Use colon-separated keys (``a:b:c``), not dot notation. -4. For custom-total mode, ensure the total path is the last kwarg value. -5. Expect in-place mutation of ``results``. - -Common pitfalls ---------------- - -- Empty kwargs in custom-total function can lead to index errors - (because ``values[-1]`` is accessed). -- Invalid paths raise ``KeyError``. -- Non-integer values raise ``ValueError`` during ``int(...)`` conversion. -- ``print(results)`` currently causes side effects during execution; remove if - quiet behavior is preferred in production. - -Testing recommendations ------------------------ - -Add unit tests for: - -- Basic path resolution with one and multiple paths. -- Whitespace and empty-path handling. -- Overall total remainder logic. -- Custom total behavior when last path ends with ``:total``. -- Behavior when no custom total is provided. -- Error handling for missing keys and non-integer values. -- Empty ``kwargs`` behavior in custom-total function. +.. needpie:: Tool Requirements Snapshot + :labels: Tool Req With Test Link, Tool Req With Code Link, Fully Linked (Overall) + :colors: #F28E2B, #4E79A7, #59A14F + :filter-func: score_metrics.sphinx_filters.get_just_metrics(metrics_by_type:tool_req:with_test_link,metrics_by_type:tool_req:with_code_link,overall_metrics:fully_linked) +Presentation checklist +---------------------- -Related Guides --------------- +Before publishing, quickly verify: -- :ref:`setup` -- :doc:`other_modules` -- :doc:`source_to_doc_links` -- :doc:`test_to_doc_links` +- labels are understandable for non-technical readers, +- colors are distinct and readable on your page theme, +- title clearly says what the chart represents. diff --git a/docs/internals/requirements/implementation_state.rst b/docs/internals/requirements/implementation_state.rst deleted file mode 100644 index 32dc974d7..000000000 --- a/docs/internals/requirements/implementation_state.rst +++ /dev/null @@ -1,136 +0,0 @@ -.. - # ******************************************************************************* - # Copyright (c) 2026 Contributors to the Eclipse Foundation - # - # See the NOTICE file(s) distributed with this work for additional - # information regarding copyright ownership. - # - # This program and the accompanying materials are made available under the - # terms of the Apache License Version 2.0 which is available at - # https://www.apache.org/licenses/LICENSE-2.0 - # - # SPDX-License-Identifier: Apache-2.0 - # ******************************************************************************* - - # Assisted-by: GitHub Copilot -.. _docs_statistics: - -Tooling Coverage -================ - -This page shows how the docs-as-code tooling covers process and tool -requirements. It focuses on tooling capabilities offered to downstream -repositories rather than on product-specific traceability inside those -repositories. - -.. Overview -.. -------- -.. - -.. needpie:: Overall Metrics with Total incooperated - :labels: without any link, overall with test link, overall with code, overall fully linked - :colors: red, yellow, blue, green - :filter-func: score_metrics.sphinx_filters.get_metrics_with_overall_total_considered(overall_metrics:with_test_link,overall_metrics:with_code_link,overall_metrics:fully_linked) - - - -.. needpie:: Metrics without any total incooperated - :labels: tool req with test link, tool req with code, overall fully linked - :colors: yellow, blue, green - :filter-func: score_metrics.sphinx_filters.get_just_metrics(metrics_by_type:tool_req:with_test_link,metrics_by_type:tool_req:with_code_link,overall_metrics:fully_linked) - - -.. Jump to evidence tables: -.. -.. - :ref:`Tool Requirement Implementation and Links table ` -.. - :ref:`Process Requirement to Tool Requirement mapping table ` -.. -.. How To Read These Levels -.. ------------------------ -.. -.. The overview pie combines implementation state and traceability evidence: -.. -.. - ``not implemented``: -.. requirement has ``implemented == NO``. -.. - ``implemented but incomplete traceability``: -.. requirement has ``implemented == YES`` or ``implemented == PARTIAL``, -.. but is missing at least one traceability link (code link and/or test link). -.. - ``fully linked``: -.. requirement is implemented and has both ``source_code_link`` and ``testlink``. -.. -.. Implementation labels used on this page: -.. -.. - ``NO``: requirement is not implemented. -.. - ``PARTIAL``: requirement is partly implemented. -.. - ``YES``: requirement is implemented. -.. -.. Why multiple pies are shown: -.. -.. - ``Requirements with Codelinks`` shows requirement-to-implementation traceability. -.. - ``Requirements with linked tests`` shows requirement-to-verification traceability. -.. - ``Requirements fully linked`` is the strict roll-up (both links present). -.. -.. These are intentionally separate because they answer different diagnostics: -.. missing code links, missing test links, or both. -.. -.. In Detail -.. --------- -.. -.. .. grid:: 2 -.. :class-container: score-grid -.. -.. .. grid-item-card:: -.. -.. .. needpie:: Requirements marked as Implemented -.. :labels: not implemented, partial, implemented -.. :colors: red, orange, green -.. -.. type == 'tool_req' and implemented == 'NO' -.. type == 'tool_req' and implemented == 'PARTIAL' -.. type == 'tool_req' and implemented == 'YES' -.. -.. .. grid-item-card:: -.. -.. .. needpie:: Requirements with Codelinks -.. :labels: no codelink, with codelink -.. :colors: red, green -.. :filter-func: src.extensions.score_metrics.traceability_dashboard.pie_requirements_with_code_links(tool_req) -.. -.. .. grid-item-card:: -.. -.. .. needpie:: Requirements with linked tests -.. :labels: no test link, with test link -.. :colors: red, green -.. :filter-func: src.extensions.score_metrics.traceability_dashboard.pie_requirements_with_test_links(tool_req) -.. -.. .. grid-item-card:: -.. -.. .. needpie:: Requirements fully linked (code + tests) -.. :labels: not fully linked, fully linked -.. :colors: orange, green -.. :filter-func: src.extensions.score_metrics.traceability_dashboard.pie_requirements_fully_linked(tool_req) -.. -.. .. grid-item-card:: -.. -.. .. needpie:: Process requirements linked by tool requirements -.. :labels: not linked, linked -.. :colors: red, green -.. :filter-func: src.extensions.score_metrics.traceability_dashboard.pie_process_requirements_linked(tool_req,true) -.. -.. -.. Process-to-Tool Mapping -.. ----------------------- -.. -.. .. _tooling_coverage_table_process_mapping: -.. -.. .. needtable:: Process requirement -> tool requirement mapping -.. :types: tool_req -.. :columns: satisfies as "Process Requirement";id as "Tool Requirement" -.. :style: table -.. -.. .. _tooling_coverage_table_impl_links: -.. -.. .. needtable:: Tool requirement implementation and links -.. :types: tool_req -.. :columns: id as "Tool Requirement";implemented;source_code_link;testlink -.. :style: table diff --git a/docs/internals/requirements/tooling_verification.rst b/docs/internals/requirements/tooling_verification.rst index 5b477043d..a78d45d84 100644 --- a/docs/internals/requirements/tooling_verification.rst +++ b/docs/internals/requirements/tooling_verification.rst @@ -53,6 +53,17 @@ Overview No skipped or disabled tests are expected in the current dataset. +How many requirements are linked +--------------------------------- + +*This shows how many of our requirements are linked with tests, in source code, both or neither.* + +.. needpie:: Overall Requirement Coverage + :labels: Remaining (no selected links), With Test Link, With Code Link, Fully Linked + :colors: #4E79A7, #F28E2B, #59A14F, #B07AA1 + :filter-func: score_metrics.sphinx_filters.get_metrics_with_first_value_total(overall_metrics:total,overall_metrics:with_test_link,overall_metrics:with_code_link,overall_metrics:fully_linked) + + Testcase Metadata Overview -------------------------- From e0f34fa9efcc7e4fa09eb2e19d0063e9af46de65 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Fri, 5 Jun 2026 16:11:23 +0200 Subject: [PATCH 08/11] Fix: Fix tests & Documentation --- docs/how-to/dashboards_and_quality_gates.rst | 2 +- docs/internals/requirements/index.rst | 3 --- scripts_bazel/tests/traceability_gate_test.py | 1 + .../score_metrics/tests/test_sphinx_filters.py | 15 +++++++++------ 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/how-to/dashboards_and_quality_gates.rst b/docs/how-to/dashboards_and_quality_gates.rst index 987147bd9..4ed4da664 100644 --- a/docs/how-to/dashboards_and_quality_gates.rst +++ b/docs/how-to/dashboards_and_quality_gates.rst @@ -182,7 +182,7 @@ This palette is generally easier to read: Example 1: Overall view (including remaining/unlinked) ------------------------------------------------------ -.. code-block:: rst +.. code-block:: rst .. needpie:: Overall Requirement Coverage :labels: Remaining (no selected links), With Test Link, With Code Link, Fully Linked diff --git a/docs/internals/requirements/index.rst b/docs/internals/requirements/index.rst index 2e635fafa..0a7513742 100644 --- a/docs/internals/requirements/index.rst +++ b/docs/internals/requirements/index.rst @@ -28,8 +28,6 @@ docs-as-code as a Bazel dependency. Pages ----- -- ``implementation_state`` describes tooling coverage: implemented capabilities, - source-code links, test links, full linkage, and process-to-tool mapping. - ``tooling_verification`` describes verification evidence for the tooling itself, including test results and testcase metadata. @@ -39,5 +37,4 @@ Pages capabilities process_overview requirements - implementation_state tooling_verification diff --git a/scripts_bazel/tests/traceability_gate_test.py b/scripts_bazel/tests/traceability_gate_test.py index 20d7e607e..d9792b693 100644 --- a/scripts_bazel/tests/traceability_gate_test.py +++ b/scripts_bazel/tests/traceability_gate_test.py @@ -25,6 +25,7 @@ import json from pathlib import Path from typing import Any + import pytest import scripts_bazel.traceability_gate as gate diff --git a/src/extensions/score_metrics/tests/test_sphinx_filters.py b/src/extensions/score_metrics/tests/test_sphinx_filters.py index 79ba4f61d..007e9317b 100644 --- a/src/extensions/score_metrics/tests/test_sphinx_filters.py +++ b/src/extensions/score_metrics/tests/test_sphinx_filters.py @@ -135,9 +135,10 @@ def test_get_metrics_with_overall_total_considered_when_metrics_loaded() -> None sphinx_filters.CALCULATED_METRICS = EXAMPLE_METRICS results: list[int] = [] - sphinx_filters.get_metrics_with_overall_total_considered( + sphinx_filters.get_metrics_with_first_value_total( needs=[], results=results, + total="overall_metrics:total", code="overall_metrics:with_code_link", test="overall_metrics:with_test_link", fully="overall_metrics:fully_linked", @@ -151,13 +152,14 @@ def test_get_metrics_with_custom_type_total_considered_with_total_suffix() -> No sphinx_filters.CALCULATED_METRICS = EXAMPLE_METRICS results: list[int] = [] - sphinx_filters.get_metrics_with_custom_type_total_considered( + sphinx_filters.get_metrics_with_first_value_total( needs=[], results=results, + total="metrics_by_type:tool_req:total", code="metrics_by_type:tool_req:with_code_link", test="metrics_by_type:tool_req:with_test_link", - total="metrics_by_type:tool_req:total", ) + print(results) assert results == [12, 46, 3] @@ -167,14 +169,15 @@ def test_get_metrics_with_custom_type_total_considered_without_total_suffix() -> sphinx_filters.CALCULATED_METRICS = EXAMPLE_METRICS results: list[int] = [] - sphinx_filters.get_metrics_with_custom_type_total_considered( + sphinx_filters.get_metrics_with_first_value_total( needs=[], results=results, code="metrics_by_type:tool_req:with_code_link", test="metrics_by_type:tool_req:with_test_link", ) + print(results) - assert results == [46, 3] + assert results == [43, 3] def test_get_just_metrics_appends_values_when_metrics_loaded() -> None: @@ -200,6 +203,6 @@ def test_get_metrics_with_custom_type_total_considered_empty_kwargs_raises_index results: list[int] = [] with pytest.raises(IndexError): - sphinx_filters.get_metrics_with_custom_type_total_considered( + sphinx_filters.get_metrics_with_first_value_total( needs=[], results=results ) From 59a317911aa786823d1f0572a152db84d4199a64 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Fri, 5 Jun 2026 16:17:19 +0200 Subject: [PATCH 09/11] Fix: Fix too complex function & linting issues --- scripts_bazel/traceability_gate.py | 60 +++++++++++-------- .../tests/test_sphinx_filters.py | 4 +- 2 files changed, 35 insertions(+), 29 deletions(-) diff --git a/scripts_bazel/traceability_gate.py b/scripts_bazel/traceability_gate.py index 44c61172e..38b58495c 100644 --- a/scripts_bazel/traceability_gate.py +++ b/scripts_bazel/traceability_gate.py @@ -152,6 +152,39 @@ def _check_type_thresholds( return failures +def find_failures( + metrics_by_type: dict[str, Any], + test_metrics: dict[str, Any], + types_to_check: list[Any], + args: argparse.Namespace, +) -> list[str]: + failures: list[str] = [] + for need_type in types_to_check: + if need_type not in metrics_by_type: + available = list(metrics_by_type.keys()) + failures.append( + f"need type '{need_type}' not found in metrics JSON " + f"(available: {available})" + ) + continue + + req_metrics = metrics_by_type[need_type] + _print_type_summary(need_type, req_metrics, test_metrics) + failures.extend( + _check_type_thresholds( + need_type, + req_metrics, + test_metrics, + args.min_req_code, + args.min_req_test, + args.min_req_fully_linked, + args.min_tests_linked, + args.fail_on_broken_test_refs, + ) + ) + return failures + + def main() -> int: """ Run the traceability threshold gate. @@ -260,32 +293,7 @@ def main() -> int: print(f"Traceability gate input: {metrics_path}") print("-" * 72) - - failures: list[str] = [] - for need_type in types_to_check: - if need_type not in metrics_by_type: - available = list(metrics_by_type.keys()) - failures.append( - f"need type '{need_type}' not found in metrics JSON " - f"(available: {available})" - ) - continue - - req_metrics = metrics_by_type[need_type] - _print_type_summary(need_type, req_metrics, tests_metrics) - failures.extend( - _check_type_thresholds( - need_type, - req_metrics, - tests_metrics, - args.min_req_code, - args.min_req_test, - args.min_req_fully_linked, - args.min_tests_linked, - args.fail_on_broken_test_refs, - ) - ) - + failures = find_failures(metrics_by_type, tests_metrics, types_to_check, args) print("-" * 72) if failures: print("Threshold check failed:") diff --git a/src/extensions/score_metrics/tests/test_sphinx_filters.py b/src/extensions/score_metrics/tests/test_sphinx_filters.py index 007e9317b..fe07cb54e 100644 --- a/src/extensions/score_metrics/tests/test_sphinx_filters.py +++ b/src/extensions/score_metrics/tests/test_sphinx_filters.py @@ -203,6 +203,4 @@ def test_get_metrics_with_custom_type_total_considered_empty_kwargs_raises_index results: list[int] = [] with pytest.raises(IndexError): - sphinx_filters.get_metrics_with_first_value_total( - needs=[], results=results - ) + sphinx_filters.get_metrics_with_first_value_total(needs=[], results=results) From d9ae44b23ce1dc95418146d6569a6e9cb5d71424 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Fri, 5 Jun 2026 16:24:24 +0200 Subject: [PATCH 10/11] Fix: Pr review comments --- docs/how-to/dashboards_and_quality_gates.rst | 11 +++++------ src/extensions/score_metrics/__init__.py | 1 + src/extensions/score_metrics/sphinx_filters.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/how-to/dashboards_and_quality_gates.rst b/docs/how-to/dashboards_and_quality_gates.rst index 4ed4da664..87e6704d7 100644 --- a/docs/how-to/dashboards_and_quality_gates.rst +++ b/docs/how-to/dashboards_and_quality_gates.rst @@ -47,9 +47,8 @@ Default Behavior (No Configuration Needed) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ By default, ``score_metamodel`` autodiscovers requirement types from the -repository needs present in the current build. Requirement types are identified -from ``needs_types`` entries tagged with ``requirement`` or -``requirement_excl_process``. +repository needs present in the current build. +Requirement types are identified from ``needs_types`` entries tagged with ``requirement``. This is the recommended setup for most repositories. @@ -152,9 +151,9 @@ For a new consumer repository: 5. Raise thresholds over time as the repository matures. .. - ╓ ╖ - ║ .. The Following part has been generated by Copilot ║ - ╙ ╜ + ╓ ╖ + ║ Some portions generated by Github Copilot ║ + ╙ ╜ Needpie Usage Guide (Quick and Practical) ========================================= diff --git a/src/extensions/score_metrics/__init__.py b/src/extensions/score_metrics/__init__.py index 4cbfbfa20..c5b3112aa 100644 --- a/src/extensions/score_metrics/__init__.py +++ b/src/extensions/score_metrics/__init__.py @@ -71,6 +71,7 @@ def setup(app: Sphinx) -> dict[str, str | bool]: "Default is False so each repo gates only its own needs." ), ) + app.setup_extension("score_metamodel") # Calculates the metrics & sets global var for access _ = app.connect("env-updated", calculate_need_metrics, priority=600) diff --git a/src/extensions/score_metrics/sphinx_filters.py b/src/extensions/score_metrics/sphinx_filters.py index 5bc21a5be..5f0bfe6eb 100644 --- a/src/extensions/score_metrics/sphinx_filters.py +++ b/src/extensions/score_metrics/sphinx_filters.py @@ -33,7 +33,7 @@ def func(needs: list[NeedItem], results: list[int], **kwargs) -> None: ... .. needpie:: My Requirements Coverage :labels: Linked, Not Linked - :filter-func: score_metamodel.sphinx_filters.generic_pie_linked_items(std_req__mystandard__, gd_) + :filter-func: score_metrics.sphinx_filters.generic_pie_linked_items(std_req__mystandard__, gd_) """ From 073064da4eebe6d359c529fc5e87df0c5a6a5221 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Fri, 5 Jun 2026 16:25:15 +0200 Subject: [PATCH 11/11] Chore: Formatting & Linting --- docs/how-to/dashboards_and_quality_gates.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/dashboards_and_quality_gates.rst b/docs/how-to/dashboards_and_quality_gates.rst index 87e6704d7..29a6d4ae4 100644 --- a/docs/how-to/dashboards_and_quality_gates.rst +++ b/docs/how-to/dashboards_and_quality_gates.rst @@ -47,7 +47,7 @@ Default Behavior (No Configuration Needed) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ By default, ``score_metamodel`` autodiscovers requirement types from the -repository needs present in the current build. +repository needs present in the current build. Requirement types are identified from ``needs_types`` entries tagged with ``requirement``. This is the recommended setup for most repositories.