From 87185b210556b7c822de8f29d08db2598580341d Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Mon, 20 Jul 2026 13:24:40 +1000 Subject: [PATCH 1/2] chore(backend): drop pre-Reader metric-value code Removes the dead CSV/outlier helpers in core/metric_values.py and all of core/outliers.py, plus AnnotatedScalarValue from models.py. These were superseded by climate_ref.results.Reader and had no production caller left, only mock-based tests. Keeps the tests that cover live code: TestParseIdList and the build_scalar_from_reader/build_series_from_reader tests. --- backend/src/ref_backend/core/metric_values.py | 268 ------------- backend/src/ref_backend/core/outliers.py | 189 --------- backend/src/ref_backend/models.py | 8 - backend/tests/test_core/test_metric_values.py | 359 +----------------- backend/tests/test_core/test_outliers.py | 325 ---------------- .../+delete-dead-metric-values.trivial.md | 1 + 6 files changed, 3 insertions(+), 1147 deletions(-) delete mode 100644 backend/src/ref_backend/core/outliers.py delete mode 100644 backend/tests/test_core/test_outliers.py create mode 100644 changelog/+delete-dead-metric-values.trivial.md diff --git a/backend/src/ref_backend/core/metric_values.py b/backend/src/ref_backend/core/metric_values.py index efef54a..8f90277 100644 --- a/backend/src/ref_backend/core/metric_values.py +++ b/backend/src/ref_backend/core/metric_values.py @@ -1,25 +1,8 @@ """Shared logic for querying and formatting metric values.""" -import csv -import io -from collections.abc import Generator, Sequence from enum import StrEnum -from typing import TYPE_CHECKING, Literal, TypeVar from fastapi import HTTPException -from sqlalchemy.orm import Query -from starlette.responses import StreamingResponse - -from climate_ref import models -from ref_backend.core.filter_utils import build_filter_clause -from ref_backend.core.json_utils import sanitize_float_value -from ref_backend.core.outliers import detect_outliers_in_scalar_values -from ref_backend.models import AnnotatedScalarValue - -if TYPE_CHECKING: - from ref_backend.models import Facet - -TMetricValueModel = TypeVar("TMetricValueModel", bound=models.ScalarMetricValue | models.SeriesMetricValue) class MetricValueType(StrEnum): @@ -35,254 +18,3 @@ def parse_id_list(id_str: str) -> list[int]: return [int(i.strip()) for i in id_str.split(",") if i.strip()] except ValueError as e: raise HTTPException(status_code=400, detail=f"Invalid id in list: {e}") from e - - -def apply_metric_filters( - query: Query[TMetricValueModel], - filters: dict[str, str], - isolate_ids: str | None, - exclude_ids: str | None, -) -> Query[TMetricValueModel]: - """ - Apply ID-based filtering to queries. - - Isolate filter takes precedence over exclude filter. - - Args: - query: Query for metric values - isolate_ids: Comma-separated IDs to include exclusively - exclude_ids: Comma-separated IDs to exclude - - Returns - ------- - Tuple of (filtered_scalar_query, filtered_series_query) - """ - for key, value in filters.items(): - if hasattr(models.MetricValue, key): - col = getattr(models.ScalarMetricValue, key) - query = query.filter(build_filter_clause(col, value)) - - if isolate_ids: - ids = parse_id_list(isolate_ids) - query = query.filter(models.MetricValue.id.in_(ids)) - elif exclude_ids: - ids = parse_id_list(exclude_ids) - query = query.filter(~models.MetricValue.id.in_(ids)) - - return query - - -METRIC_VALUES_NON_FILTER_PARAMS = frozenset( - { - "format", - "value_type", - "offset", - "limit", - "detect_outliers", - "include_unverified", - "isolate_ids", - "exclude_ids", - } -) - - -def collect_facets_from_query(query: Query[TMetricValueModel]) -> list["Facet"]: - """Compute facet values from the full filtered query (before pagination). - - Uses ``DISTINCT`` per CV dimension column so the work scales with - the number of unique facet values rather than the total row count. - Returns a list of :class:`ref_backend.models.Facet` objects. - """ - from ref_backend.models import Facet # noqa: PLC0415 - - # Determine which CV dimension columns are registered - entity = query.column_descriptions[0]["entity"] - cv_dims: list[str] = getattr(entity, "_cv_dimensions", []) - - if not cv_dims: - return [] - - facets: list[Facet] = [] - - for key in cv_dims: - col = getattr(entity, key) - distinct_values = [ - value for (value,) in query.with_entities(col).order_by(None).distinct() if value is not None - ] - if distinct_values: - facets.append(Facet(key=key, values=distinct_values)) - - return facets - - -def process_scalar_values( - scalar_values: Sequence[models.ScalarMetricValue], - detect_outliers: Literal["off", "iqr"], - include_unverified: bool, -) -> tuple[list[AnnotatedScalarValue], bool, int, bool]: - """ - Process scalar values with optional outlier detection. - - Outlier detection runs on the **full** (unpaginated) set so that IQR - bounds are computed globally. When ``include_unverified`` is False the - flagged outliers are removed from the returned list. Callers are - responsible for paginating the result afterwards. - - Args: - scalar_values: List of scalar metric values (should be the full query result, not a paginated slice) - detect_outliers: Outlier detection method - include_unverified: Whether to include outlier values - - Returns - ------- - Tuple of (annotated_values, had_outliers, outlier_count, detection_ran) - """ - had_outliers = False - outlier_count = 0 - detection_ran = False - - if detect_outliers == "iqr" and scalar_values: - detection_ran = True - annotated_scalar_values, outlier_count = detect_outliers_in_scalar_values(scalar_values, factor=10.0) - had_outliers = outlier_count > 0 - if not include_unverified: - annotated_scalar_values = [item for item in annotated_scalar_values if not item.is_outlier] - else: - annotated_scalar_values = [AnnotatedScalarValue(value=v) for v in scalar_values] - - return annotated_scalar_values, had_outliers, outlier_count, detection_ran - - -def paginate_annotated_values( - values: list[AnnotatedScalarValue], - offset: int, - limit: int, -) -> list[AnnotatedScalarValue]: - """Return a slice of annotated values for the requested page.""" - return values[offset : offset + limit] - - -def generate_csv_response_scalar( - scalar_values: list[AnnotatedScalarValue], - detection_ran: bool, - had_outliers: bool, - outlier_count: int, - filename: str, -) -> StreamingResponse: - """ - Generate CSV streaming response for metric values. - - Args: - scalar_values: Processed scalar values with annotations - series_values: Series metric values - detection_ran: Whether outlier detection was performed - had_outliers: Whether outliers were detected - outlier_count: Number of outliers detected - filename: Filename for the CSV attachment - - Returns - ------- - StreamingResponse with CSV content - """ - - def generate_csv() -> Generator[str]: - output = io.StringIO() - writer = csv.writer(output) - - if not scalar_values: - yield "" - return - - # Write scalar values - dimensions = sorted(scalar_values[0].value.dimensions.keys()) - header = [*dimensions, "value", "type"] - if detection_ran: - header.extend(["is_outlier", "verification_status"]) - writer.writerow(header) - - for item in scalar_values: - mv = item.value - row = [mv.dimensions.get(d) for d in dimensions] + [ - sanitize_float_value(mv.value), - "scalar", - ] - if detection_ran: - row.extend([item.is_outlier, item.verification_status]) - writer.writerow(row) - - output.seek(0) - yield output.read() - - headers = {"Content-Disposition": f"attachment; filename={filename}"} - if detection_ran: - headers["X-REF-Had-Outliers"] = "true" if had_outliers else "false" - headers["X-REF-Outlier-Count"] = str(outlier_count) - - return StreamingResponse( - generate_csv(), - media_type="text/csv", - headers=headers, - ) - - -def generate_csv_response_series( - series_values: list[models.SeriesMetricValue], - detection_ran: bool, - had_outliers: bool, - outlier_count: int, - filename: str, -) -> StreamingResponse: - """ - Generate CSV streaming response for metric values. - - Args: - series_values: Series metric values - detection_ran: Whether outlier detection was performed - had_outliers: Whether outliers were detected - outlier_count: Number of outliers detected - filename: Filename for the CSV attachment - - Returns - ------- - StreamingResponse with CSV content - """ - - def generate_csv() -> Generator[str]: - output = io.StringIO() - writer = csv.writer(output) - - if not series_values: - yield "" - return - - # Write series values (flattened) - for sv in series_values: - dimensions = sorted(sv.dimensions.keys()) - # Write header if not already written - header = [*dimensions, "value", "index", "index_name", "type"] - writer.writerow(header) - - # Flatten series into multiple rows - for i, value in enumerate(sv.values): - index_value = sv.index[i] if sv.index and i < len(sv.index) else i - row = [sv.dimensions.get(d) for d in dimensions] + [ - sanitize_float_value(value), - index_value, - sv.index_name or "index", - "series", - ] - writer.writerow(row) - - output.seek(0) - yield output.read() - - headers = {"Content-Disposition": f"attachment; filename={filename}"} - if detection_ran: - headers["X-REF-Had-Outliers"] = "true" if had_outliers else "false" - headers["X-REF-Outlier-Count"] = str(outlier_count) - - return StreamingResponse( - generate_csv(), - media_type="text/csv", - headers=headers, - ) diff --git a/backend/src/ref_backend/core/outliers.py b/backend/src/ref_backend/core/outliers.py deleted file mode 100644 index c07ee09..0000000 --- a/backend/src/ref_backend/core/outliers.py +++ /dev/null @@ -1,189 +0,0 @@ -import math -import statistics -from collections.abc import Sequence -from typing import Literal - -import pandas as pd - -from climate_ref import models -from ref_backend.models import AnnotatedScalarValue - - -def flag_outliers_iqr(values: Sequence[float], factor: float = 5.0, min_n: int = 10) -> list[bool]: - """ - Flag outliers using the IQR method. - - Returns a list of booleans where True indicates an outlier. - """ - n = len(values) - if n < min_n: - return [False] * n - - # Compute Q1 and Q3 - quantiles = statistics.quantiles(values, n=4, method="inclusive") - q1, q3 = quantiles[0], quantiles[2] - iqr = q3 - q1 - - lower_bound = q1 - factor * iqr - upper_bound = q3 + factor * iqr - - return [v < lower_bound or v > upper_bound for v in values] - - -def calculate_iqr_bounds_by_source_id( - df: pd.DataFrame, factor: float = 3.0, min_n: int = 4 -) -> tuple[float, float] | None: - """ - Calculate IQR bounds using source_id means for equal model weighting. - - This function calculates mean value for each source_id and then - computes IQR bounds on these means, ensuring each model gets equal - weight regardless of number of ensemble members. - - Parameters - ---------- - df : pd.DataFrame - DataFrame containing scalar values with dimensions including source_id - factor : float - The factor to multiply IQR by to determine outlier bounds - min_n : int - Minimum number of source_ids required to perform outlier detection - - Returns - ------- - tuple[float, float] | None - Tuple of (lower_bound, upper_bound) or None if insufficient data - """ - # Check if source_id column exists - if "source_id" not in df.columns: - return None - - # Separate reference values (exclude from IQR calculation) using the - # first-class `kind` signal. A missing/None/empty kind is treated as a - # model. If the `kind` column is absent entirely (older data), fall back - # to treating all rows as models (no reference exclusion). - if "kind" in df.columns: - reference_mask = df["kind"] == "reference" - else: - reference_mask = pd.Series(False, index=df.index) - non_reference_df = df[~reference_mask] - - # Group by source_id and calculate mean for each - source_id_means = non_reference_df.groupby("source_id")["value"].mean() - - # Check if we have enough source_ids for outlier detection - if len(source_id_means) < min_n: - return None - - # Calculate IQR on source_id means - means_list = source_id_means.tolist() - quantiles = statistics.quantiles(means_list, n=4, method="inclusive") - q1, q3 = quantiles[0], quantiles[2] - iqr = q3 - q1 - - lower_bound = q1 - factor * iqr - upper_bound = q3 + factor * iqr - - return lower_bound, upper_bound - - -def detect_outliers_in_scalar_values( - scalar_values: Sequence[models.ScalarMetricValue], - factor: float = 3.0, - min_n: int = 4, - group_by: Sequence[str] = ("statistic", "metric"), -) -> tuple[list[AnnotatedScalarValue], int]: - """Detect outliers in scalar metric values grouped by stable diagnostic facets. - - This function uses source_id-aware outlier detection, where IQR bounds are calculated - using the mean value of each source_id rather than on all individual ensemble members. - This ensures each model gets equal weight regardless of number of ensemble members. - The calculated bounds are then applied to individual values for outlier detection. - - Parameters - ---------- - scalar_values - A list of scalar metric value objects to be analyzed. - factor - The factor to multiply the IQR by to determine the outlier bounds. - A value is an outlier if it is less than Q1 - factor * IQR or - greater than Q3 + factor * IQR. - - Defaults to 3.0. - min_n - The minimum number of source_ids required in a group to perform - IQR outlier detection. Defaults to 4. - group_by - A sequence of dimension names to group the `scalar_values` by before - performing outlier detection. Defaults to ("statistic", "metric"). - - Returns - ------- - tuple[list[AnnotatedScalarValue], int] - A tuple containing: - - A list of annotated values. Each item contains the original value and outlier info. - - The total count of detected outliers. - """ - # Group by stable diagnostic facets (exclude stoplist keys) - df = pd.DataFrame( - [{"scalar_value": sv, "value": sv.value, **sv.dimensions, "id": sv.id} for sv in scalar_values] - ) - annotated = [] - total_outliers = 0 - - group_by = [g for g in group_by if g in df.columns] - - for _, group_values in df.groupby(list(group_by)): - # Identify non-finite values (NaN, inf) as outliers - finite_flags = group_values.value.apply( - lambda x: isinstance(x, int | float) and not math.isinf(x) and not math.isnan(x) - ) - # Apply source_id-aware outlier detection if source_id exists - if "source_id" in group_values.columns and len(group_values) >= min_n: - iqr_bounds = calculate_iqr_bounds_by_source_id(group_values, factor=factor, min_n=min_n) - - if iqr_bounds is not None: - lower_bound, upper_bound = iqr_bounds - # Apply bounds to individual values (reference values always non-outlier) - source_id_flags = group_values.apply( - lambda row: ( - (row["value"] < lower_bound or row["value"] > upper_bound) - if row.get("kind") != "reference" - else False - ), - axis=1, - ) - else: - # Fallback if insufficient source_ids - source_id_flags = [False] * len(group_values) # type: ignore - else: - # Fallback to original IQR method if no source_id or insufficient data - if len(group_values) >= min_n: - iqr_flags = flag_outliers_iqr(group_values.value.to_list(), factor=factor) - # Reference values are always non-outlier, even on this fallback path. - if "kind" in group_values.columns: - iqr_flags = [ - False if kind == "reference" else flag - for flag, kind in zip(iqr_flags, group_values["kind"]) - ] - else: - iqr_flags = [False] * len(group_values) - source_id_flags = iqr_flags # type: ignore - - # Combine flags: item is outlier if flagged by source_id method OR non-finite - for sv, is_source_outlier, is_finite in zip(group_values.scalar_value, source_id_flags, finite_flags): - is_outlier = is_source_outlier or not is_finite - verification_status: Literal["verified", "unverified"] = ( - "unverified" if is_outlier else "verified" - ) - annotated.append( - AnnotatedScalarValue( - value=sv, - is_outlier=is_outlier, - verification_status=verification_status, - ) - ) - if is_outlier: - total_outliers += 1 - - return annotated, total_outliers diff --git a/backend/src/ref_backend/models.py b/backend/src/ref_backend/models.py index 98d9a68..75ed9bc 100644 --- a/backend/src/ref_backend/models.py +++ b/backend/src/ref_backend/models.py @@ -2,7 +2,6 @@ from datetime import datetime from typing import TYPE_CHECKING, ClassVar, Generic, Literal, TypeVar, Union, cast -from attr import define from loguru import logger from pydantic import BaseModel, HttpUrl, computed_field from sqlalchemy import func @@ -561,13 +560,6 @@ class Facet(BaseModel): NON_FACET_DIMENSIONS = frozenset({"kind"}) -@define -class AnnotatedScalarValue: - value: models.ScalarMetricValue - is_outlier: bool | None = None - verification_status: Literal["verified", "unverified"] | None = None - - _PRESENTATION_ATTRIBUTE_FALLBACKS: dict[str, tuple[str, ...]] = { "value_units": ("value_units", "units"), "value_long_name": ("value_long_name", "long_name"), diff --git a/backend/tests/test_core/test_metric_values.py b/backend/tests/test_core/test_metric_values.py index 4ff28fb..1164ffb 100644 --- a/backend/tests/test_core/test_metric_values.py +++ b/backend/tests/test_core/test_metric_values.py @@ -1,20 +1,12 @@ """Tests for ref_backend.core.metric_values module.""" -import asyncio -import csv -import io from unittest.mock import Mock import pytest from fastapi import HTTPException -from ref_backend.core.metric_values import ( - generate_csv_response_scalar, - generate_csv_response_series, - parse_id_list, - process_scalar_values, -) -from ref_backend.models import AnnotatedScalarValue, MetricValueCollection +from ref_backend.core.metric_values import parse_id_list +from ref_backend.models import MetricValueCollection class TestParseIdList: @@ -54,353 +46,6 @@ def test_mixed_valid_invalid(self): assert exc_info.value.status_code == 400 -class TestProcessScalarValues: - """Test the process_scalar_values function.""" - - def test_detect_outliers_iqr_with_values(self): - """Test with detect_outliers='iqr' and non-empty values.""" - # Need enough values (min_n=4) and varied enough for IQR detection - mock_values = [ - Mock(id=1, dimensions={"metric": "rmse"}, value=1.0), - Mock(id=2, dimensions={"metric": "rmse"}, value=2.0), - Mock(id=3, dimensions={"metric": "rmse"}, value=2.5), - Mock(id=4, dimensions={"metric": "rmse"}, value=3.0), - Mock(id=5, dimensions={"metric": "rmse"}, value=3.5), - Mock(id=6, dimensions={"metric": "rmse"}, value=4.0), - Mock(id=7, dimensions={"metric": "rmse"}, value=4.5), - Mock(id=8, dimensions={"metric": "rmse"}, value=5.0), - Mock(id=9, dimensions={"metric": "rmse"}, value=5.5), - Mock(id=10, dimensions={"metric": "rmse"}, value=6.0), - Mock(id=11, dimensions={"metric": "rmse"}, value=100.0), # outlier - ] - - annotated, had_outliers, outlier_count, detection_ran = process_scalar_values( - mock_values, detect_outliers="iqr", include_unverified=True - ) - - assert detection_ran is True - assert len(annotated) == 11 - assert all(isinstance(item, AnnotatedScalarValue) for item in annotated) - # The last value should be flagged as an outlier - assert annotated[10].is_outlier is True - assert outlier_count >= 1 - assert had_outliers is True - - def test_detect_outliers_off(self): - """Test with detect_outliers='off' returns plain wrappers.""" - mock_values = [ - Mock(dimensions={"metric": "rmse"}, value=1.0), - Mock(dimensions={"metric": "rmse"}, value=2.0), - ] - - annotated, had_outliers, outlier_count, detection_ran = process_scalar_values( - mock_values, detect_outliers="off", include_unverified=True - ) - - assert detection_ran is False - assert had_outliers is False - assert outlier_count == 0 - assert len(annotated) == 2 - # All should be plain wrappers without outlier flags - assert all(item.is_outlier is None for item in annotated) - - def test_include_unverified_false_filters_outliers(self): - """Test that include_unverified=False filters out outliers.""" - mock_values = [ - Mock(id=1, dimensions={"metric": "rmse"}, value=1.0), - Mock(id=2, dimensions={"metric": "rmse"}, value=2.0), - Mock(id=3, dimensions={"metric": "rmse"}, value=2.5), - Mock(id=4, dimensions={"metric": "rmse"}, value=3.0), - Mock(id=5, dimensions={"metric": "rmse"}, value=3.5), - Mock(id=6, dimensions={"metric": "rmse"}, value=4.0), - Mock(id=7, dimensions={"metric": "rmse"}, value=4.5), - Mock(id=8, dimensions={"metric": "rmse"}, value=5.0), - Mock(id=9, dimensions={"metric": "rmse"}, value=5.5), - Mock(id=10, dimensions={"metric": "rmse"}, value=6.0), - Mock(id=11, dimensions={"metric": "rmse"}, value=100.0), # outlier - ] - - annotated, had_outliers, outlier_count, detection_ran = process_scalar_values( - mock_values, detect_outliers="iqr", include_unverified=False - ) - - assert detection_ran is True - assert had_outliers is True - assert outlier_count >= 1 - # Outliers should be filtered out - assert len(annotated) < len(mock_values) - assert all(not item.is_outlier for item in annotated) - - def test_empty_input(self): - """Test that empty input returns empty list with detection_ran=False.""" - annotated, had_outliers, outlier_count, detection_ran = process_scalar_values( - [], detect_outliers="iqr", include_unverified=True - ) - - assert detection_ran is False - assert had_outliers is False - assert outlier_count == 0 - assert annotated == [] - - -class TestGenerateCsvResponseScalar: - """Test the generate_csv_response_scalar function.""" - - def test_non_empty_scalar_values(self): - """Test CSV generation with non-empty scalar values.""" - mock_value1 = Mock(dimensions={"region": "global", "metric": "rmse"}, value=1.5) - mock_value2 = Mock(dimensions={"region": "global", "metric": "bias"}, value=2.5) - - annotated_values = [ - AnnotatedScalarValue(value=mock_value1), - AnnotatedScalarValue(value=mock_value2), - ] - - response = generate_csv_response_scalar( - scalar_values=annotated_values, - detection_ran=False, - had_outliers=False, - outlier_count=0, - filename="test.csv", - ) - - # The body_iterator is created from the generator passed to StreamingResponse - # We need to call the internal generate_csv function directly - # Or use list comprehension to consume the sync generator before it's wrapped - - async def consume(): - content = "" - async for chunk in response.body_iterator: - content += chunk - return content - - content = asyncio.run(consume()) - reader = csv.reader(io.StringIO(content)) - rows = list(reader) - - # Check header (sorted dimensions + value + type) - assert rows[0] == ["metric", "region", "value", "type"] - - # Check data rows - assert rows[1] == ["rmse", "global", "1.5", "scalar"] - assert rows[2] == ["bias", "global", "2.5", "scalar"] - - # Check headers - assert response.headers["Content-Disposition"] == "attachment; filename=test.csv" - assert "X-REF-Had-Outliers" not in response.headers - - def test_with_detection_ran_includes_outlier_columns(self): - """Test that detection_ran=True adds outlier columns to CSV.""" - mock_value1 = Mock(dimensions={"region": "global"}, value=1.5) - mock_value2 = Mock(dimensions={"region": "global"}, value=100.0) - - annotated_values = [ - AnnotatedScalarValue(value=mock_value1, is_outlier=False, verification_status="verified"), - AnnotatedScalarValue(value=mock_value2, is_outlier=True, verification_status="unverified"), - ] - - response = generate_csv_response_scalar( - scalar_values=annotated_values, - detection_ran=True, - had_outliers=True, - outlier_count=1, - filename="test.csv", - ) - - # Consume the async streaming response - async def consume(): - content = "" - async for chunk in response.body_iterator: - content += chunk - return content - - content = asyncio.run(consume()) - reader = csv.reader(io.StringIO(content)) - rows = list(reader) - - # Check header includes outlier columns - assert rows[0] == ["region", "value", "type", "is_outlier", "verification_status"] - - # Check data rows - assert rows[1] == ["global", "1.5", "scalar", "False", "verified"] - assert rows[2] == ["global", "100.0", "scalar", "True", "unverified"] - - # Check custom headers - assert response.headers["X-REF-Had-Outliers"] == "true" - assert response.headers["X-REF-Outlier-Count"] == "1" - - def test_with_detection_ran_no_outliers(self): - """Test that detection_ran=True with no outliers sets headers correctly.""" - mock_value1 = Mock(dimensions={"region": "global"}, value=1.5) - - annotated_values = [ - AnnotatedScalarValue(value=mock_value1, is_outlier=False, verification_status="verified"), - ] - - response = generate_csv_response_scalar( - scalar_values=annotated_values, - detection_ran=True, - had_outliers=False, - outlier_count=0, - filename="test.csv", - ) - - # Check custom headers - assert response.headers["X-REF-Had-Outliers"] == "false" - assert response.headers["X-REF-Outlier-Count"] == "0" - - def test_empty_values(self): - """Test that empty values yields empty string.""" - response = generate_csv_response_scalar( - scalar_values=[], - detection_ran=False, - had_outliers=False, - outlier_count=0, - filename="test.csv", - ) - - # Consume the async streaming response - async def consume(): - content = "" - async for chunk in response.body_iterator: - content += chunk - return content - - content = asyncio.run(consume()) - assert content == "" - - -class TestGenerateCsvResponseSeries: - """Test the generate_csv_response_series function.""" - - def test_non_empty_series(self): - """Test CSV generation with non-empty series values.""" - mock_series1 = Mock( - dimensions={"region": "global", "metric": "temp"}, - values=[1.0, 2.0, 3.0], - index=[2020, 2021, 2022], - index_name="year", - ) - mock_series2 = Mock( - dimensions={"region": "north", "metric": "temp"}, - values=[4.0, 5.0], - index=[2020, 2021], - index_name="year", - ) - - response = generate_csv_response_series( - series_values=[mock_series1, mock_series2], - detection_ran=False, - had_outliers=False, - outlier_count=0, - filename="test.csv", - ) - - # Consume the async streaming response - async def consume(): - content = "" - async for chunk in response.body_iterator: - content += chunk - return content - - content = asyncio.run(consume()) - reader = csv.reader(io.StringIO(content)) - rows = list(reader) - - # First series header - assert rows[0] == ["metric", "region", "value", "index", "index_name", "type"] - - # First series data (flattened) - assert rows[1] == ["temp", "global", "1.0", "2020", "year", "series"] - assert rows[2] == ["temp", "global", "2.0", "2021", "year", "series"] - assert rows[3] == ["temp", "global", "3.0", "2022", "year", "series"] - - # Second series header - assert rows[4] == ["metric", "region", "value", "index", "index_name", "type"] - - # Second series data (flattened) - assert rows[5] == ["temp", "north", "4.0", "2020", "year", "series"] - assert rows[6] == ["temp", "north", "5.0", "2021", "year", "series"] - - def test_series_without_index(self): - """Test series values without explicit index use position.""" - mock_series = Mock( - dimensions={"region": "global"}, - values=[10.0, 20.0, 30.0], - index=None, - index_name=None, - ) - - response = generate_csv_response_series( - series_values=[mock_series], - detection_ran=False, - had_outliers=False, - outlier_count=0, - filename="test.csv", - ) - - # Consume the async streaming response - async def consume(): - content = "" - async for chunk in response.body_iterator: - content += chunk - return content - - content = asyncio.run(consume()) - reader = csv.reader(io.StringIO(content)) - rows = list(reader) - - # Header is: region, value, index, index_name, type - # So indices are in column 2 (0-indexed) - assert rows[1][2] == "0" # index column - assert rows[2][2] == "1" - assert rows[3][2] == "2" - # Check index_name defaults to "index" (column 3) - assert rows[1][3] == "index" - - def test_empty_series(self): - """Test that empty series yields empty string.""" - response = generate_csv_response_series( - series_values=[], - detection_ran=False, - had_outliers=False, - outlier_count=0, - filename="test.csv", - ) - - # Consume the async streaming response - async def consume(): - content = "" - async for chunk in response.body_iterator: - content += chunk - return content - - content = asyncio.run(consume()) - assert content == "" - - def test_series_with_detection_headers(self): - """Test that detection headers are set for series.""" - mock_series = Mock( - dimensions={"region": "global"}, - values=[1.0, 2.0], - index=[2020, 2021], - index_name="year", - ) - - response = generate_csv_response_series( - series_values=[mock_series], - detection_ran=True, - had_outliers=True, - outlier_count=5, - filename="test.csv", - ) - - # Check custom headers - assert response.headers["X-REF-Had-Outliers"] == "true" - assert response.headers["X-REF-Outlier-Count"] == "5" - assert response.headers["Content-Disposition"] == "attachment; filename=test.csv" - - def _reader_collection(items): """Build a stand-in reader collection (only the fields the builders read).""" return Mock(items=items, total_count=len(items), facets=[], had_outliers=False, outlier_count=0) diff --git a/backend/tests/test_core/test_outliers.py b/backend/tests/test_core/test_outliers.py deleted file mode 100644 index 16bf7d8..0000000 --- a/backend/tests/test_core/test_outliers.py +++ /dev/null @@ -1,325 +0,0 @@ -from unittest.mock import Mock - -import pandas as pd - -from ref_backend.core.outliers import ( - calculate_iqr_bounds_by_source_id, - detect_outliers_in_scalar_values, - flag_outliers_iqr, -) - - -class TestFlagOutliersIQR: - """Test the IQR-based outlier detection function.""" - - def test_outlier_detection_with_factor_3(self): - """Test outlier detection with factor=3.0 on controlled data.""" - values = [1, 2, 2, 2, 100] - result = flag_outliers_iqr(values, factor=3.0, min_n=4) - expected = [True, False, False, False, True] - assert result == expected - - def test_outlier_detection_with_factor_1_5(self): - """Test outlier detection with factor=1.5 on controlled data. - - Both factor=3.0 and factor=1.5 should flag 100 as an outlier because: - - Q1 = 2, Q3 = 2, IQR = 0 - - Lower bound = Q1 - factor * IQR = 2 - 1.5 * 0 = 2 - - Upper bound = Q3 + factor * IQR = 2 + 1.5 * 0 = 2 - - Any value outside [2, 2] is an outlier, so 100 is flagged - """ - values = [1, 2, 2, 2, 100] - result = flag_outliers_iqr(values, factor=1.5, min_n=4) - expected = [True, False, False, False, True] - assert result == expected - - def test_insufficient_data_points(self): - """Test that len(values) < min_n returns all False.""" - values = [1, 2, 2] # len < 4 - result = flag_outliers_iqr(values, factor=3.0) - expected = [False, False, False] - assert result == expected - - def test_zero_iqr_case(self): - """Test case where IQR == 0 returns all False.""" - values = [5, 5, 5, 5, 5] - result = flag_outliers_iqr(values, factor=3.0) - expected = [False, False, False, False, False] - assert result == expected - - def test_min_n_parameter(self): - """Test that min_n=4 behavior is validated.""" - # With min_n=4, values with len < 4 should return all False - values = [1, 2, 2, 2, 100] - result = flag_outliers_iqr(values, factor=3.0, min_n=4) - expected = [True, False, False, False, True] - assert result == expected - - # With min_n=6, this should return all False since len(values) < 6 - result = flag_outliers_iqr(values, factor=3.0, min_n=6) - expected = [False, False, False, False, False] - assert result == expected - - def test_normal_outlier_detection(self): - """Test normal outlier detection with varied data.""" - values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 100] - result = flag_outliers_iqr(values, factor=1.5) - # Q1 = 3.25, Q3 = 7.75, IQR = 4.5 - # Lower bound = 3.25 - 1.5 * 4.5 = 3.25 - 6.75 = -3.5 - # Upper bound = 7.75 + 1.5 * 4.5 = 7.75 + 6.75 = 14.5 - # So 100 is an outlier - expected = [False] * 9 + [True] - assert result == expected - - -class TestCalculateIqrBoundsBySourceId: - """Test the kind-aware IQR bound calculation.""" - - def test_excludes_rows_tagged_kind_reference(self): - """Rows with kind == 'reference' are excluded from the IQR calculation, - regardless of their source_id.""" - df = pd.DataFrame( - [ - {"source_id": "model-a", "kind": "model", "value": 1.0}, - {"source_id": "model-b", "kind": "model", "value": 2.0}, - {"source_id": "model-c", "kind": "model", "value": 3.0}, - {"source_id": "model-d", "kind": "model", "value": 4.0}, - # A reference row with an extreme value that must not skew the bounds. - {"source_id": "model-e", "kind": "reference", "value": 1000.0}, - ] - ) - bounds_with_reference = calculate_iqr_bounds_by_source_id(df, factor=3.0, min_n=4) - - df_without_reference = df[df["kind"] != "reference"] - bounds_without_reference = calculate_iqr_bounds_by_source_id( - df_without_reference, factor=3.0, min_n=4 - ) - - assert bounds_with_reference == bounds_without_reference - - def test_missing_kind_column_falls_back_to_no_exclusion(self): - """When the kind column is absent entirely (older data), no rows are - treated as reference — fall back to treating all rows as models.""" - df = pd.DataFrame( - [ - {"source_id": "model-a", "value": 1.0}, - {"source_id": "model-b", "value": 2.0}, - {"source_id": "model-c", "value": 3.0}, - {"source_id": "model-d", "value": 4.0}, - ] - ) - bounds = calculate_iqr_bounds_by_source_id(df, factor=3.0, min_n=4) - assert bounds is not None - - def test_none_or_empty_kind_treated_as_model(self): - """A None or empty kind is treated as a model, not a reference.""" - df = pd.DataFrame( - [ - {"source_id": "model-a", "kind": None, "value": 1.0}, - {"source_id": "model-b", "kind": "", "value": 2.0}, - {"source_id": "model-c", "kind": "model", "value": 3.0}, - {"source_id": "model-d", "kind": "model", "value": 4.0}, - ] - ) - bounds = calculate_iqr_bounds_by_source_id(df, factor=3.0, min_n=4) - # All 4 source_ids should participate (none excluded as reference). - assert bounds is not None - - -class TestDetectOutliersInScalarValues: - """Test the detect_outliers_in_scalar_values function with grouping and auto-flags.""" - - def test_auto_flag_non_finite_values(self): - """Test that non-finite values are auto-flagged as outliers.""" - mock_values = [ - Mock(dimensions={"metric": "rmse", "region": "global"}, value=1.0), - Mock(dimensions={"metric": "rmse", "region": "global"}, value=float("nan")), - Mock(dimensions={"metric": "rmse", "region": "global"}, value=float("inf")), - ] - - annotated, outlier_count = detect_outliers_in_scalar_values(mock_values) # type: ignore - flags = [item.is_outlier for item in annotated] - - assert flags == [False, True, True] - assert outlier_count == 2 - - def test_reference_rows_excluded_and_always_non_outlier(self): - """Rows tagged kind='reference' are excluded from IQR-bound computation - and are always flagged as non-outlier, regardless of source_id.""" - mock_values = [ - Mock( - dimensions={ - "metric": "rmse", - "statistic": "mean", - "source_id": f"model-{i}", - "kind": "model", - }, - value=float(i), - id=i, - ) - for i in range(4) - ] + [ - Mock( - dimensions={ - "metric": "rmse", - "statistic": "mean", - "source_id": "obs-source", - "kind": "reference", - }, - value=1000.0, - id=100, - ) - ] - - annotated, _ = detect_outliers_in_scalar_values(mock_values, min_n=4) - by_id = {item.value.id: item for item in annotated} - - assert by_id[100].is_outlier is False - assert by_id[100].verification_status == "verified" - - def test_reference_row_non_outlier_on_fallback_path_without_source_id(self): - """Rows tagged kind='reference' stay non-outlier even on the fallback - IQR path taken when no source_id dimension is present. Without the - kind override on that path, the extreme reference value would be - flagged an outlier.""" - mock_values = [ - Mock( - dimensions={"metric": "rmse", "statistic": "mean", "kind": "model"}, - value=float(i), - id=i, - ) - for i in range(4) - ] + [ - Mock( - dimensions={"metric": "rmse", "statistic": "mean", "kind": "reference"}, - value=1000.0, - id=100, - ) - ] - - annotated, _ = detect_outliers_in_scalar_values(mock_values, min_n=4) - by_id = {item.value.id: item for item in annotated} - - assert by_id[100].is_outlier is False - assert by_id[100].verification_status == "verified" - - def test_source_id_reference_sentinel_with_kind_model_is_treated_as_model(self): - """Regression: a row with source_id == 'Reference' but kind == 'model' - is now treated as a model — it participates in IQR and can be flagged - an outlier. This proves the source_id sentinel is gone.""" - mock_values = [ - Mock( - dimensions={"metric": "rmse", "statistic": "mean", "source_id": "model-a", "kind": "model"}, - value=1.0, - id=0, - ), - Mock( - dimensions={"metric": "rmse", "statistic": "mean", "source_id": "model-b", "kind": "model"}, - value=2.0, - id=1, - ), - Mock( - dimensions={"metric": "rmse", "statistic": "mean", "source_id": "model-c", "kind": "model"}, - value=3.0, - id=2, - ), - Mock( - dimensions={"metric": "rmse", "statistic": "mean", "source_id": "model-d", "kind": "model"}, - value=4.0, - id=3, - ), - # Sentinel source_id, but kind says it's actually a model — should - # be an extreme outlier that participates in (and blows up) the IQR bounds. - Mock( - dimensions={"metric": "rmse", "statistic": "mean", "source_id": "Reference", "kind": "model"}, - value=1000.0, - id=4, - ), - ] - - annotated, outlier_count = detect_outliers_in_scalar_values(mock_values, factor=3.0, min_n=4) - by_id = {item.value.id: item for item in annotated} - - assert by_id[4].is_outlier is True - assert by_id[4].verification_status == "unverified" - assert outlier_count == 1 - - def test_arbitrary_source_id_with_kind_reference_is_treated_as_reference(self): - """A row with source_id == 'ACCESS-CM2' but kind == 'reference' is - treated as a reference: excluded from IQR computation and always - flagged non-outlier.""" - mock_values = [ - Mock( - dimensions={"metric": "rmse", "statistic": "mean", "source_id": "model-a", "kind": "model"}, - value=1.0, - id=0, - ), - Mock( - dimensions={"metric": "rmse", "statistic": "mean", "source_id": "model-b", "kind": "model"}, - value=2.0, - id=1, - ), - Mock( - dimensions={"metric": "rmse", "statistic": "mean", "source_id": "model-c", "kind": "model"}, - value=2.0, - id=2, - ), - Mock( - dimensions={"metric": "rmse", "statistic": "mean", "source_id": "model-d", "kind": "model"}, - value=2.0, - id=3, - ), - # Real model source_id, but kind says it's a reference — should be - # excluded from IQR and always non-outlier despite the extreme value. - Mock( - dimensions={ - "metric": "rmse", - "statistic": "mean", - "source_id": "ACCESS-CM2", - "kind": "reference", - }, - value=1000.0, - id=4, - ), - ] - - annotated, outlier_count = detect_outliers_in_scalar_values(mock_values, factor=3.0, min_n=4) - by_id = {item.value.id: item for item in annotated} - - assert by_id[4].is_outlier is False - assert by_id[4].verification_status == "verified" - assert outlier_count == 0 - - def test_plain_model_rows_outlier_behaviour_unchanged(self): - """Existing outlier behaviour for plain model rows (kind='model') still - holds: an extreme model value is flagged an outlier.""" - mock_values = [ - Mock( - dimensions={ - "metric": "rmse", - "statistic": "mean", - "source_id": f"model-{i}", - "kind": "model", - }, - value=2.0, - id=i, - ) - for i in range(4) - ] + [ - Mock( - dimensions={ - "metric": "rmse", - "statistic": "mean", - "source_id": "model-outlier", - "kind": "model", - }, - value=1000.0, - id=100, - ) - ] - - annotated, outlier_count = detect_outliers_in_scalar_values(mock_values, factor=3.0, min_n=4) - by_id = {item.value.id: item for item in annotated} - - assert by_id[100].is_outlier is True - assert outlier_count == 1 diff --git a/changelog/+delete-dead-metric-values.trivial.md b/changelog/+delete-dead-metric-values.trivial.md new file mode 100644 index 0000000..75e211d --- /dev/null +++ b/changelog/+delete-dead-metric-values.trivial.md @@ -0,0 +1 @@ +Removed the unused pre-Reader metric-value helpers and their tests. From 4a6411d74546cefda1c7ec6c63b943070919d2b9 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Mon, 20 Jul 2026 13:26:34 +1000 Subject: [PATCH 2/2] docs(changelog): number the fragment for PR #42 --- .../{+delete-dead-metric-values.trivial.md => 42.trivial.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changelog/{+delete-dead-metric-values.trivial.md => 42.trivial.md} (100%) diff --git a/changelog/+delete-dead-metric-values.trivial.md b/changelog/42.trivial.md similarity index 100% rename from changelog/+delete-dead-metric-values.trivial.md rename to changelog/42.trivial.md