From 62287eb232d2d77a197cd754b0bab869a8aaa9ac Mon Sep 17 00:00:00 2001 From: sray014 Date: Thu, 6 Aug 2026 11:39:48 -0400 Subject: [PATCH 1/3] feat: Add NWS Local Storm Report reaper --- src/cosecha/__init__.py | 2 + src/cosecha/reaping/__init__.py | 2 + src/cosecha/reaping/lsr.py | 220 +++++++++++++++++++++ tests/test_reaping/test_lsr_reapers.py | 264 +++++++++++++++++++++++++ 4 files changed, 488 insertions(+) create mode 100644 src/cosecha/reaping/lsr.py create mode 100644 tests/test_reaping/test_lsr_reapers.py diff --git a/src/cosecha/__init__.py b/src/cosecha/__init__.py index d985f58..db1e440 100644 --- a/src/cosecha/__init__.py +++ b/src/cosecha/__init__.py @@ -9,6 +9,7 @@ from cosecha.reaping import ( ASOSReaper, GriddedReaper, + LSRReaper, MRMSReaper, NWPReaper, TimeSeriesReaper, @@ -23,6 +24,7 @@ __all__ = [ "ASOSReaper", "GriddedReaper", + "LSRReaper", "MRMSReaper", "NWPReaper", "TimeSeriesReaper", diff --git a/src/cosecha/reaping/__init__.py b/src/cosecha/reaping/__init__.py index b3193c9..77e1390 100644 --- a/src/cosecha/reaping/__init__.py +++ b/src/cosecha/reaping/__init__.py @@ -8,6 +8,7 @@ from cosecha.reaping.asos import ASOSReaper from cosecha.reaping.base import GriddedReaper, TimeSeriesReaper +from cosecha.reaping.lsr import LSRReaper from cosecha.reaping.mrms import MRMSReaper from cosecha.reaping.nwis import USGSNWISReaper from cosecha.reaping.nwp import NWPReaper @@ -16,6 +17,7 @@ __all__ = [ "ASOSReaper", "GriddedReaper", + "LSRReaper", "MRMSReaper", "NWPReaper", "ReservoirReaper", diff --git a/src/cosecha/reaping/lsr.py b/src/cosecha/reaping/lsr.py new file mode 100644 index 0000000..b650aa2 --- /dev/null +++ b/src/cosecha/reaping/lsr.py @@ -0,0 +1,220 @@ +"""NWS Local Storm Reports (LSR) reaper. + +This module provides an implementation for harvesting NWS Local Storm Reports +from the Iowa Environmental Mesonet (IEM) GeoJSON API at +https://mesonet.agron.iastate.edu/geojson/lsr.php +""" + +from __future__ import annotations + +from typing import Any + +import pandas as pd +import tiny_retriever + +from cosecha._logging import logger +from cosecha._utils import apply_ts_transformations, wrap_errors +from cosecha.exceptions import APIError, DataNotFoundError, DateRangeError +from cosecha.reaping.base import TimeSeriesReaper + +__all__ = [ + "LSRReaper", +] + +BASE_URL = "https://mesonet.agron.iastate.edu/geojson/lsr.php" + +FLOOD_EVENT_TYPES = [ + "HEAVY RAIN", + "HEAVY SNOW", + "DEBRIS FLOW", + "FLASH FLOOD", + "FLOOD", +] + + +class LSRReaper(TimeSeriesReaper): + """Reaper for NWS Local Storm Reports via the IEM GeoJSON endpoint. + + Fetches storm report features for a given time window, optionally filtering + by WFO (Weather Forecast Office), event type, and state. + """ + + def _validate_params(self) -> None: + """Validate initialization parameters. + + Raises + ------ + DateRangeError + If dates are invalid. + """ + if self.start_date >= self.end_date: + raise DateRangeError( + f"start_date ({self.start_date}) must be < end_date ({self.end_date})" + ) + + def __init__( + self, + start_date: str, + end_date: str, + wfos: list[str] | None = None, + event_types: list[str] | None = None, + state: str | None = None, + transformations: dict[str, Any] | None = None, + timeout: int = 120, + ) -> None: + """Fetch NWS Local Storm Reports from the IEM GeoJSON API. + + Parameters + ---------- + start_date : str + Start datetime in ISO 8601 format (e.g., "2026-07-01T00:00:00Z"). + end_date : str + End datetime in ISO 8601 format (e.g., "2026-07-02T00:00:00Z"). + wfos : list[str] | None, optional + List of Weather Forecast Office codes to query (e.g., ["BOU", "GJT"]). + If None, fetches reports from all WFOs. + event_types : list[str] | None, optional + Event types to include (e.g., ["FLASH FLOOD", "HEAVY RAIN"]). + If None, returns all event types. Use ``FLOOD_EVENT_TYPES`` for + a convenient preset of flood-related types. + state : str | None, optional + Two-letter state abbreviation to filter results (e.g., "CO"). + If None, no state filtering is applied. + transformations : dict[str, Any], optional + Optional transformations to apply to the resulting DataFrame. + timeout : int, optional + Request timeout in seconds, by default 120. + + Examples + -------- + >>> reaper = LSRReaper( + ... start_date="2026-07-01T00:00:00Z", + ... end_date="2026-07-02T00:00:00Z", + ... wfos=["BOU", "GJT"], + ... event_types=["FLASH FLOOD", "HEAVY RAIN"], + ... state="CO", + ... ) + >>> data = reaper.reap() + """ + super().__init__() + + try: + self.start_date = pd.to_datetime(start_date) + self.end_date = pd.to_datetime(end_date) + except Exception as e: + raise DateRangeError(f"Could not parse date: {e}") from e + + self.wfos = [w.upper().strip() for w in wfos] if wfos else None + self.event_types = [et.upper().strip() for et in event_types] if event_types else None + self.state = state.upper().strip() if state else None + self.transformations = transformations + self.timeout = timeout + + self._validate_params() + logger.debug( + f"Initialized {self.__class__.__name__}: " + f"wfos={self.wfos}, event_types={self.event_types}, state={self.state}, " + f"dates={self.start_date} to {self.end_date}" + ) + + def _build_url(self) -> str: + """Build the IEM LSR GeoJSON request URL.""" + sts = self.start_date.strftime("%Y%m%d%H%M") + ets = self.end_date.strftime("%Y%m%d%H%M") + url = f"{BASE_URL}?sts={sts}&ets={ets}" + if self.wfos: + url += f"&wfos={','.join(self.wfos)}" + return url + + def _fetch(self, url: str) -> dict: + """Fetch GeoJSON from IEM LSR endpoint.""" + with wrap_errors(APIError, "Failed to fetch LSR data from IEM"): + return tiny_retriever.fetch(url, "json", timeout=self.timeout) + + def _parse_features(self, geojson: dict) -> pd.DataFrame: + """Parse GeoJSON features into a DataFrame, applying filters. + + Parameters + ---------- + geojson : dict + GeoJSON FeatureCollection from the IEM LSR endpoint. + + Returns + ------- + pd.DataFrame + Filtered storm report records. + + Raises + ------ + DataNotFoundError + If no features match the filters. + """ + features = geojson.get("features", []) + if not features: + raise DataNotFoundError( + f"LSR returned no features for {self.start_date} to {self.end_date}" + ) + + records = [] + for feature in features: + props = feature.get("properties", {}) + geom = feature.get("geometry", {}) + + # Filter by event type + event_type = (props.get("typetext") or "").upper().strip() + if not event_type: + continue + if self.event_types and event_type not in self.event_types: + continue + + # Filter by state + if self.state: + event_state = (props.get("st") or "").upper().strip() + if event_state != self.state: + continue + + # Extract coordinates from geometry + coords = geom.get("coordinates", [None, None]) + + records.append({ + "valid": props.get("valid"), + "event_type": event_type, + "magnitude": props.get("magnitude"), + "unit": props.get("unit"), + "wfo": props.get("wfo"), + "county": props.get("county"), + "state": props.get("st"), + "city": props.get("city"), + "source": props.get("source"), + "remark": props.get("remark"), + "longitude": coords[0] if coords else None, + "latitude": coords[1] if len(coords) > 1 else None, + }) + + if not records: + raise DataNotFoundError( + f"No LSR features matched filters (event_types={self.event_types}, " + f"state={self.state}) for {self.start_date} to {self.end_date}" + ) + + df = pd.DataFrame(records) + df["valid"] = pd.to_datetime(df["valid"], errors="coerce") + logger.debug(f"Parsed {len(df)} storm reports matching filters") + return df + + def _reap(self) -> pd.DataFrame: + """Fetch and parse NWS Local Storm Reports.""" + logger.info( + f"Reaping LSR data: wfos={self.wfos or 'all'}, " + f"event_types={self.event_types}, state={self.state or 'all'}" + ) + + url = self._build_url() + geojson = self._fetch(url) + df = self._parse_features(geojson) + + if self.transformations and not df.empty: + df = apply_ts_transformations(df, self.transformations) + + logger.info(f"Reaped {len(df)} storm reports") + return df diff --git a/tests/test_reaping/test_lsr_reapers.py b/tests/test_reaping/test_lsr_reapers.py new file mode 100644 index 0000000..6892f02 --- /dev/null +++ b/tests/test_reaping/test_lsr_reapers.py @@ -0,0 +1,264 @@ +"""Tests for LSR reaper.""" + +from __future__ import annotations + +from unittest.mock import patch + +import pandas as pd +import pytest + +from cosecha.exceptions import APIError, DataNotFoundError, DateRangeError +from cosecha.reaping.lsr import LSRReaper + + +SAMPLE_GEOJSON = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [-105.0, 39.7]}, + "properties": { + "valid": "2026-07-01T12:00:00Z", + "typetext": "FLASH FLOOD", + "magnitude": None, + "unit": None, + "wfo": "BOU", + "county": "Denver", + "st": "CO", + "city": "Denver", + "source": "Emergency Manager", + "remark": "Water over road", + }, + }, + { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [-104.5, 39.5]}, + "properties": { + "valid": "2026-07-01T14:00:00Z", + "typetext": "HEAVY RAIN", + "magnitude": 2.5, + "unit": "INCH", + "wfo": "BOU", + "county": "Arapahoe", + "st": "CO", + "city": "Aurora", + "source": "CoCoRaHS", + "remark": "Heavy rainfall", + }, + }, + { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [-106.0, 40.0]}, + "properties": { + "valid": "2026-07-01T15:00:00Z", + "typetext": "HAIL", + "magnitude": 1.0, + "unit": "INCH", + "wfo": "BOU", + "county": "Boulder", + "st": "CO", + "city": "Boulder", + "source": "Public", + "remark": "Golf ball hail", + }, + }, + { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [-102.0, 37.5]}, + "properties": { + "valid": "2026-07-01T16:00:00Z", + "typetext": "FLASH FLOOD", + "magnitude": None, + "unit": None, + "wfo": "GLD", + "county": "Baca", + "st": "KS", + "city": "Springfield", + "source": "Law Enforcement", + "remark": "Water over road", + }, + }, + ], +} + + +class TestLSRReaper: + """Tests for LSRReaper.""" + + def test_initialization_valid(self): + """Test valid initialization with all parameters.""" + reaper = LSRReaper( + start_date="2026-07-01T00:00:00Z", + end_date="2026-07-02T00:00:00Z", + wfos=["BOU", "GJT"], + event_types=["FLASH FLOOD", "HEAVY RAIN"], + state="CO", + ) + assert reaper.start_date == pd.Timestamp("2026-07-01T00:00:00Z") + assert reaper.end_date == pd.Timestamp("2026-07-02T00:00:00Z") + assert reaper.wfos == ["BOU", "GJT"] + assert reaper.event_types == ["FLASH FLOOD", "HEAVY RAIN"] + assert reaper.state == "CO" + + def test_initialization_defaults(self): + """Test initialization with default parameters.""" + reaper = LSRReaper( + start_date="2026-07-01", + end_date="2026-07-02", + ) + assert reaper.wfos is None + assert reaper.state is None + assert reaper.event_types is None + + def test_invalid_date_range(self): + """Test initialization fails with start >= end.""" + with pytest.raises(DateRangeError): + LSRReaper( + start_date="2026-07-02", + end_date="2026-07-01", + ) + + def test_invalid_date_format(self): + """Test initialization fails with unparsable date.""" + with pytest.raises(DateRangeError, match="Could not parse date"): + LSRReaper( + start_date="not-a-date", + end_date="2026-07-02", + ) + + def test_build_url_with_wfos(self): + """Test URL construction includes WFOs.""" + reaper = LSRReaper( + start_date="2026-07-01T00:00:00Z", + end_date="2026-07-02T00:00:00Z", + wfos=["BOU", "GJT"], + ) + url = reaper._build_url() + assert "sts=202607010000" in url + assert "ets=202607020000" in url + assert "wfos=BOU,GJT" in url + + def test_build_url_without_wfos(self): + """Test URL construction without WFOs.""" + reaper = LSRReaper( + start_date="2026-07-01T00:00:00Z", + end_date="2026-07-02T00:00:00Z", + ) + url = reaper._build_url() + assert "wfos" not in url + + @patch("cosecha.reaping.lsr.tiny_retriever.fetch") + def test_reap_success(self, mock_fetch): + """Test successful data retrieval with filters.""" + mock_fetch.return_value = SAMPLE_GEOJSON + + reaper = LSRReaper( + start_date="2026-07-01T00:00:00Z", + end_date="2026-07-02T00:00:00Z", + wfos=["BOU"], + event_types=["FLASH FLOOD", "HEAVY RAIN"], + state="CO", + ) + df = reaper.reap() + + assert isinstance(df, pd.DataFrame) + # HAIL and KS features should be filtered out + assert len(df) == 2 + assert set(df["event_type"]) == {"FLASH FLOOD", "HEAVY RAIN"} + assert all(df["state"] == "CO") + + @patch("cosecha.reaping.lsr.tiny_retriever.fetch") + def test_reap_no_state_filter(self, mock_fetch): + """Test retrieval without state filter returns more results.""" + mock_fetch.return_value = SAMPLE_GEOJSON + + reaper = LSRReaper( + start_date="2026-07-01T00:00:00Z", + end_date="2026-07-02T00:00:00Z", + event_types=["FLASH FLOOD", "HEAVY RAIN"], + ) + df = reaper.reap() + + assert len(df) == 3 # Both CO and KS flash floods + heavy rain + + @patch("cosecha.reaping.lsr.tiny_retriever.fetch") + def test_reap_all_event_types(self, mock_fetch): + """Test retrieval with no event_types filter returns all types.""" + mock_fetch.return_value = SAMPLE_GEOJSON + + reaper = LSRReaper( + start_date="2026-07-01T00:00:00Z", + end_date="2026-07-02T00:00:00Z", + ) + df = reaper.reap() + + assert len(df) == 4 # All features including HAIL + assert set(df["event_type"]) == {"FLASH FLOOD", "HEAVY RAIN", "HAIL"} + + @patch("cosecha.reaping.lsr.tiny_retriever.fetch") + def test_reap_empty_features(self, mock_fetch): + """Test raises DataNotFoundError when no features returned.""" + mock_fetch.return_value = {"type": "FeatureCollection", "features": []} + + reaper = LSRReaper( + start_date="2026-07-01T00:00:00Z", + end_date="2026-07-02T00:00:00Z", + ) + with pytest.raises(DataNotFoundError, match="no features"): + reaper.reap() + + @patch("cosecha.reaping.lsr.tiny_retriever.fetch") + def test_reap_no_matching_features(self, mock_fetch): + """Test raises DataNotFoundError when no features match filters.""" + mock_fetch.return_value = SAMPLE_GEOJSON + + reaper = LSRReaper( + start_date="2026-07-01T00:00:00Z", + end_date="2026-07-02T00:00:00Z", + event_types=["TORNADO"], + ) + with pytest.raises(DataNotFoundError, match="No LSR features matched"): + reaper.reap() + + @patch("cosecha.reaping.lsr.tiny_retriever.fetch") + def test_reap_api_error(self, mock_fetch): + """Test wraps fetch errors as APIError.""" + mock_fetch.side_effect = Exception("Connection timeout") + + reaper = LSRReaper( + start_date="2026-07-01T00:00:00Z", + end_date="2026-07-02T00:00:00Z", + ) + with pytest.raises(APIError, match="Failed to fetch LSR data"): + reaper.reap() + + @patch("cosecha.reaping.lsr.tiny_retriever.fetch") + def test_reap_with_transformations(self, mock_fetch): + """Test transformations are applied to result.""" + mock_fetch.return_value = SAMPLE_GEOJSON + + reaper = LSRReaper( + start_date="2026-07-01T00:00:00Z", + end_date="2026-07-02T00:00:00Z", + event_types=["FLASH FLOOD", "HEAVY RAIN"], + state="CO", + transformations={"rename_columns": {"event_type": "type"}}, + ) + df = reaper.reap() + assert "type" in df.columns + assert "event_type" not in df.columns + + @patch("cosecha.reaping.lsr.tiny_retriever.fetch") + def test_coordinates_parsed(self, mock_fetch): + """Test lat/lon are extracted from geometry.""" + mock_fetch.return_value = SAMPLE_GEOJSON + + reaper = LSRReaper( + start_date="2026-07-01T00:00:00Z", + end_date="2026-07-02T00:00:00Z", + event_types=["FLASH FLOOD"], + state="CO", + ) + df = reaper.reap() + assert df.iloc[0]["longitude"] == -105.0 + assert df.iloc[0]["latitude"] == 39.7 From 07fd473d58690cfb7de76e8c7180e301a8cde517 Mon Sep 17 00:00:00 2001 From: sray014 Date: Thu, 6 Aug 2026 11:41:48 -0400 Subject: [PATCH 2/3] chore: Remove unneeded list --- src/cosecha/reaping/lsr.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/cosecha/reaping/lsr.py b/src/cosecha/reaping/lsr.py index b650aa2..9be4755 100644 --- a/src/cosecha/reaping/lsr.py +++ b/src/cosecha/reaping/lsr.py @@ -23,14 +23,6 @@ BASE_URL = "https://mesonet.agron.iastate.edu/geojson/lsr.php" -FLOOD_EVENT_TYPES = [ - "HEAVY RAIN", - "HEAVY SNOW", - "DEBRIS FLOW", - "FLASH FLOOD", - "FLOOD", -] - class LSRReaper(TimeSeriesReaper): """Reaper for NWS Local Storm Reports via the IEM GeoJSON endpoint. @@ -75,8 +67,7 @@ def __init__( If None, fetches reports from all WFOs. event_types : list[str] | None, optional Event types to include (e.g., ["FLASH FLOOD", "HEAVY RAIN"]). - If None, returns all event types. Use ``FLOOD_EVENT_TYPES`` for - a convenient preset of flood-related types. + If None, returns all event types. state : str | None, optional Two-letter state abbreviation to filter results (e.g., "CO"). If None, no state filtering is applied. From 2504f4d49e3ff1c205c31426452d0fcf32de2af6 Mon Sep 17 00:00:00 2001 From: sray014 Date: Thu, 6 Aug 2026 11:42:33 -0400 Subject: [PATCH 3/3] chore: lint --- src/cosecha/reaping/lsr.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/cosecha/reaping/lsr.py b/src/cosecha/reaping/lsr.py index 9be4755..2cc3842 100644 --- a/src/cosecha/reaping/lsr.py +++ b/src/cosecha/reaping/lsr.py @@ -167,20 +167,22 @@ def _parse_features(self, geojson: dict) -> pd.DataFrame: # Extract coordinates from geometry coords = geom.get("coordinates", [None, None]) - records.append({ - "valid": props.get("valid"), - "event_type": event_type, - "magnitude": props.get("magnitude"), - "unit": props.get("unit"), - "wfo": props.get("wfo"), - "county": props.get("county"), - "state": props.get("st"), - "city": props.get("city"), - "source": props.get("source"), - "remark": props.get("remark"), - "longitude": coords[0] if coords else None, - "latitude": coords[1] if len(coords) > 1 else None, - }) + records.append( + { + "valid": props.get("valid"), + "event_type": event_type, + "magnitude": props.get("magnitude"), + "unit": props.get("unit"), + "wfo": props.get("wfo"), + "county": props.get("county"), + "state": props.get("st"), + "city": props.get("city"), + "source": props.get("source"), + "remark": props.get("remark"), + "longitude": coords[0] if coords else None, + "latitude": coords[1] if len(coords) > 1 else None, + } + ) if not records: raise DataNotFoundError(