From 72c4195deee1d7410fb07564e00b272029512bf7 Mon Sep 17 00:00:00 2001 From: sray014 Date: Tue, 14 Apr 2026 16:08:01 -0400 Subject: [PATCH 01/11] feat: Add ASOS reaper --- src/cosecha/__init__.py | 2 + src/cosecha/reaping/__init__.py | 2 + src/cosecha/reaping/asos.py | 177 ++++++++++++++++++++++++++++++++ 3 files changed, 181 insertions(+) create mode 100644 src/cosecha/reaping/asos.py diff --git a/src/cosecha/__init__.py b/src/cosecha/__init__.py index a0ca0a7..d985f58 100644 --- a/src/cosecha/__init__.py +++ b/src/cosecha/__init__.py @@ -7,6 +7,7 @@ from cosecha import exceptions from cosecha._logging import configure_logger from cosecha.reaping import ( + ASOSReaper, GriddedReaper, MRMSReaper, NWPReaper, @@ -20,6 +21,7 @@ __version__ = "999" __all__ = [ + "ASOSReaper", "GriddedReaper", "MRMSReaper", "NWPReaper", diff --git a/src/cosecha/reaping/__init__.py b/src/cosecha/reaping/__init__.py index 939658d..12d2f74 100644 --- a/src/cosecha/reaping/__init__.py +++ b/src/cosecha/reaping/__init__.py @@ -6,12 +6,14 @@ from __future__ import annotations +from cosecha.reaping.asos import ASOSReaper from cosecha.reaping.base import GriddedReaper, TimeSeriesReaper from cosecha.reaping.mrms import MRMSReaper from cosecha.reaping.nwis import USGSNWISReaper from cosecha.reaping.nwp import NWPReaper __all__ = [ + "ASOSReaper", "GriddedReaper", "MRMSReaper", "NWPReaper", diff --git a/src/cosecha/reaping/asos.py b/src/cosecha/reaping/asos.py new file mode 100644 index 0000000..9ea6bc8 --- /dev/null +++ b/src/cosecha/reaping/asos.py @@ -0,0 +1,177 @@ +"""IEM ASOS data reapers. + +This module provides implementations for harvesting ASOS observations +from the Iowa Environmental Mesonet (IEM) API. +""" + +from __future__ import annotations + +from io import StringIO +from typing import Any + +import pandas as pd +import requests +import time + +from cosecha._logging import logger +from cosecha._utils import apply_ts_transformations, wrap_errors +from cosecha.exceptions import APIError, DateRangeError, InvalidSiteError +from cosecha.reaping.base import TimeSeriesReaper + +__all__ = [ + "ASOSReaper", +] + + +class ASOSReaper(TimeSeriesReaper): + """Reaper for IEM ASOS data.""" + + def _validate_params(self) -> None: + """Validate initialization parameters. + + Raises + ------ + InvalidSiteError + If no state is provided. + DateRangeError + If dates are invalid. + """ + if not self.state: + raise InvalidSiteError("state cannot be empty") + + if self.start_date > self.end_date: + raise DateRangeError( + f"start_date ({self.start_date}) must be <= end_date ({self.end_date})" + ) + + def __init__( + self, + start_date: str, + end_date: str, + state: str = "ASOS", + data: str | list[str] | None = None, + transformations: dict[str, Any] | None = None, + ) -> None: + """Fetch data from IEM ASOS API. + + Parameters + ---------- + start_date : str + Start date in ISO 8601 format (YYYY-MM-DD). + end_date : str + End date in ISO 8601 format (YYYY-MM-DD). + state : str, optional + State abbreviation (e.g., 'TX' or 'ASOS' for all), by default "ASOS". + data : str | list[str] | None, optional + Variable to fetch (e.g., 'p01i', 'tmpf', or ['p01i', 'tmpf']). + If None, fetches 'all'. + transformations : dict[str, Any], optional + Optional transformations to apply to the data. + """ + super().__init__() + self.state = state + self.network = "ASOS" if state.upper() == "ASOS" else f"{state.upper()}_ASOS" + + try: + self.start_date = pd.to_datetime(start_date) + self.end_date = pd.to_datetime(end_date) + except Exception as e: + raise DateRangeError(f"Could not parse date: {e}") from e + + if data is None: + self.data_vars = ["all"] + elif isinstance(data, str): + self.data_vars = [data] + else: + self.data_vars = data + + self.transformations = transformations + + self._validate_params() + logger.debug( + f"Initialized {self.__class__.__name__}: " + f"network={self.network}, dates={self.start_date} to {self.end_date}, " + f"data={self.data_vars}" + ) + + def _fetch_and_parse(self) -> pd.DataFrame: + """Fetch data from IEM and parse into DataFrame. + + Returns + ------- + pd.DataFrame + Parsed data from ASOS CSV output. + + Raises + ------ + APIError + If fetching fails. + """ + url = "https://mesonet.agron.iastate.edu/cgi-bin/request/asos.py" + + params: dict[str, Any] = { + "network": self.network, + "data": self.data_vars, + "year1": self.start_date.year, + "month1": self.start_date.month, + "day1": self.start_date.day, + "year2": self.end_date.year, + "month2": self.end_date.month, + "day2": self.end_date.day, + "format": "comma", + "latlon": "yes", + } + + logger.debug( + f"Fetching ASOS data for network {self.network} " + f"from {self.start_date} to {self.end_date}" + ) + + with wrap_errors(APIError, f"Failed to fetch ASOS data for {self.network}"): + response: requests.Response | None = None + for attempt in range(3): + try: + response = requests.get(url, params=params) + response.raise_for_status() + break + except requests.RequestException as e: + if attempt == 2: + raise e + logger.warning(f"Fetch failed (attempt {attempt + 1}/3): {e}. Retrying in 5 seconds...") + time.sleep(5) + + if response is None: + raise APIError(f"Failed to fetch ASOS data for {self.network}: No response") + + # The first 5 rows of IEM ASOS output are comments (skiprows=5) + df = pd.read_csv(StringIO(response.text), skiprows=5) + + if df.empty: + logger.warning(f"ASOS returned no data for network {self.network}") + return pd.DataFrame() + + logger.debug(f"Fetched {len(df)} records from ASOS for {self.network}") + return df + + def _reap(self) -> pd.DataFrame: + """Fetch data from ASOS and return as a pandas DataFrame. + + Returns + ------- + pd.DataFrame + DataFrame with ASOS data. + + Raises + ------ + APIError + If fetching fails. + """ + logger.info(f"Reaping ASOS data from network {self.network}") + + df = self._fetch_and_parse() + + if self.transformations and not df.empty: + df = apply_ts_transformations(df, self.transformations) + + logger.info(f"Successfully reaped {len(df)} records from {self.network}") + return df From c9b73c50145186f9f6e0f0a8345d5f26f70e6e3f Mon Sep 17 00:00:00 2001 From: sray014 Date: Tue, 14 Apr 2026 16:08:45 -0400 Subject: [PATCH 02/11] test: Add ASOS reaper tests --- tests/test_reaping/test_asos_reapers.py | 223 ++++++++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 tests/test_reaping/test_asos_reapers.py diff --git a/tests/test_reaping/test_asos_reapers.py b/tests/test_reaping/test_asos_reapers.py new file mode 100644 index 0000000..f116a8f --- /dev/null +++ b/tests/test_reaping/test_asos_reapers.py @@ -0,0 +1,223 @@ +"""Tests for ASOS reapers.""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pandas as pd +import pytest +import requests + +from cosecha.exceptions import APIError, DateRangeError, InvalidSiteError +from cosecha.reaping.asos import ASOSReaper + + +class TestASOSReaper: + """Tests for ASOSReaper.""" + + def test_initialization_valid(self): + """Test valid initialization.""" + reaper = ASOSReaper( + state="TX", + data="p01i", + start_date="2026-04-12", + end_date="2026-04-13", + ) + assert reaper.state == "TX" + assert reaper.network == "TX_ASOS" + assert reaper.start_date == pd.Timestamp("2026-04-12") + assert reaper.end_date == pd.Timestamp("2026-04-13") + assert reaper.data_vars == ["p01i"] + + def test_initialization_multiple_data_vars(self): + """Test initialization with multiple data variables.""" + reaper = ASOSReaper( + state="IA", + data=["p01i", "tmpf"], + start_date="2026-04-12", + end_date="2026-04-13", + ) + assert reaper.data_vars == ["p01i", "tmpf"] + assert reaper.network == "IA_ASOS" + + def test_initialization_all_network(self): + """Test initialization with ASOS state.""" + reaper = ASOSReaper( + state="ASOS", + start_date="2026-04-12", + end_date="2026-04-13", + ) + assert reaper.network == "ASOS" + assert reaper.data_vars == ["all"] + + def test_empty_state(self): + """Test initialization fails with empty state.""" + with pytest.raises(InvalidSiteError, match="state cannot be empty"): + ASOSReaper( + state="", + start_date="2026-04-12", + end_date="2026-04-13", + ) + + def test_invalid_date_range(self): + """Test initialization fails with invalid date range.""" + with pytest.raises(DateRangeError): + ASOSReaper( + state="TX", + data="p01i", + start_date="2026-04-13", + end_date="2026-04-12", + ) + + def test_invalid_start_date(self): + """Test initialization fails with unparsable start date.""" + with pytest.raises(DateRangeError, match="Could not parse date"): + ASOSReaper( + state="TX", + data="p01i", + start_date="not-a-date", + end_date="2026-04-13", + ) + + @pytest.mark.network + def test_reap_network(self): + """Test live network retrieval from ASOS.""" + reaper = ASOSReaper( + state="TX", + data="p01i", + start_date="2022-01-01", + end_date="2022-01-02", + ) + harvested = reaper.reap() + assert len(harvested) > 0 + assert "p01i" in harvested.columns + assert isinstance(harvested, pd.DataFrame) + + @patch("cosecha.reaping.asos.requests.get") + def test_reap_success(self, mock_get): + """Test successful data retrieval.""" + mock_response = MagicMock() + mock_response.text = ( + "# comment 1\n" + "# comment 2\n" + "# comment 3\n" + "# comment 4\n" + "# comment 5\n" + "station,valid,lon,lat,p01i\n" + "AUS,2026-04-12 00:00,-97.6698,30.1945,0.01\n" + "AUS,2026-04-12 01:00,-97.6698,30.1945,0.02\n" + ) + mock_response.raise_for_status.return_value = None + mock_get.return_value = mock_response + + reaper = ASOSReaper( + state="TX", + data="p01i", + start_date="2026-04-12", + end_date="2026-04-12", + ) + + harvested = reaper.reap() + + assert len(harvested) == 2 + assert isinstance(harvested, pd.DataFrame) + assert "p01i" in harvested.columns + assert "station" in harvested.columns + mock_get.assert_called_once() + + args, kwargs = mock_get.call_args + assert kwargs["params"]["network"] == "TX_ASOS" + assert kwargs["params"]["data"] == ["p01i"] + + @patch("cosecha.reaping.asos.time.sleep") + @patch("cosecha.reaping.asos.requests.get") + def test_reap_retry_logic(self, mock_get, mock_sleep): + """Test retry logic when API fails temporarily.""" + mock_response = MagicMock() + mock_response.text = ( + "# 1\n# 2\n# 3\n# 4\n# 5\n" + "station,p01i\n" + "AUS,0.01\n" + ) + + # Fail twice, succeed on third + mock_get.side_effect = [ + requests.RequestException("Timeout 1"), + requests.RequestException("Timeout 2"), + mock_response + ] + + reaper = ASOSReaper( + state="TX", + start_date="2026-04-12", + end_date="2026-04-12", + ) + + harvested = reaper.reap() + + assert len(harvested) == 1 + assert mock_get.call_count == 3 + assert mock_sleep.call_count == 2 + + @patch("cosecha.reaping.asos.time.sleep") + @patch("cosecha.reaping.asos.requests.get") + def test_reap_api_error(self, mock_get, mock_sleep): + """Test error handling when API fails permanently.""" + mock_get.side_effect = requests.RequestException("Connection failed") + + reaper = ASOSReaper( + state="TX", + start_date="2026-04-12", + end_date="2026-04-12", + ) + + with pytest.raises(APIError, match="Failed to fetch ASOS data for TX_ASOS"): + reaper.reap() + + assert mock_get.call_count == 3 + assert mock_sleep.call_count == 2 + + @patch("cosecha.reaping.asos.requests.get") + def test_reap_empty_response(self, mock_get): + """Test reap handles empty output after skipping comments.""" + mock_response = MagicMock() + # Ensure that parsing returns an empty dataframe (e.g., just comments or header only) + mock_response.text = ( + "# 1\n# 2\n# 3\n# 4\n# 5\n" + "station,valid,lon,lat,p01i\n" + ) + mock_response.raise_for_status.return_value = None + mock_get.return_value = mock_response + + reaper = ASOSReaper( + state="TX", + start_date="2026-04-12", + end_date="2026-04-12", + ) + + harvested = reaper.reap() + assert isinstance(harvested, pd.DataFrame) + assert harvested.empty + + @patch("cosecha.reaping.asos.requests.get") + def test_reap_with_transformations(self, mock_get): + """Test reap applies format transformations when not empty.""" + mock_response = MagicMock() + mock_response.text = ( + "# 1\n# 2\n# 3\n# 4\n# 5\n" + "station,p01i\n" + "AUS,0.01\n" + ) + mock_response.raise_for_status.return_value = None + mock_get.return_value = mock_response + + reaper = ASOSReaper( + state="TX", + start_date="2026-04-12", + end_date="2026-04-12", + transformations={"rename_columns": {"p01i": "precip"}}, + ) + + harvested = reaper.reap() + assert "precip" in harvested.columns + assert "p01i" not in harvested.columns From 73a12c98d7fe2bde1a63c01eb53d1c1084944cc6 Mon Sep 17 00:00:00 2001 From: sray014 Date: Mon, 22 Jun 2026 13:26:40 -0400 Subject: [PATCH 03/11] refactor: Use tiny_retriever instead of requests Co-authored-by: Copilot --- src/cosecha/reaping/asos.py | 129 ++++++++++-------------- tests/test_reaping/test_asos_reapers.py | 95 +++++------------ 2 files changed, 77 insertions(+), 147 deletions(-) diff --git a/src/cosecha/reaping/asos.py b/src/cosecha/reaping/asos.py index 9ea6bc8..4aee194 100644 --- a/src/cosecha/reaping/asos.py +++ b/src/cosecha/reaping/asos.py @@ -8,10 +8,10 @@ from io import StringIO from typing import Any +from urllib.parse import urlencode import pandas as pd -import requests -import time +import tiny_retriever from cosecha._logging import logger from cosecha._utils import apply_ts_transformations, wrap_errors @@ -22,10 +22,14 @@ "ASOSReaper", ] +BASE_URL = "https://mesonet.agron.iastate.edu/cgi-bin/request/asos.py" + class ASOSReaper(TimeSeriesReaper): """Reaper for IEM ASOS data.""" + timeout: int = 120 + def _validate_params(self) -> None: """Validate initialization parameters. @@ -49,7 +53,7 @@ def __init__( start_date: str, end_date: str, state: str = "ASOS", - data: str | list[str] | None = None, + variable: str | list[str] | None = None, transformations: dict[str, Any] | None = None, ) -> None: """Fetch data from IEM ASOS API. @@ -62,7 +66,7 @@ def __init__( End date in ISO 8601 format (YYYY-MM-DD). state : str, optional State abbreviation (e.g., 'TX' or 'ASOS' for all), by default "ASOS". - data : str | list[str] | None, optional + variable : str | list[str] | None, optional Variable to fetch (e.g., 'p01i', 'tmpf', or ['p01i', 'tmpf']). If None, fetches 'all'. transformations : dict[str, Any], optional @@ -78,12 +82,12 @@ def __init__( except Exception as e: raise DateRangeError(f"Could not parse date: {e}") from e - if data is None: + if variable is None: self.data_vars = ["all"] - elif isinstance(data, str): - self.data_vars = [data] + elif isinstance(variable, str): + self.data_vars = [variable] else: - self.data_vars = data + self.data_vars = variable self.transformations = transformations @@ -94,84 +98,53 @@ def __init__( f"data={self.data_vars}" ) - def _fetch_and_parse(self) -> pd.DataFrame: - """Fetch data from IEM and parse into DataFrame. + def _build_url(self) -> str: + """Build the IEM ASOS request URL with query parameters.""" + params = [ + ("network", self.network), + ("year1", self.start_date.year), + ("month1", self.start_date.month), + ("day1", self.start_date.day), + ("year2", self.end_date.year), + ("month2", self.end_date.month), + ("day2", self.end_date.day), + ("format", "comma"), + ("latlon", "yes"), + ] + for var in self.data_vars: + params.append(("data", var)) + return f"{BASE_URL}?{urlencode(params)}" + + def _fetch(self, url: str) -> str: + """Fetch CSV text from IEM via tiny_retriever.""" + with wrap_errors(APIError, f"Failed to fetch ASOS data for {self.network}"): + return tiny_retriever.fetch(url, "text", timeout=self.timeout) - Returns - ------- - pd.DataFrame - Parsed data from ASOS CSV output. + def _parse_response(self, text: str) -> pd.DataFrame: + """Parse IEM CSV text into a DataFrame. - Raises - ------ - APIError - If fetching fails. + The first 5 rows of IEM ASOS output are comments (skiprows=5). """ - url = "https://mesonet.agron.iastate.edu/cgi-bin/request/asos.py" - - params: dict[str, Any] = { - "network": self.network, - "data": self.data_vars, - "year1": self.start_date.year, - "month1": self.start_date.month, - "day1": self.start_date.day, - "year2": self.end_date.year, - "month2": self.end_date.month, - "day2": self.end_date.day, - "format": "comma", - "latlon": "yes", - } - - logger.debug( - f"Fetching ASOS data for network {self.network} " - f"from {self.start_date} to {self.end_date}" - ) - - with wrap_errors(APIError, f"Failed to fetch ASOS data for {self.network}"): - response: requests.Response | None = None - for attempt in range(3): - try: - response = requests.get(url, params=params) - response.raise_for_status() - break - except requests.RequestException as e: - if attempt == 2: - raise e - logger.warning(f"Fetch failed (attempt {attempt + 1}/3): {e}. Retrying in 5 seconds...") - time.sleep(5) - - if response is None: - raise APIError(f"Failed to fetch ASOS data for {self.network}: No response") - - # The first 5 rows of IEM ASOS output are comments (skiprows=5) - df = pd.read_csv(StringIO(response.text), skiprows=5) - - if df.empty: - logger.warning(f"ASOS returned no data for network {self.network}") - return pd.DataFrame() - - logger.debug(f"Fetched {len(df)} records from ASOS for {self.network}") - return df + df = pd.read_csv(StringIO(text), skiprows=5) + if df.empty: + logger.warning(f"ASOS returned no data for network {self.network}") + return pd.DataFrame() + logger.debug(f"Fetched {len(df)} records from ASOS for {self.network}") + return df def _reap(self) -> pd.DataFrame: - """Fetch data from ASOS and return as a pandas DataFrame. - - Returns - ------- - pd.DataFrame - DataFrame with ASOS data. - - Raises - ------ - APIError - If fetching fails. - """ - logger.info(f"Reaping ASOS data from network {self.network}") + """Fetch data from ASOS and return as a pandas DataFrame.""" + logger.info( + f"Reaping ASOS data: network={self.network}, " + f"data={self.data_vars}" + ) - df = self._fetch_and_parse() + url = self._build_url() + text = self._fetch(url) + df = self._parse_response(text) if self.transformations and not df.empty: df = apply_ts_transformations(df, self.transformations) - logger.info(f"Successfully reaped {len(df)} records from {self.network}") + logger.info(f"Reaped {len(df)} records from {self.network}") return df diff --git a/tests/test_reaping/test_asos_reapers.py b/tests/test_reaping/test_asos_reapers.py index f116a8f..73768fb 100644 --- a/tests/test_reaping/test_asos_reapers.py +++ b/tests/test_reaping/test_asos_reapers.py @@ -2,11 +2,10 @@ from __future__ import annotations -from unittest.mock import MagicMock, patch +from unittest.mock import patch import pandas as pd import pytest -import requests from cosecha.exceptions import APIError, DateRangeError, InvalidSiteError from cosecha.reaping.asos import ASOSReaper @@ -19,7 +18,7 @@ def test_initialization_valid(self): """Test valid initialization.""" reaper = ASOSReaper( state="TX", - data="p01i", + variable="p01i", start_date="2026-04-12", end_date="2026-04-13", ) @@ -33,7 +32,7 @@ def test_initialization_multiple_data_vars(self): """Test initialization with multiple data variables.""" reaper = ASOSReaper( state="IA", - data=["p01i", "tmpf"], + variable=["p01i", "tmpf"], start_date="2026-04-12", end_date="2026-04-13", ) @@ -64,7 +63,7 @@ def test_invalid_date_range(self): with pytest.raises(DateRangeError): ASOSReaper( state="TX", - data="p01i", + variable="p01i", start_date="2026-04-13", end_date="2026-04-12", ) @@ -74,7 +73,7 @@ def test_invalid_start_date(self): with pytest.raises(DateRangeError, match="Could not parse date"): ASOSReaper( state="TX", - data="p01i", + variable="p01i", start_date="not-a-date", end_date="2026-04-13", ) @@ -84,7 +83,7 @@ def test_reap_network(self): """Test live network retrieval from ASOS.""" reaper = ASOSReaper( state="TX", - data="p01i", + variable="p01i", start_date="2022-01-01", end_date="2022-01-02", ) @@ -93,11 +92,10 @@ def test_reap_network(self): assert "p01i" in harvested.columns assert isinstance(harvested, pd.DataFrame) - @patch("cosecha.reaping.asos.requests.get") - def test_reap_success(self, mock_get): + @patch("cosecha.reaping.asos.tiny_retriever.fetch") + def test_reap_success(self, mock_fetch): """Test successful data retrieval.""" - mock_response = MagicMock() - mock_response.text = ( + mock_fetch.return_value = ( "# comment 1\n" "# comment 2\n" "# comment 3\n" @@ -107,12 +105,10 @@ def test_reap_success(self, mock_get): "AUS,2026-04-12 00:00,-97.6698,30.1945,0.01\n" "AUS,2026-04-12 01:00,-97.6698,30.1945,0.02\n" ) - mock_response.raise_for_status.return_value = None - mock_get.return_value = mock_response reaper = ASOSReaper( state="TX", - data="p01i", + variable="p01i", start_date="2026-04-12", end_date="2026-04-12", ) @@ -123,47 +119,16 @@ def test_reap_success(self, mock_get): assert isinstance(harvested, pd.DataFrame) assert "p01i" in harvested.columns assert "station" in harvested.columns - mock_get.assert_called_once() - - args, kwargs = mock_get.call_args - assert kwargs["params"]["network"] == "TX_ASOS" - assert kwargs["params"]["data"] == ["p01i"] - - @patch("cosecha.reaping.asos.time.sleep") - @patch("cosecha.reaping.asos.requests.get") - def test_reap_retry_logic(self, mock_get, mock_sleep): - """Test retry logic when API fails temporarily.""" - mock_response = MagicMock() - mock_response.text = ( - "# 1\n# 2\n# 3\n# 4\n# 5\n" - "station,p01i\n" - "AUS,0.01\n" - ) - - # Fail twice, succeed on third - mock_get.side_effect = [ - requests.RequestException("Timeout 1"), - requests.RequestException("Timeout 2"), - mock_response - ] - - reaper = ASOSReaper( - state="TX", - start_date="2026-04-12", - end_date="2026-04-12", - ) + mock_fetch.assert_called_once() - harvested = reaper.reap() - - assert len(harvested) == 1 - assert mock_get.call_count == 3 - assert mock_sleep.call_count == 2 + url = mock_fetch.call_args[0][0] + assert "network=TX_ASOS" in url + assert "data=p01i" in url - @patch("cosecha.reaping.asos.time.sleep") - @patch("cosecha.reaping.asos.requests.get") - def test_reap_api_error(self, mock_get, mock_sleep): - """Test error handling when API fails permanently.""" - mock_get.side_effect = requests.RequestException("Connection failed") + @patch("cosecha.reaping.asos.tiny_retriever.fetch") + def test_reap_api_error(self, mock_fetch): + """Test error handling when API fails.""" + mock_fetch.side_effect = Exception("Connection failed") reaper = ASOSReaper( state="TX", @@ -173,21 +138,16 @@ def test_reap_api_error(self, mock_get, mock_sleep): with pytest.raises(APIError, match="Failed to fetch ASOS data for TX_ASOS"): reaper.reap() - - assert mock_get.call_count == 3 - assert mock_sleep.call_count == 2 - @patch("cosecha.reaping.asos.requests.get") - def test_reap_empty_response(self, mock_get): + mock_fetch.assert_called_once() + + @patch("cosecha.reaping.asos.tiny_retriever.fetch") + def test_reap_empty_response(self, mock_fetch): """Test reap handles empty output after skipping comments.""" - mock_response = MagicMock() - # Ensure that parsing returns an empty dataframe (e.g., just comments or header only) - mock_response.text = ( + mock_fetch.return_value = ( "# 1\n# 2\n# 3\n# 4\n# 5\n" "station,valid,lon,lat,p01i\n" ) - mock_response.raise_for_status.return_value = None - mock_get.return_value = mock_response reaper = ASOSReaper( state="TX", @@ -199,17 +159,14 @@ def test_reap_empty_response(self, mock_get): assert isinstance(harvested, pd.DataFrame) assert harvested.empty - @patch("cosecha.reaping.asos.requests.get") - def test_reap_with_transformations(self, mock_get): + @patch("cosecha.reaping.asos.tiny_retriever.fetch") + def test_reap_with_transformations(self, mock_fetch): """Test reap applies format transformations when not empty.""" - mock_response = MagicMock() - mock_response.text = ( + mock_fetch.return_value = ( "# 1\n# 2\n# 3\n# 4\n# 5\n" "station,p01i\n" "AUS,0.01\n" ) - mock_response.raise_for_status.return_value = None - mock_get.return_value = mock_response reaper = ASOSReaper( state="TX", From 441de81e5bced763749054360816fb8f5eade300 Mon Sep 17 00:00:00 2001 From: sray014 Date: Mon, 6 Jul 2026 14:18:50 -0400 Subject: [PATCH 04/11] chore: Add DataNotFoundError exception --- src/cosecha/exceptions.py | 5 +++++ src/cosecha/reaping/asos.py | 7 ++++--- tests/test_reaping/test_asos_reapers.py | 9 ++++----- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/cosecha/exceptions.py b/src/cosecha/exceptions.py index 5617239..a35c1c9 100644 --- a/src/cosecha/exceptions.py +++ b/src/cosecha/exceptions.py @@ -4,6 +4,7 @@ __all__ = [ "APIError", + "DataNotFoundError", "DateRangeError", "InvalidSiteError", "ReaperError", @@ -27,5 +28,9 @@ class APIError(ReaperError): """Raised when an API call fails.""" +class DataNotFoundError(ReaperError): + """Raised when a query returns no data.""" + + class TransformationError(ReaperError): """Raised when data transformation fails.""" diff --git a/src/cosecha/reaping/asos.py b/src/cosecha/reaping/asos.py index 4aee194..04c85b6 100644 --- a/src/cosecha/reaping/asos.py +++ b/src/cosecha/reaping/asos.py @@ -15,7 +15,7 @@ from cosecha._logging import logger from cosecha._utils import apply_ts_transformations, wrap_errors -from cosecha.exceptions import APIError, DateRangeError, InvalidSiteError +from cosecha.exceptions import APIError, DataNotFoundError, DateRangeError, InvalidSiteError from cosecha.reaping.base import TimeSeriesReaper __all__ = [ @@ -127,8 +127,9 @@ def _parse_response(self, text: str) -> pd.DataFrame: """ df = pd.read_csv(StringIO(text), skiprows=5) if df.empty: - logger.warning(f"ASOS returned no data for network {self.network}") - return pd.DataFrame() + raise DataNotFoundError( + f"ASOS returned no data for network {self.network} and time range {self.start_date} to {self.end_date}" + ) logger.debug(f"Fetched {len(df)} records from ASOS for {self.network}") return df diff --git a/tests/test_reaping/test_asos_reapers.py b/tests/test_reaping/test_asos_reapers.py index 73768fb..2d9c037 100644 --- a/tests/test_reaping/test_asos_reapers.py +++ b/tests/test_reaping/test_asos_reapers.py @@ -7,7 +7,7 @@ import pandas as pd import pytest -from cosecha.exceptions import APIError, DateRangeError, InvalidSiteError +from cosecha.exceptions import APIError, DataNotFoundError, DateRangeError, InvalidSiteError from cosecha.reaping.asos import ASOSReaper @@ -143,7 +143,7 @@ def test_reap_api_error(self, mock_fetch): @patch("cosecha.reaping.asos.tiny_retriever.fetch") def test_reap_empty_response(self, mock_fetch): - """Test reap handles empty output after skipping comments.""" + """Test reap raises DataNotFoundError on empty output.""" mock_fetch.return_value = ( "# 1\n# 2\n# 3\n# 4\n# 5\n" "station,valid,lon,lat,p01i\n" @@ -155,9 +155,8 @@ def test_reap_empty_response(self, mock_fetch): end_date="2026-04-12", ) - harvested = reaper.reap() - assert isinstance(harvested, pd.DataFrame) - assert harvested.empty + with pytest.raises(DataNotFoundError, match="no data"): + reaper.reap() @patch("cosecha.reaping.asos.tiny_retriever.fetch") def test_reap_with_transformations(self, mock_fetch): From bf489e3f25489027eee461f26c2ca4100f35b3fd Mon Sep 17 00:00:00 2001 From: sray014 Date: Mon, 6 Jul 2026 14:21:02 -0400 Subject: [PATCH 05/11] chore: Move constant to module level --- src/cosecha/reaping/asos.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cosecha/reaping/asos.py b/src/cosecha/reaping/asos.py index 04c85b6..efee2a9 100644 --- a/src/cosecha/reaping/asos.py +++ b/src/cosecha/reaping/asos.py @@ -23,6 +23,7 @@ ] BASE_URL = "https://mesonet.agron.iastate.edu/cgi-bin/request/asos.py" +_IEM_ALL_VARS = "all" class ASOSReaper(TimeSeriesReaper): @@ -83,7 +84,7 @@ def __init__( raise DateRangeError(f"Could not parse date: {e}") from e if variable is None: - self.data_vars = ["all"] + self.data_vars = [_IEM_ALL_VARS] elif isinstance(variable, str): self.data_vars = [variable] else: From a94c3e3a2339f72e10fd537eceeca6f143fb4b71 Mon Sep 17 00:00:00 2001 From: sray014 Date: Mon, 6 Jul 2026 14:23:38 -0400 Subject: [PATCH 06/11] chore: Move timeout to init param --- src/cosecha/reaping/asos.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/cosecha/reaping/asos.py b/src/cosecha/reaping/asos.py index efee2a9..e4e0cf1 100644 --- a/src/cosecha/reaping/asos.py +++ b/src/cosecha/reaping/asos.py @@ -29,8 +29,6 @@ class ASOSReaper(TimeSeriesReaper): """Reaper for IEM ASOS data.""" - timeout: int = 120 - def _validate_params(self) -> None: """Validate initialization parameters. @@ -56,6 +54,7 @@ def __init__( state: str = "ASOS", variable: str | list[str] | None = None, transformations: dict[str, Any] | None = None, + timeout: int = 120, ) -> None: """Fetch data from IEM ASOS API. @@ -72,6 +71,8 @@ def __init__( If None, fetches 'all'. transformations : dict[str, Any], optional Optional transformations to apply to the data. + timeout : int, optional + Request timeout in seconds, by default 120. """ super().__init__() self.state = state @@ -91,6 +92,7 @@ def __init__( self.data_vars = variable self.transformations = transformations + self.timeout = timeout self._validate_params() logger.debug( From 0a8910aeaeef8df7f4f24c6853e97b69bbd242de Mon Sep 17 00:00:00 2001 From: sray014 Date: Mon, 6 Jul 2026 14:44:49 -0400 Subject: [PATCH 07/11] chore: Remove 'ASOS' identifier, allow 'None' for all networks --- src/cosecha/reaping/asos.py | 29 +++++++++++-------------- tests/test_reaping/test_asos_reapers.py | 16 +++----------- 2 files changed, 16 insertions(+), 29 deletions(-) diff --git a/src/cosecha/reaping/asos.py b/src/cosecha/reaping/asos.py index e4e0cf1..5e8d4bb 100644 --- a/src/cosecha/reaping/asos.py +++ b/src/cosecha/reaping/asos.py @@ -15,7 +15,7 @@ from cosecha._logging import logger from cosecha._utils import apply_ts_transformations, wrap_errors -from cosecha.exceptions import APIError, DataNotFoundError, DateRangeError, InvalidSiteError +from cosecha.exceptions import APIError, DataNotFoundError, DateRangeError from cosecha.reaping.base import TimeSeriesReaper __all__ = [ @@ -34,14 +34,9 @@ def _validate_params(self) -> None: Raises ------ - InvalidSiteError - If no state is provided. DateRangeError If dates are invalid. """ - if not self.state: - raise InvalidSiteError("state cannot be empty") - if self.start_date > self.end_date: raise DateRangeError( f"start_date ({self.start_date}) must be <= end_date ({self.end_date})" @@ -51,7 +46,7 @@ def __init__( self, start_date: str, end_date: str, - state: str = "ASOS", + state: str | None = None, variable: str | list[str] | None = None, transformations: dict[str, Any] | None = None, timeout: int = 120, @@ -64,8 +59,9 @@ def __init__( Start date in ISO 8601 format (YYYY-MM-DD). end_date : str End date in ISO 8601 format (YYYY-MM-DD). - state : str, optional - State abbreviation (e.g., 'TX' or 'ASOS' for all), by default "ASOS". + state : str | None, optional + State abbreviation (e.g., 'TX'). If None (default), fetches from + all networks (IEM limits this to a 24-hour window). variable : str | list[str] | None, optional Variable to fetch (e.g., 'p01i', 'tmpf', or ['p01i', 'tmpf']). If None, fetches 'all'. @@ -76,7 +72,7 @@ def __init__( """ super().__init__() self.state = state - self.network = "ASOS" if state.upper() == "ASOS" else f"{state.upper()}_ASOS" + self.network = f"{state.upper()}_ASOS" if state else None try: self.start_date = pd.to_datetime(start_date) @@ -104,7 +100,6 @@ def __init__( def _build_url(self) -> str: """Build the IEM ASOS request URL with query parameters.""" params = [ - ("network", self.network), ("year1", self.start_date.year), ("month1", self.start_date.month), ("day1", self.start_date.day), @@ -114,13 +109,15 @@ def _build_url(self) -> str: ("format", "comma"), ("latlon", "yes"), ] + if self.network is not None: + params.insert(0, ("network", self.network)) for var in self.data_vars: params.append(("data", var)) return f"{BASE_URL}?{urlencode(params)}" def _fetch(self, url: str) -> str: """Fetch CSV text from IEM via tiny_retriever.""" - with wrap_errors(APIError, f"Failed to fetch ASOS data for {self.network}"): + with wrap_errors(APIError, f"Failed to fetch ASOS data for {self.network or 'all networks'}"): return tiny_retriever.fetch(url, "text", timeout=self.timeout) def _parse_response(self, text: str) -> pd.DataFrame: @@ -131,15 +128,15 @@ def _parse_response(self, text: str) -> pd.DataFrame: df = pd.read_csv(StringIO(text), skiprows=5) if df.empty: raise DataNotFoundError( - f"ASOS returned no data for network {self.network} and time range {self.start_date} to {self.end_date}" + f"ASOS returned no data for network {self.network or 'all networks'} and time range {self.start_date} to {self.end_date}" ) - logger.debug(f"Fetched {len(df)} records from ASOS for {self.network}") + logger.debug(f"Fetched {len(df)} records from ASOS for {self.network or 'all networks'}") return df def _reap(self) -> pd.DataFrame: """Fetch data from ASOS and return as a pandas DataFrame.""" logger.info( - f"Reaping ASOS data: network={self.network}, " + f"Reaping ASOS data: network={self.network or 'all networks'}, " f"data={self.data_vars}" ) @@ -150,5 +147,5 @@ def _reap(self) -> pd.DataFrame: if self.transformations and not df.empty: df = apply_ts_transformations(df, self.transformations) - logger.info(f"Reaped {len(df)} records from {self.network}") + logger.info(f"Reaped {len(df)} records from {self.network or 'all networks'}") return df diff --git a/tests/test_reaping/test_asos_reapers.py b/tests/test_reaping/test_asos_reapers.py index 2d9c037..d194416 100644 --- a/tests/test_reaping/test_asos_reapers.py +++ b/tests/test_reaping/test_asos_reapers.py @@ -7,7 +7,7 @@ import pandas as pd import pytest -from cosecha.exceptions import APIError, DataNotFoundError, DateRangeError, InvalidSiteError +from cosecha.exceptions import APIError, DataNotFoundError, DateRangeError from cosecha.reaping.asos import ASOSReaper @@ -40,24 +40,14 @@ def test_initialization_multiple_data_vars(self): assert reaper.network == "IA_ASOS" def test_initialization_all_network(self): - """Test initialization with ASOS state.""" + """Test initialization with no state fetches all networks.""" reaper = ASOSReaper( - state="ASOS", start_date="2026-04-12", end_date="2026-04-13", ) - assert reaper.network == "ASOS" + assert reaper.network is None assert reaper.data_vars == ["all"] - def test_empty_state(self): - """Test initialization fails with empty state.""" - with pytest.raises(InvalidSiteError, match="state cannot be empty"): - ASOSReaper( - state="", - start_date="2026-04-12", - end_date="2026-04-13", - ) - def test_invalid_date_range(self): """Test initialization fails with invalid date range.""" with pytest.raises(DateRangeError): From 749bb8d9bacb27ebe0fc4382f61179dc5f6a9524 Mon Sep 17 00:00:00 2001 From: sray014 Date: Mon, 6 Jul 2026 15:02:48 -0400 Subject: [PATCH 08/11] chore: Add report type, improve params handling --- src/cosecha/reaping/asos.py | 41 ++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/src/cosecha/reaping/asos.py b/src/cosecha/reaping/asos.py index 5e8d4bb..cc6cddc 100644 --- a/src/cosecha/reaping/asos.py +++ b/src/cosecha/reaping/asos.py @@ -48,6 +48,7 @@ def __init__( end_date: str, state: str | None = None, variable: str | list[str] | None = None, + report_type: int | list[int] | None = None, transformations: dict[str, Any] | None = None, timeout: int = 120, ) -> None: @@ -65,6 +66,9 @@ def __init__( variable : str | list[str] | None, optional Variable to fetch (e.g., 'p01i', 'tmpf', or ['p01i', 'tmpf']). If None, fetches 'all'. + report_type : int | list[int] | None, optional + IEM report type filter: 1 (HFMETAR/5-min), 3 (routine), 4 (specials). + If None (default), returns all report types. transformations : dict[str, Any], optional Optional transformations to apply to the data. timeout : int, optional @@ -90,6 +94,13 @@ def __init__( self.transformations = transformations self.timeout = timeout + if report_type is None: + self.report_type = None + elif isinstance(report_type, int): + self.report_type = [report_type] + else: + self.report_type = report_type + self._validate_params() logger.debug( f"Initialized {self.__class__.__name__}: " @@ -99,21 +110,23 @@ def __init__( def _build_url(self) -> str: """Build the IEM ASOS request URL with query parameters.""" - params = [ - ("year1", self.start_date.year), - ("month1", self.start_date.month), - ("day1", self.start_date.day), - ("year2", self.end_date.year), - ("month2", self.end_date.month), - ("day2", self.end_date.day), - ("format", "comma"), - ("latlon", "yes"), - ] + params = { + "year1": self.start_date.year, + "month1": self.start_date.month, + "day1": self.start_date.day, + "year2": self.end_date.year, + "month2": self.end_date.month, + "day2": self.end_date.day, + "tz": "Etc/UTC", + "format": "comma", + "latlon": "yes", + "data": self.data_vars, + } if self.network is not None: - params.insert(0, ("network", self.network)) - for var in self.data_vars: - params.append(("data", var)) - return f"{BASE_URL}?{urlencode(params)}" + params["network"] = self.network + if self.report_type is not None: + params["report_type"] = self.report_type + return f"{BASE_URL}?{urlencode(params, doseq=True)}" def _fetch(self, url: str) -> str: """Fetch CSV text from IEM via tiny_retriever.""" From 3184a24f6f3dd04101d55ca65b1f7e34c4030e9e Mon Sep 17 00:00:00 2001 From: sray014 Date: Mon, 6 Jul 2026 15:03:33 -0400 Subject: [PATCH 09/11] chore: Lint format --- src/cosecha/reaping/asos.py | 7 ++++--- tests/test_reaping/test_asos_reapers.py | 11 ++--------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/cosecha/reaping/asos.py b/src/cosecha/reaping/asos.py index cc6cddc..874e11b 100644 --- a/src/cosecha/reaping/asos.py +++ b/src/cosecha/reaping/asos.py @@ -130,7 +130,9 @@ def _build_url(self) -> str: def _fetch(self, url: str) -> str: """Fetch CSV text from IEM via tiny_retriever.""" - with wrap_errors(APIError, f"Failed to fetch ASOS data for {self.network or 'all networks'}"): + with wrap_errors( + APIError, f"Failed to fetch ASOS data for {self.network or 'all networks'}" + ): return tiny_retriever.fetch(url, "text", timeout=self.timeout) def _parse_response(self, text: str) -> pd.DataFrame: @@ -149,8 +151,7 @@ def _parse_response(self, text: str) -> pd.DataFrame: def _reap(self) -> pd.DataFrame: """Fetch data from ASOS and return as a pandas DataFrame.""" logger.info( - f"Reaping ASOS data: network={self.network or 'all networks'}, " - f"data={self.data_vars}" + f"Reaping ASOS data: network={self.network or 'all networks'}, data={self.data_vars}" ) url = self._build_url() diff --git a/tests/test_reaping/test_asos_reapers.py b/tests/test_reaping/test_asos_reapers.py index d194416..956536c 100644 --- a/tests/test_reaping/test_asos_reapers.py +++ b/tests/test_reaping/test_asos_reapers.py @@ -134,10 +134,7 @@ def test_reap_api_error(self, mock_fetch): @patch("cosecha.reaping.asos.tiny_retriever.fetch") def test_reap_empty_response(self, mock_fetch): """Test reap raises DataNotFoundError on empty output.""" - mock_fetch.return_value = ( - "# 1\n# 2\n# 3\n# 4\n# 5\n" - "station,valid,lon,lat,p01i\n" - ) + mock_fetch.return_value = "# 1\n# 2\n# 3\n# 4\n# 5\nstation,valid,lon,lat,p01i\n" reaper = ASOSReaper( state="TX", @@ -151,11 +148,7 @@ def test_reap_empty_response(self, mock_fetch): @patch("cosecha.reaping.asos.tiny_retriever.fetch") def test_reap_with_transformations(self, mock_fetch): """Test reap applies format transformations when not empty.""" - mock_fetch.return_value = ( - "# 1\n# 2\n# 3\n# 4\n# 5\n" - "station,p01i\n" - "AUS,0.01\n" - ) + mock_fetch.return_value = "# 1\n# 2\n# 3\n# 4\n# 5\nstation,p01i\nAUS,0.01\n" reaper = ASOSReaper( state="TX", From d5c9761ec5daf5ddeb828542b451578b80064617 Mon Sep 17 00:00:00 2001 From: sray014 Date: Mon, 6 Jul 2026 15:33:11 -0400 Subject: [PATCH 10/11] chore: Add hour and minute to API params --- src/cosecha/reaping/asos.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/cosecha/reaping/asos.py b/src/cosecha/reaping/asos.py index 874e11b..45aefc4 100644 --- a/src/cosecha/reaping/asos.py +++ b/src/cosecha/reaping/asos.py @@ -57,9 +57,9 @@ def __init__( Parameters ---------- start_date : str - Start date in ISO 8601 format (YYYY-MM-DD). + Start date in ISO 8601 format (YYYY-MM-DD HH:MMZ). end_date : str - End date in ISO 8601 format (YYYY-MM-DD). + End date in ISO 8601 format (YYYY-MM-DD HH:MMZ). state : str | None, optional State abbreviation (e.g., 'TX'). If None (default), fetches from all networks (IEM limits this to a 24-hour window). @@ -114,9 +114,13 @@ def _build_url(self) -> str: "year1": self.start_date.year, "month1": self.start_date.month, "day1": self.start_date.day, + "hour1": self.start_date.hour, + "minute1": self.start_date.minute, "year2": self.end_date.year, "month2": self.end_date.month, "day2": self.end_date.day, + "hour2": self.end_date.hour, + "minute2": self.end_date.minute, "tz": "Etc/UTC", "format": "comma", "latlon": "yes", From 13e9d07f0db8dd0a034d566fb330d010d4da84f8 Mon Sep 17 00:00:00 2001 From: sray014 Date: Mon, 6 Jul 2026 15:35:46 -0400 Subject: [PATCH 11/11] docs: Fix time format in doc string --- src/cosecha/reaping/asos.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cosecha/reaping/asos.py b/src/cosecha/reaping/asos.py index 45aefc4..1804d95 100644 --- a/src/cosecha/reaping/asos.py +++ b/src/cosecha/reaping/asos.py @@ -57,9 +57,9 @@ def __init__( Parameters ---------- start_date : str - Start date in ISO 8601 format (YYYY-MM-DD HH:MMZ). + Start date in ISO 8601 format (YYYY-MM-DDTHH:MMZ). end_date : str - End date in ISO 8601 format (YYYY-MM-DD HH:MMZ). + End date in ISO 8601 format (YYYY-MM-DDTHH:MMZ). state : str | None, optional State abbreviation (e.g., 'TX'). If None (default), fetches from all networks (IEM limits this to a 24-hour window).