Add more test cases for mastr.download()

.github/workflows/ci-develop.yml

@@ -16,7 +16,7 @@ jobs:
       max-parallel: 1
       matrix:
         os: [macos-latest, ubuntu-latest, windows-latest]
-        python-version: ['3.10', '3.11', '3.12']
+        python-version: ['3.12', '3.13', '3.14']
     name: pytest (Python ${{ matrix.python-version }}) (OS ${{ matrix.os }})
 
     # Configure tests

.github/workflows/ci-production.yml

@@ -16,7 +16,7 @@ jobs:
       max-parallel: 1
       matrix:
         os: [macos-latest, ubuntu-latest, windows-latest]
-        python-version: ['3.10', '3.11', '3.12']
+        python-version: ['3.12', '3.13', '3.14']
     name: pytest (Python ${{ matrix.python-version }}) (OS ${{ matrix.os }})
 
     # Configure tests

.github/workflows/pypi-publish.yml

@@ -18,10 +18,10 @@ jobs:
     - uses: actions/checkout@v6
       with:
         ref: production
-    - name: Set up Python 3.11
+    - name: Set up Python 3.14
       uses: actions/setup-python@v6
       with:
-        python-version: "3.11"
+        python-version: "3.14"
 
     - name: Install pypa/build
       run: >-

.github/workflows/test-pypi-publish.yml

@@ -17,10 +17,10 @@ jobs:
       id-token: write
     steps:
     - uses: actions/checkout@v6
-    - name: Set up Python 3.11
+    - name: Set up Python 3.14
       uses: actions/setup-python@v6
       with:
-        python-version: "3.11"
+        python-version: "3.14"
 
     - name: Install pypa/build
       run: >-

.pre-commit-config.yaml

@@ -3,4 +3,4 @@ repos:
     rev: 22.6.0
     hooks:
       - id: black
-        language_version: python3.11
+        language_version: python3.14

.readthedocs.yml

@@ -5,7 +5,7 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.11"
+    python: "3.14"
 
 mkdocs:
   configuration: mkdocs.yml

CHANGELOG.md

@@ -9,16 +9,18 @@ and the versioning aims to respect [Semantic Versioning](http://semver.org/spec/
 
 ## [v0.xx.x] Unreleased - 202x-xx-xx
   ### Added
-  - Add `Mastr.generate_data_model` method that downloads the current MaStR documentation and generates SQLAlchemy tables from the XSD definitions; supports `english=True` for English column
-  names
+  - Add `Mastr.generate_data_model` method that downloads the current MaStR documentation and generates SQLAlchemy tables from the XSD definitions; supports `english=True` for English column names
     [#718](https://github.com/OpenEnergyPlatform/open-MaStR/pull/718)
   - Add `mastr_table_to_db_table` argument to `Mastr.download` to pass a custom database schema
     [#718](https://github.com/OpenEnergyPlatform/open-MaStR/pull/718)
   - Add `alter_database_tables` argument to `Mastr.download` to prevent open-mastr from issuing DDL statements
     [#718](https://github.com/OpenEnergyPlatform/open-MaStR/pull/718)
+  - Add more integration tests
+    [#753](https://github.com/OpenEnergyPlatform/open-MaStR/pull/753)
   - Add missing english translations
     [#754](https://github.com/OpenEnergyPlatform/open-MaStR/pull/754)
 
+
   ### Changed
   - Refactor data path configuration: remove `get_data_version_dir()`, add `MASTR_PROJECT_HOME_DIR` env var support to `get_project_home_dir()`
     [#748](https://github.com/OpenEnergyPlatform/open-MaStR/pull/748)
@@ -32,6 +34,8 @@ and the versioning aims to respect [Semantic Versioning](http://semver.org/spec/
     [#749](https://github.com/OpenEnergyPlatform/open-MaStR/pull/749)
   - Skip credential-dependent SOAP tests when credentials are absent
     [#749](https://github.com/OpenEnergyPlatform/open-MaStR/pull/749)
+    - Change default python version to 3.14
+    [#753](https://github.com/OpenEnergyPlatform/open-MaStR/pull/753)
 
   ### Removed
   - Remove `Mastr.translate`; English table and column names are now available via the `english=True` parameter in `generate_data_model` and `download`

environment.yml

@@ -3,4 +3,4 @@ channels:
     - conda-forge
     - defaults
 dependencies:
-    - python=3.11
+    - python=3.14

scripts/create_bulk_download_mockup.py

@@ -26,30 +26,10 @@
 
 sys.path.insert(0, str(Path(__file__).parent.parent))
 from open_mastr import Mastr
-from open_mastr.utils.constants import BULK_INCLUDE_TABLES_MAP
+from open_mastr.utils.constants import BULK_INCLUDE_TABLES_MAP, BULK_DATA
 
 MAX_RECORDS = 100  # should be dividable by four
-TECHNOLOGIES = [
-    "wind",
-    "solar",
-    "biomass",
-    "hydro",
-    "gsgk",
-    "combustion",
-    "nuclear",
-    "gas",
-    "storage",
-    "storage_units",
-    "electricity_consumer",
-    "location",
-    "market",
-    "grid",
-    "balancing_area",
-    "permit",
-    "deleted_units",
-    "deleted_market_actors",
-    "retrofit_units",
-]
+
 OUTPUT_ZIP = (
     Path(__file__).parent.parent / "tests" / "data" / "Gesamtdatenexport_mockup.zip"
 )
@@ -59,7 +39,7 @@
 DEBUG_DIR: str | None = "tmp/mastr-debug"
 
 INCLUDED_TABLE_NAMES = {
-    table for tech in TECHNOLOGIES for table in BULK_INCLUDE_TABLES_MAP[tech]
+    table for tech in BULK_DATA for table in BULK_INCLUDE_TABLES_MAP[tech]
 } | {"katalogwerte"}
 
 
@@ -151,7 +131,7 @@ def main() -> None:
     )
     print("-------IMPORTANT NOTE-------")
     mastr = Mastr(output_dir=tmpdir)
-    mastr.download(data=TECHNOLOGIES, bulk_cleansing=False)
+    mastr.download(data=BULK_DATA, date="20260401", bulk_cleansing=False)
 
     xml_dir = Path(tmpdir) / "data" / "xml_download"
     zips = sorted(xml_dir.glob("Gesamtdatenexport_*.zip"))

tests/test_mastr.py

@@ -111,6 +111,27 @@ def test_download_multiple_technologies(
     assert len(df_hydro) == NUMBER_ROWS_IN_MOCK_XML_FILES
 
 
+def test_download_data_none(
+    mastr: Mastr,
+    mockup_xml_zip_in_output_dir: Path,
+    mockup_docs_zip_in_output_dir: Path,
+) -> None:
+    mastr.download()
+
+    with mastr.engine.connect() as conn:
+        for table in [
+            "EinheitenWind",
+            "EinheitenBiomasse",
+            "EinheitenStromSpeicher",
+            "EinheitenSolar",
+            "EinheitenVerbrennung",
+            "EinheitenWasser",
+            "EinheitenStromVerbraucher",
+        ]:
+            result = conn.execute(sqlalchemy.text(f"SELECT COUNT(*) FROM {table}"))
+            assert result.scalar() == NUMBER_ROWS_IN_MOCK_XML_FILES
+
+
 def test_download_accumulates_across_calls(
     mastr: Mastr,
     mockup_xml_zip_in_output_dir: Path,
@@ -196,6 +217,93 @@ def _place_today_zip(save_path, *args, **kwargs):
     assert old_zip.exists()
 
 
+def test_download_different_dates_different_technologies(
+    mastr: Mastr,
+    mockup_docs_zip_in_output_dir: Path,
+) -> None:
+    """When the target zip doesn't exist,
+    delete_xml_files_not_from_given_date deletes ALL xml files before
+    downloading. Data for different technologies is preserved since they
+    are in separate tables."""
+    xml_dir = Path(mastr.output_dir) / "data" / "xml_download"
+    xml_dir.mkdir(parents=True, exist_ok=True)
+    date1 = "20230101"
+    date2 = "20230201"
+
+    def _place_zip(save_path, *args, **kwargs):
+        shutil.copy(MOCKUP_XML_ZIP, save_path)
+
+    with patch("open_mastr.mastr.download_xml_Mastr", side_effect=_place_zip):
+        mastr.download(data="wind", date=date1, keep_old_downloads=False)
+        mastr.download(data="biomass", date=date2, keep_old_downloads=False)
+
+    zip1 = xml_dir / f"Gesamtdatenexport_{date1}.zip"
+    zip2 = xml_dir / f"Gesamtdatenexport_{date2}.zip"
+    assert not zip1.exists()
+    assert zip2.exists()
+
+    df_wind = pd.read_sql("EinheitenWind", con=mastr.engine)
+    df_biomass = pd.read_sql("EinheitenBiomasse", con=mastr.engine)
+    assert len(df_wind) == NUMBER_ROWS_IN_MOCK_XML_FILES
+    assert len(df_biomass) == NUMBER_ROWS_IN_MOCK_XML_FILES
+    assert df_wind["DatumDownload"].iloc[0] == date1
+    assert df_biomass["DatumDownload"].iloc[0] == date2
+
+
+def test_download_different_dates_same_technology(
+    mastr: Mastr,
+    mockup_docs_zip_in_output_dir: Path,
+) -> None:
+    """When downloading same technology with different dates,
+    the first xml downloads are deleted and data is replaced.
+    """
+    xml_dir = Path(mastr.output_dir) / "data" / "xml_download"
+    xml_dir.mkdir(parents=True, exist_ok=True)
+    date1 = "20230101"
+    date2 = "20230201"
+
+    def _place_zip(save_path, *args, **kwargs):
+        shutil.copy(MOCKUP_XML_ZIP, save_path)
+
+    with patch("open_mastr.mastr.download_xml_Mastr", side_effect=_place_zip):
+        mastr.download(data="wind", date=date1, keep_old_downloads=False)
+        mastr.download(data="wind", date=date2, keep_old_downloads=False)
+
+    zip1 = xml_dir / f"Gesamtdatenexport_{date1}.zip"
+    zip2 = xml_dir / f"Gesamtdatenexport_{date2}.zip"
+    assert not zip1.exists()
+    assert zip2.exists()
+
+    df_wind = pd.read_sql("EinheitenWind", con=mastr.engine)
+    assert len(df_wind) == NUMBER_ROWS_IN_MOCK_XML_FILES
+    assert df_wind["DatumDownload"].iloc[0] == date2
+
+
+def test_download_date_zip_already_exists(
+    mastr: Mastr,
+    mockup_docs_zip_in_output_dir: Path,
+) -> None:
+    """When the target zip already exists, no deletion occurs.
+    Data for different technologies accumulates in their respective tables."""
+    xml_dir = Path(mastr.output_dir) / "data" / "xml_download"
+    xml_dir.mkdir(parents=True, exist_ok=True)
+    date1 = "20230101"
+    zip1 = xml_dir / f"Gesamtdatenexport_{date1}.zip"
+    shutil.copy(MOCKUP_XML_ZIP, zip1)
+
+    mastr.download(data="wind", date=date1, keep_old_downloads=False)
+    mastr.download(data="biomass", date=date1, keep_old_downloads=False)
+
+    assert zip1.exists()
+
+    df_wind = pd.read_sql("EinheitenWind", con=mastr.engine)
+    df_biomass = pd.read_sql("EinheitenBiomasse", con=mastr.engine)
+    assert len(df_wind) == NUMBER_ROWS_IN_MOCK_XML_FILES
+    assert len(df_biomass) == NUMBER_ROWS_IN_MOCK_XML_FILES
+    assert df_wind["DatumDownload"].iloc[0] == date1
+    assert df_biomass["DatumDownload"].iloc[0] == date1
+
+
 def test_download_no_alter_tables(
     mastr: Mastr,
     mockup_xml_zip_in_output_dir: Path,