diff --git a/.github/workflows/ci-develop.yml b/.github/workflows/ci-develop.yml index 5794d864..f83506c2 100644 --- a/.github/workflows/ci-develop.yml +++ b/.github/workflows/ci-develop.yml @@ -16,7 +16,7 @@ jobs: max-parallel: 1 matrix: os: [macos-latest, ubuntu-latest, windows-latest] - python-version: ['3.10', '3.11', '3.12'] + python-version: ['3.12', '3.13', '3.14'] name: pytest (Python ${{ matrix.python-version }}) (OS ${{ matrix.os }}) # Configure tests diff --git a/.github/workflows/ci-production.yml b/.github/workflows/ci-production.yml index cb8c3ec3..ca7d8d49 100644 --- a/.github/workflows/ci-production.yml +++ b/.github/workflows/ci-production.yml @@ -16,7 +16,7 @@ jobs: max-parallel: 1 matrix: os: [macos-latest, ubuntu-latest, windows-latest] - python-version: ['3.10', '3.11', '3.12'] + python-version: ['3.12', '3.13', '3.14'] name: pytest (Python ${{ matrix.python-version }}) (OS ${{ matrix.os }}) # Configure tests diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 3530461f..7760e358 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -18,10 +18,10 @@ jobs: - uses: actions/checkout@v6 with: ref: production - - name: Set up Python 3.11 + - name: Set up Python 3.14 uses: actions/setup-python@v6 with: - python-version: "3.11" + python-version: "3.14" - name: Install pypa/build run: >- diff --git a/.github/workflows/test-pypi-publish.yml b/.github/workflows/test-pypi-publish.yml index 31ae3d56..44b94c47 100644 --- a/.github/workflows/test-pypi-publish.yml +++ b/.github/workflows/test-pypi-publish.yml @@ -17,10 +17,10 @@ jobs: id-token: write steps: - uses: actions/checkout@v6 - - name: Set up Python 3.11 + - name: Set up Python 3.14 uses: actions/setup-python@v6 with: - python-version: "3.11" + python-version: "3.14" - name: Install pypa/build run: >- diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 821c6237..9b7bce2d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,4 +3,4 @@ repos: rev: 22.6.0 hooks: - id: black - language_version: python3.11 + language_version: python3.14 diff --git a/.readthedocs.yml b/.readthedocs.yml index 5944fdae..36e84b8f 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,7 +5,7 @@ version: 2 build: os: ubuntu-22.04 tools: - python: "3.11" + python: "3.14" mkdocs: configuration: mkdocs.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 57e93740..7f1405da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,16 +9,18 @@ and the versioning aims to respect [Semantic Versioning](http://semver.org/spec/ ## [v0.xx.x] Unreleased - 202x-xx-xx ### Added - - Add `Mastr.generate_data_model` method that downloads the current MaStR documentation and generates SQLAlchemy tables from the XSD definitions; supports `english=True` for English column - names + - Add `Mastr.generate_data_model` method that downloads the current MaStR documentation and generates SQLAlchemy tables from the XSD definitions; supports `english=True` for English column names [#718](https://github.com/OpenEnergyPlatform/open-MaStR/pull/718) - Add `mastr_table_to_db_table` argument to `Mastr.download` to pass a custom database schema [#718](https://github.com/OpenEnergyPlatform/open-MaStR/pull/718) - Add `alter_database_tables` argument to `Mastr.download` to prevent open-mastr from issuing DDL statements [#718](https://github.com/OpenEnergyPlatform/open-MaStR/pull/718) + - Add more integration tests + [#753](https://github.com/OpenEnergyPlatform/open-MaStR/pull/753) - Add missing english translations [#754](https://github.com/OpenEnergyPlatform/open-MaStR/pull/754) + ### Changed - Refactor data path configuration: remove `get_data_version_dir()`, add `MASTR_PROJECT_HOME_DIR` env var support to `get_project_home_dir()` [#748](https://github.com/OpenEnergyPlatform/open-MaStR/pull/748) @@ -32,6 +34,8 @@ and the versioning aims to respect [Semantic Versioning](http://semver.org/spec/ [#749](https://github.com/OpenEnergyPlatform/open-MaStR/pull/749) - Skip credential-dependent SOAP tests when credentials are absent [#749](https://github.com/OpenEnergyPlatform/open-MaStR/pull/749) + - Change default python version to 3.14 + [#753](https://github.com/OpenEnergyPlatform/open-MaStR/pull/753) ### Removed - Remove `Mastr.translate`; English table and column names are now available via the `english=True` parameter in `generate_data_model` and `download` diff --git a/environment.yml b/environment.yml index 66f2bd03..41980427 100644 --- a/environment.yml +++ b/environment.yml @@ -3,4 +3,4 @@ channels: - conda-forge - defaults dependencies: - - python=3.11 + - python=3.14 diff --git a/scripts/create_bulk_download_mockup.py b/scripts/create_bulk_download_mockup.py index 7c21c270..0fb74bc6 100644 --- a/scripts/create_bulk_download_mockup.py +++ b/scripts/create_bulk_download_mockup.py @@ -26,30 +26,10 @@ sys.path.insert(0, str(Path(__file__).parent.parent)) from open_mastr import Mastr -from open_mastr.utils.constants import BULK_INCLUDE_TABLES_MAP +from open_mastr.utils.constants import BULK_INCLUDE_TABLES_MAP, BULK_DATA MAX_RECORDS = 100 # should be dividable by four -TECHNOLOGIES = [ - "wind", - "solar", - "biomass", - "hydro", - "gsgk", - "combustion", - "nuclear", - "gas", - "storage", - "storage_units", - "electricity_consumer", - "location", - "market", - "grid", - "balancing_area", - "permit", - "deleted_units", - "deleted_market_actors", - "retrofit_units", -] + OUTPUT_ZIP = ( Path(__file__).parent.parent / "tests" / "data" / "Gesamtdatenexport_mockup.zip" ) @@ -59,7 +39,7 @@ DEBUG_DIR: str | None = "tmp/mastr-debug" INCLUDED_TABLE_NAMES = { - table for tech in TECHNOLOGIES for table in BULK_INCLUDE_TABLES_MAP[tech] + table for tech in BULK_DATA for table in BULK_INCLUDE_TABLES_MAP[tech] } | {"katalogwerte"} @@ -151,7 +131,7 @@ def main() -> None: ) print("-------IMPORTANT NOTE-------") mastr = Mastr(output_dir=tmpdir) - mastr.download(data=TECHNOLOGIES, bulk_cleansing=False) + mastr.download(data=BULK_DATA, date="20260401", bulk_cleansing=False) xml_dir = Path(tmpdir) / "data" / "xml_download" zips = sorted(xml_dir.glob("Gesamtdatenexport_*.zip")) diff --git a/tests/data/Gesamtdatenexport_mockup.zip b/tests/data/Gesamtdatenexport_mockup.zip index 6aefa57e..a882b2d0 100644 Binary files a/tests/data/Gesamtdatenexport_mockup.zip and b/tests/data/Gesamtdatenexport_mockup.zip differ diff --git a/tests/test_mastr.py b/tests/test_mastr.py index 7016a7c3..3fe184b2 100644 --- a/tests/test_mastr.py +++ b/tests/test_mastr.py @@ -111,6 +111,27 @@ def test_download_multiple_technologies( assert len(df_hydro) == NUMBER_ROWS_IN_MOCK_XML_FILES +def test_download_data_none( + mastr: Mastr, + mockup_xml_zip_in_output_dir: Path, + mockup_docs_zip_in_output_dir: Path, +) -> None: + mastr.download() + + with mastr.engine.connect() as conn: + for table in [ + "EinheitenWind", + "EinheitenBiomasse", + "EinheitenStromSpeicher", + "EinheitenSolar", + "EinheitenVerbrennung", + "EinheitenWasser", + "EinheitenStromVerbraucher", + ]: + result = conn.execute(sqlalchemy.text(f"SELECT COUNT(*) FROM {table}")) + assert result.scalar() == NUMBER_ROWS_IN_MOCK_XML_FILES + + def test_download_accumulates_across_calls( mastr: Mastr, mockup_xml_zip_in_output_dir: Path, @@ -196,6 +217,93 @@ def _place_today_zip(save_path, *args, **kwargs): assert old_zip.exists() +def test_download_different_dates_different_technologies( + mastr: Mastr, + mockup_docs_zip_in_output_dir: Path, +) -> None: + """When the target zip doesn't exist, + delete_xml_files_not_from_given_date deletes ALL xml files before + downloading. Data for different technologies is preserved since they + are in separate tables.""" + xml_dir = Path(mastr.output_dir) / "data" / "xml_download" + xml_dir.mkdir(parents=True, exist_ok=True) + date1 = "20230101" + date2 = "20230201" + + def _place_zip(save_path, *args, **kwargs): + shutil.copy(MOCKUP_XML_ZIP, save_path) + + with patch("open_mastr.mastr.download_xml_Mastr", side_effect=_place_zip): + mastr.download(data="wind", date=date1, keep_old_downloads=False) + mastr.download(data="biomass", date=date2, keep_old_downloads=False) + + zip1 = xml_dir / f"Gesamtdatenexport_{date1}.zip" + zip2 = xml_dir / f"Gesamtdatenexport_{date2}.zip" + assert not zip1.exists() + assert zip2.exists() + + df_wind = pd.read_sql("EinheitenWind", con=mastr.engine) + df_biomass = pd.read_sql("EinheitenBiomasse", con=mastr.engine) + assert len(df_wind) == NUMBER_ROWS_IN_MOCK_XML_FILES + assert len(df_biomass) == NUMBER_ROWS_IN_MOCK_XML_FILES + assert df_wind["DatumDownload"].iloc[0] == date1 + assert df_biomass["DatumDownload"].iloc[0] == date2 + + +def test_download_different_dates_same_technology( + mastr: Mastr, + mockup_docs_zip_in_output_dir: Path, +) -> None: + """When downloading same technology with different dates, + the first xml downloads are deleted and data is replaced. + """ + xml_dir = Path(mastr.output_dir) / "data" / "xml_download" + xml_dir.mkdir(parents=True, exist_ok=True) + date1 = "20230101" + date2 = "20230201" + + def _place_zip(save_path, *args, **kwargs): + shutil.copy(MOCKUP_XML_ZIP, save_path) + + with patch("open_mastr.mastr.download_xml_Mastr", side_effect=_place_zip): + mastr.download(data="wind", date=date1, keep_old_downloads=False) + mastr.download(data="wind", date=date2, keep_old_downloads=False) + + zip1 = xml_dir / f"Gesamtdatenexport_{date1}.zip" + zip2 = xml_dir / f"Gesamtdatenexport_{date2}.zip" + assert not zip1.exists() + assert zip2.exists() + + df_wind = pd.read_sql("EinheitenWind", con=mastr.engine) + assert len(df_wind) == NUMBER_ROWS_IN_MOCK_XML_FILES + assert df_wind["DatumDownload"].iloc[0] == date2 + + +def test_download_date_zip_already_exists( + mastr: Mastr, + mockup_docs_zip_in_output_dir: Path, +) -> None: + """When the target zip already exists, no deletion occurs. + Data for different technologies accumulates in their respective tables.""" + xml_dir = Path(mastr.output_dir) / "data" / "xml_download" + xml_dir.mkdir(parents=True, exist_ok=True) + date1 = "20230101" + zip1 = xml_dir / f"Gesamtdatenexport_{date1}.zip" + shutil.copy(MOCKUP_XML_ZIP, zip1) + + mastr.download(data="wind", date=date1, keep_old_downloads=False) + mastr.download(data="biomass", date=date1, keep_old_downloads=False) + + assert zip1.exists() + + df_wind = pd.read_sql("EinheitenWind", con=mastr.engine) + df_biomass = pd.read_sql("EinheitenBiomasse", con=mastr.engine) + assert len(df_wind) == NUMBER_ROWS_IN_MOCK_XML_FILES + assert len(df_biomass) == NUMBER_ROWS_IN_MOCK_XML_FILES + assert df_wind["DatumDownload"].iloc[0] == date1 + assert df_biomass["DatumDownload"].iloc[0] == date1 + + def test_download_no_alter_tables( mastr: Mastr, mockup_xml_zip_in_output_dir: Path,