From 413cf95d442b5d0fd77785aec51e7ab440b6d2aa Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 25 Jan 2024 14:27:22 +0100 Subject: [PATCH 01/72] Add ixmp4 to pyproject.toml optional deps --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 13989297e..1faa78e66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,10 +57,11 @@ docs = [ "sphinx_rtd_theme", "sphinxcontrib-bibtex", ] +ixmp4 = ["ixmp4"] report = ["genno[compat,graphviz]"] tutorial = ["jupyter"] tests = [ - "ixmp[report,tutorial]", + "ixmp[ixmp4,report,tutorial]", "memory_profiler", "nbclient >= 0.5", "pytest >= 5", From 68f9a4399528a330cce4d8ba11450c3e0734614a Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 25 Jan 2024 14:30:26 +0100 Subject: [PATCH 02/72] Add minimal/empty .backend.ixmp4.IXMP4Backend --- ixmp/__init__.py | 2 ++ ixmp/backend/ixmp4.py | 5 +++++ 2 files changed, 7 insertions(+) create mode 100644 ixmp/backend/ixmp4.py diff --git a/ixmp/__init__.py b/ixmp/__init__.py index a3c881884..2d51580aa 100644 --- a/ixmp/__init__.py +++ b/ixmp/__init__.py @@ -4,6 +4,7 @@ from ixmp._config import config from ixmp.backend import BACKENDS, IAMC_IDX, ItemType +from ixmp.backend.ixmp4 import IXMP4Backend from ixmp.backend.jdbc import JDBCBackend from ixmp.core.platform import Platform from ixmp.core.scenario import Scenario, TimeSeries @@ -47,6 +48,7 @@ # Register Backends provided by ixmp BACKENDS["jdbc"] = JDBCBackend +BACKENDS["ixmp4"] = IXMP4Backend # Register Models provided by ixmp MODELS.update( diff --git a/ixmp/backend/ixmp4.py b/ixmp/backend/ixmp4.py new file mode 100644 index 000000000..83ea19871 --- /dev/null +++ b/ixmp/backend/ixmp4.py @@ -0,0 +1,5 @@ +from ixmp.backend.base import CachingBackend + + +class IXMP4Backend(CachingBackend): + """Backend using :mod:`ixmp4`.""" From 1083b27d05b9e1c0a08489399bf90a45f8ad13ea Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 25 Jan 2024 14:31:12 +0100 Subject: [PATCH 03/72] Adjust test_platform.test_init --- ixmp/tests/core/test_platform.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ixmp/tests/core/test_platform.py b/ixmp/tests/core/test_platform.py index bca620420..c69bf33ba 100644 --- a/ixmp/tests/core/test_platform.py +++ b/ixmp/tests/core/test_platform.py @@ -22,7 +22,8 @@ class TestPlatform: def test_init(self): with pytest.raises( - ValueError, match=re.escape("backend class 'foo' not among ['jdbc']") + ValueError, + match=re.escape("backend class 'foo' not among ['ixmp4', 'jdbc']"), ): ixmp.Platform(backend="foo") From 1c608af06dc0175112a21eaf935733dcf71251ea Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 25 Jan 2024 14:41:06 +0100 Subject: [PATCH 04/72] Add TestPlatform.test_init1() Attempts to initialize each of JDBCBackend and IXMP4Backend. --- ixmp/tests/core/test_platform.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/ixmp/tests/core/test_platform.py b/ixmp/tests/core/test_platform.py index c69bf33ba..3efbe8b48 100644 --- a/ixmp/tests/core/test_platform.py +++ b/ixmp/tests/core/test_platform.py @@ -20,7 +20,7 @@ class TestPlatform: - def test_init(self): + def test_init0(self): with pytest.raises( ValueError, match=re.escape("backend class 'foo' not among ['ixmp4', 'jdbc']"), @@ -31,6 +31,17 @@ def test_init(self): mp = ixmp.Platform() assert "local" == mp.name + @pytest.mark.parametrize( + "backend, backend_args", + ( + ("jdbc", dict(driver="hsqldb", url="jdbc:hsqldb:mem:TestPlatform")), + ("ixmp4", dict()), + ), + ) + def test_init1(self, backend, backend_args): + # Platform can be instantiated + ixmp.Platform(backend=backend, **backend_args) + def test_getattr(self, test_mp): """Test __getattr__.""" with pytest.raises(AttributeError): From 89a615eb975456b36dd8a2e6eb9d37de28e47fe2 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 25 Jan 2024 14:42:58 +0100 Subject: [PATCH 05/72] Add stub/missing methods to IXMP4Backend --- ixmp/backend/ixmp4.py | 50 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/ixmp/backend/ixmp4.py b/ixmp/backend/ixmp4.py index 83ea19871..73bb5d398 100644 --- a/ixmp/backend/ixmp4.py +++ b/ixmp/backend/ixmp4.py @@ -3,3 +3,53 @@ class IXMP4Backend(CachingBackend): """Backend using :mod:`ixmp4`.""" + + # The below methods of base.Backend are not yet implemented + def _ni(self, *args, **kwargs): + raise NotImplementedError + + add_model_name = _ni + add_scenario_name = _ni + cat_get_elements = _ni + cat_list = _ni + cat_set_elements = _ni + check_out = _ni + clear_solution = _ni + clone = _ni + commit = _ni + delete = _ni + delete_geo = _ni + delete_item = _ni + delete_meta = _ni + discard_changes = _ni + get = _ni + get_data = _ni + get_doc = _ni + get_geo = _ni + get_meta = _ni + get_model_names = _ni + get_nodes = _ni + get_scenarios = _ni + get_scenario_names = _ni + get_timeslices = _ni + get_units = _ni + has_solution = _ni + init = _ni + init_item = _ni + is_default = _ni + item_delete_elements = _ni + item_get_elements = _ni + item_index = _ni + item_set_elements = _ni + last_update = _ni + list_items = _ni + remove_meta = _ni + run_id = _ni + set_as_default = _ni + set_data = _ni + set_doc = _ni + set_geo = _ni + set_meta = _ni + set_node = _ni + set_timeslice = _ni + set_unit = _ni From a1cc2876059d13215e6ecc5fcf10928d41dca9d4 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 25 Jan 2024 14:55:51 +0100 Subject: [PATCH 06/72] Implement IXMP4Backend.__init__ --- ixmp/backend/ixmp4.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/ixmp/backend/ixmp4.py b/ixmp/backend/ixmp4.py index 73bb5d398..86dfc4e91 100644 --- a/ixmp/backend/ixmp4.py +++ b/ixmp/backend/ixmp4.py @@ -1,9 +1,35 @@ +from typing import TYPE_CHECKING + from ixmp.backend.base import CachingBackend +if TYPE_CHECKING: + import ixmp4 + class IXMP4Backend(CachingBackend): """Backend using :mod:`ixmp4`.""" + _platform: "ixmp4.Platform" + + def __init__(self): + import ixmp4 + + # TODO Obtain `name` from the ixmp.Platform creating this Backend + name = "test" + + # Add an ixmp4.Platform using ixmp4's own configuration code + # TODO Move this to a test fixture + import ixmp4.conf + + dsn = "sqlite:///:memory:" + try: + ixmp4.conf.settings.toml.add_platform(name, dsn) + except ixmp4.core.exceptions.PlatformNotUnique: + pass + + # Instantiate and store + self._platform = ixmp4.Platform(name) + # The below methods of base.Backend are not yet implemented def _ni(self, *args, **kwargs): raise NotImplementedError From 4d8b0530939e772f7c819b207fea385bc3c27caf Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 25 Jan 2024 15:04:11 +0100 Subject: [PATCH 07/72] Add parametrized TestPlatform.mp fixture --- ixmp/tests/core/test_platform.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/ixmp/tests/core/test_platform.py b/ixmp/tests/core/test_platform.py index 3efbe8b48..b9a6f2106 100644 --- a/ixmp/tests/core/test_platform.py +++ b/ixmp/tests/core/test_platform.py @@ -3,7 +3,7 @@ import logging import re from sys import getrefcount -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Generator from weakref import getweakrefcount import pandas as pd @@ -20,6 +20,17 @@ class TestPlatform: + @pytest.fixture(params=list(ixmp.BACKENDS)) + def mp(self, request, test_mp) -> Generator[ixmp.Platform, None, None]: + """Fixture that yields 2 different platforms: one JDBC-backed, one ixmp4.""" + backend = request.param + + if backend == "jdbc": + yield test_mp + elif backend == "ixmp4": + # TODO Use a fixture similar to test_mp (with same contents) backed by ixmp4 + yield ixmp.Platform(backend="ixmp4") + def test_init0(self): with pytest.raises( ValueError, From 1f4a7c6473d4ca29885f736f0253dfe5aa141183 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 25 Jan 2024 15:06:36 +0100 Subject: [PATCH 08/72] Add parametrized TestPlatform.test_scenario_list() --- ixmp/tests/core/test_platform.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ixmp/tests/core/test_platform.py b/ixmp/tests/core/test_platform.py index b9a6f2106..4727106ef 100644 --- a/ixmp/tests/core/test_platform.py +++ b/ixmp/tests/core/test_platform.py @@ -58,6 +58,10 @@ def test_getattr(self, test_mp): with pytest.raises(AttributeError): test_mp.not_a_direct_backend_method + def test_scenario_list(self, mp): + scenario = mp.scenario_list() + assert isinstance(scenario, pd.DataFrame) + @pytest.fixture def log_level_mp(test_mp): From fc2ed4bc58ed52ae95f775058c8c3df777866adb Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 25 Jan 2024 15:22:10 +0100 Subject: [PATCH 09/72] Implement IXMP4Backend.get_scenarios() --- ixmp/backend/ixmp4.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ixmp/backend/ixmp4.py b/ixmp/backend/ixmp4.py index 86dfc4e91..b9294a205 100644 --- a/ixmp/backend/ixmp4.py +++ b/ixmp/backend/ixmp4.py @@ -19,6 +19,8 @@ def __init__(self): # Add an ixmp4.Platform using ixmp4's own configuration code # TODO Move this to a test fixture + # NB ixmp.tests.conftest.test_sqlite_mp exists, but is not importable (missing + # __init__.py) import ixmp4.conf dsn = "sqlite:///:memory:" @@ -30,6 +32,15 @@ def __init__(self): # Instantiate and store self._platform = ixmp4.Platform(name) + def get_scenarios(self, default, model, scenario): + # Current fails with: + # sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) no such table: run + # [SQL: SELECT DISTINCT run.model__id, run.scenario__id, run.version, + # run.is_default, run.id + # FROM run + # WHERE run.is_default = 1 ORDER BY run.id ASC] + return self._platform.runs.list() + # The below methods of base.Backend are not yet implemented def _ni(self, *args, **kwargs): raise NotImplementedError @@ -55,7 +66,6 @@ def _ni(self, *args, **kwargs): get_meta = _ni get_model_names = _ni get_nodes = _ni - get_scenarios = _ni get_scenario_names = _ni get_timeslices = _ni get_units = _ni From 1183b0c9cdcf71d1b03b9e6bfdff8de70e9a8f52 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 25 Jan 2024 15:39:46 +0100 Subject: [PATCH 10/72] Add ixmp4 to type checking packages --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8cb52c583..74dc91f22 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,6 +7,7 @@ repos: additional_dependencies: - genno - GitPython + - ixmp4 - nbclient - pandas-stubs - pytest From 86536c191da66683b39ddaf98b8a9dc613c72406 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Mon, 20 Jan 2025 14:25:05 +0100 Subject: [PATCH 11/72] =?UTF-8?q?=E2=9C=A8=20feat:=20Add=20default=20ixmp4?= =?UTF-8?q?=20platform?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ixmp/_config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ixmp/_config.py b/ixmp/_config.py index d5f9f139d..b5cc50149 100644 --- a/ixmp/_config.py +++ b/ixmp/_config.py @@ -68,6 +68,10 @@ def _platform_default(): "driver": "hsqldb", "path": next(_iter_config_paths())[1].joinpath("localdb", "default"), }, + "ixmp4-local": { + "class": "ixmp4", + "dsn": "sqlite:///:memory:", + }, } From 6d461f52b3a5783c3c6c8e6036ab393f480d9d15 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Mon, 20 Jan 2025 14:27:57 +0100 Subject: [PATCH 12/72] =?UTF-8?q?=E2=9C=A8=20feat:=20Parametrize=20test=5F?= =?UTF-8?q?mp=20and=20enable=20IXMP4Backend.=5F=5Finit=5F=5F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ixmp/backend/ixmp4.py | 28 +++++------ ixmp/core/platform.py | 3 ++ ixmp/testing/__init__.py | 79 ++++++++++++++++++++++++-------- ixmp/testing/data.py | 2 +- ixmp/tests/core/test_platform.py | 18 ++------ 5 files changed, 81 insertions(+), 49 deletions(-) diff --git a/ixmp/backend/ixmp4.py b/ixmp/backend/ixmp4.py index b9294a205..e2e60f1e6 100644 --- a/ixmp/backend/ixmp4.py +++ b/ixmp/backend/ixmp4.py @@ -11,34 +11,34 @@ class IXMP4Backend(CachingBackend): _platform: "ixmp4.Platform" - def __init__(self): - import ixmp4 + def __init__(self, **kwargs) -> None: + from ixmp4 import Platform + from ixmp4.core.exceptions import PlatformNotFound - # TODO Obtain `name` from the ixmp.Platform creating this Backend - name = "test" + # TODO Handle errors or make sure name is always present for this backend + name = kwargs.pop("name") # Add an ixmp4.Platform using ixmp4's own configuration code # TODO Move this to a test fixture # NB ixmp.tests.conftest.test_sqlite_mp exists, but is not importable (missing # __init__.py) + # NB test_platform is parametrized for both backends, but + # TestPlatform::test_init1 calls this function without defining an + # ixmp4.Platform first import ixmp4.conf - dsn = "sqlite:///:memory:" try: + ixmp4.conf.settings.toml.get_platform(name) + except PlatformNotFound: + # TODO Handle errors or make sure dsn is always present when the platform is + # not known + dsn = kwargs.pop("dsn") ixmp4.conf.settings.toml.add_platform(name, dsn) - except ixmp4.core.exceptions.PlatformNotUnique: - pass # Instantiate and store - self._platform = ixmp4.Platform(name) + self._platform = Platform(name) def get_scenarios(self, default, model, scenario): - # Current fails with: - # sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) no such table: run - # [SQL: SELECT DISTINCT run.model__id, run.scenario__id, run.version, - # run.is_default, run.id - # FROM run - # WHERE run.is_default = 1 ORDER BY run.id ASC] return self._platform.runs.list() # The below methods of base.Backend are not yet implemented diff --git a/ixmp/core/platform.py b/ixmp/core/platform.py index 7107773e0..bc5c7686a 100644 --- a/ixmp/core/platform.py +++ b/ixmp/core/platform.py @@ -81,6 +81,9 @@ def __init__( # Overwrite any platform config with explicit keyword arguments kwargs.update(backend_args) + if backend == "ixmp4": + kwargs["name"] = self.name + # Retrieve the Backend class try: backend_class_name = kwargs.pop("class") diff --git a/ixmp/testing/__init__.py b/ixmp/testing/__init__.py index 1eee1f923..63080c66b 100644 --- a/ixmp/testing/__init__.py +++ b/ixmp/testing/__init__.py @@ -35,16 +35,20 @@ import logging import os import shutil +from collections.abc import Generator from contextlib import contextmanager, nullcontext from copy import deepcopy from itertools import chain from pathlib import Path +from typing import Any import pint import pytest from click.testing import CliRunner +from ixmp4.conf.base import PlatformInfo +from ixmp4.data.backend import SqliteTestBackend -from ixmp import Platform, cli +from ixmp import BACKENDS, Platform, cli from ixmp import config as ixmp_config from .data import ( @@ -89,7 +93,7 @@ # Pytest hooks -def pytest_addoption(parser): +def pytest_addoption(parser: pytest.Parser) -> None: """Add the ``--user-config`` command-line option to pytest.""" parser.addoption( "--ixmp-jvm-mem", @@ -103,7 +107,7 @@ def pytest_addoption(parser): ) -def pytest_sessionstart(session): +def pytest_sessionstart(session: pytest.Session) -> None: """Unset any configuration read from the user's directory.""" from ixmp.backend import jdbc @@ -117,7 +121,7 @@ def pytest_sessionstart(session): jdbc._GC_AGGRESSIVE = False -def pytest_report_header(config, start_path): +def pytest_report_header(config, start_path) -> str: """Add the ixmp configuration to the pytest report header.""" return f"ixmp config: {repr(ixmp_config.values)}" @@ -137,7 +141,7 @@ def invoke(self, *args, **kwargs): @pytest.fixture(scope="module") -def mp(test_mp): +def mp(test_mp: Platform) -> Generator[Platform, Any, None]: """A :class:`.Platform` containing test data. This fixture is **module** -scoped, and is used in :mod:`.test_platform`, @@ -149,23 +153,29 @@ def mp(test_mp): @pytest.fixture(scope="session") -def test_data_path(): +def test_data_path() -> Path: """Path to the directory containing test data.""" return Path(__file__).parents[1].joinpath("tests", "data") -@pytest.fixture(scope="module") -def test_mp(request, tmp_env, test_data_path): +@pytest.fixture(scope="module", params=list(BACKENDS.keys())) +def test_mp( + request: pytest.FixtureRequest, tmp_env, test_data_path +) -> Generator[Platform, Any, None]: """An empty :class:`.Platform` connected to a temporary, in-memory database. This fixture has **module** scope: the same Platform is reused for all tests in a module. """ - yield from _platform_fixture(request, tmp_env, test_data_path) + yield from _platform_fixture( + request, tmp_env, test_data_path, backend=request.param + ) @pytest.fixture(scope="session") -def tmp_env(pytestconfig, tmp_path_factory): +def tmp_env( + pytestconfig: pytest.Config, tmp_path_factory: pytest.TempPathFactory +) -> Generator[os._Environ[str], Any, None]: """Return the os.environ dict with the IXMP_DATA variable set. IXMP_DATA will point to a temporary directory that is unique to the test session. @@ -187,13 +197,13 @@ def tmp_env(pytestconfig, tmp_path_factory): @pytest.fixture(scope="session") -def tutorial_path(): +def tutorial_path() -> Path: """Path to the directory containing the tutorials.""" return Path(__file__).parents[2].joinpath("tutorial") @pytest.fixture(scope="session") -def ureg(): +def ureg() -> Generator[pint.UnitRegistry, Any, None]: """Application-wide units registry.""" registry = pint.get_application_registry() @@ -242,8 +252,8 @@ def protect_rename_dims(): RENAME_DIMS.update(saved) -@pytest.fixture(scope="function") -def test_mp_f(request, tmp_env, test_data_path): +@pytest.fixture(scope="function", params=list(BACKENDS.keys())) +def test_mp_f(request: pytest.FixtureRequest, tmp_env, test_data_path): """An empty :class:`Platform` connected to a temporary, in-memory database. This fixture has **function** scope: the same Platform is reused for one test @@ -253,7 +263,9 @@ def test_mp_f(request, tmp_env, test_data_path): -------- test_mp """ - yield from _platform_fixture(request, tmp_env, test_data_path) + yield from _platform_fixture( + request, tmp_env, test_data_path, backend=request.param + ) # Assertions @@ -384,19 +396,38 @@ def create_test_platform(tmp_path, data_path, name, **properties): # Private utilities -def _platform_fixture(request, tmp_env, test_data_path): +def _platform_fixture( + request: pytest.FixtureRequest, tmp_env, test_data_path, backend: str +) -> Generator[Platform, Any, None]: """Helper for :func:`test_mp` and other fixtures.""" # Long, unique name for the platform. # Remove '/' so that the name can be used in URL tests. platform_name = request.node.nodeid.replace("/", " ") # Add a platform - ixmp_config.add_platform( - platform_name, "jdbc", "hsqldb", url=f"jdbc:hsqldb:mem:{platform_name}" - ) + if backend == "jdbc": + ixmp_config.add_platform( + platform_name, backend, "hsqldb", url=f"jdbc:hsqldb:mem:{platform_name}" + ) + elif backend == "ixmp4": + import ixmp4.conf + + # Setup ixmp4 backend and run DB migrations + sqlite = SqliteTestBackend( + PlatformInfo(name=platform_name, dsn="sqlite:///:memory:") + ) + sqlite.setup() + + # Add DB to ixmp4 config + ixmp4.conf.settings.toml.add_platform( + name=platform_name, dsn="sqlite:///:memory:" + ) + + # Add ixmp4 backend to ixmp platforms + ixmp_config.add_platform(platform_name, backend) # Launch Platform - mp = Platform(name=platform_name) + mp = Platform(name=platform_name, backend=backend) yield mp # Teardown: don't show log messages when destroying the platform, even if @@ -406,3 +437,11 @@ def _platform_fixture(request, tmp_env, test_data_path): # Remove from config ixmp_config.remove_platform(platform_name) + + if backend == "ixmp4": + assert sqlite # to satisfy type checkers + + # Close DB connection and remove platform + sqlite.close() + sqlite.teardown() + ixmp4.conf.settings.toml.remove_platform(platform_name) diff --git a/ixmp/testing/data.py b/ixmp/testing/data.py index 8f551bf67..f00043550 100644 --- a/ixmp/testing/data.py +++ b/ixmp/testing/data.py @@ -250,7 +250,7 @@ def make_dantzig( return scen -def populate_test_platform(platform): +def populate_test_platform(platform: Platform) -> None: """Populate `platform` with data for testing. Many of the tests in :mod:`ixmp.tests.core` depend on this set of data. diff --git a/ixmp/tests/core/test_platform.py b/ixmp/tests/core/test_platform.py index 4727106ef..6a069a1c1 100644 --- a/ixmp/tests/core/test_platform.py +++ b/ixmp/tests/core/test_platform.py @@ -3,7 +3,7 @@ import logging import re from sys import getrefcount -from typing import TYPE_CHECKING, Generator +from typing import TYPE_CHECKING from weakref import getweakrefcount import pandas as pd @@ -20,17 +20,6 @@ class TestPlatform: - @pytest.fixture(params=list(ixmp.BACKENDS)) - def mp(self, request, test_mp) -> Generator[ixmp.Platform, None, None]: - """Fixture that yields 2 different platforms: one JDBC-backed, one ixmp4.""" - backend = request.param - - if backend == "jdbc": - yield test_mp - elif backend == "ixmp4": - # TODO Use a fixture similar to test_mp (with same contents) backed by ixmp4 - yield ixmp.Platform(backend="ixmp4") - def test_init0(self): with pytest.raises( ValueError, @@ -46,7 +35,8 @@ def test_init0(self): "backend, backend_args", ( ("jdbc", dict(driver="hsqldb", url="jdbc:hsqldb:mem:TestPlatform")), - ("ixmp4", dict()), + # TODO use this/default name for ixmp4 platforms without passing it manually + ("ixmp4", dict(name="ixmp4-local")), ), ) def test_init1(self, backend, backend_args): @@ -58,7 +48,7 @@ def test_getattr(self, test_mp): with pytest.raises(AttributeError): test_mp.not_a_direct_backend_method - def test_scenario_list(self, mp): + def test_scenario_list(self, mp: ixmp.Platform) -> None: scenario = mp.scenario_list() assert isinstance(scenario, pd.DataFrame) From 90765e6a8e3de9925d00e13729a639aa12dc0335 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 21 Jan 2025 12:34:35 +0100 Subject: [PATCH 13/72] =?UTF-8?q?=E2=9C=A8=20feat:=20Use=20correct=20DB=20?= =?UTF-8?q?in=20test=20setup=20&=20add=20some=20backend=20functions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ixmp/backend/ixmp4.py | 90 ++++++++++++++++++++++---------- ixmp/core/platform.py | 3 -- ixmp/testing/__init__.py | 10 +--- ixmp/tests/core/test_platform.py | 3 +- 4 files changed, 65 insertions(+), 41 deletions(-) diff --git a/ixmp/backend/ixmp4.py b/ixmp/backend/ixmp4.py index e2e60f1e6..729efcf27 100644 --- a/ixmp/backend/ixmp4.py +++ b/ixmp/backend/ixmp4.py @@ -1,52 +1,90 @@ -from typing import TYPE_CHECKING +import logging +from collections.abc import Generator, MutableMapping, Sequence +from typing import TYPE_CHECKING, Any, Optional + +from ixmp4 import Platform +from ixmp4.conf.base import PlatformInfo +from ixmp4.data.backend import SqliteTestBackend +from ixmp4.data.backend.base import Backend as ixmp4_backend from ixmp.backend.base import CachingBackend if TYPE_CHECKING: import ixmp4 +log = logging.getLogger(__name__) + class IXMP4Backend(CachingBackend): """Backend using :mod:`ixmp4`.""" _platform: "ixmp4.Platform" + _backend: ixmp4_backend - def __init__(self, **kwargs) -> None: - from ixmp4 import Platform - from ixmp4.core.exceptions import PlatformNotFound + def __init__(self, _backend: Optional[ixmp4_backend] = None) -> None: + # Create a default backend if None is provided + if not _backend: + log.warning("Falling back to default SqliteBackend 'ixmp4-local'") + sqlite = SqliteTestBackend( + PlatformInfo(name="ixmp4-local", dsn="sqlite:///:memory:") + ) + sqlite.setup() - # TODO Handle errors or make sure name is always present for this backend - name = kwargs.pop("name") + self._backend = _backend if _backend else sqlite + + # Instantiate and store + self._platform = Platform(_backend=self._backend) - # Add an ixmp4.Platform using ixmp4's own configuration code - # TODO Move this to a test fixture - # NB ixmp.tests.conftest.test_sqlite_mp exists, but is not importable (missing - # __init__.py) - # NB test_platform is parametrized for both backends, but - # TestPlatform::test_init1 calls this function without defining an - # ixmp4.Platform first - import ixmp4.conf + # def __del__(self) -> None: + # self.close_db() + # Platform methods + @classmethod + def handle_config(cls, args: Sequence, kwargs: MutableMapping) -> dict[str, Any]: + msg = "Unhandled {} args to Backend.handle_config(): {!r}" + if len(args): + raise ValueError(msg.format("positional", args)) + + info: dict[str, Any] = {} try: - ixmp4.conf.settings.toml.get_platform(name) - except PlatformNotFound: - # TODO Handle errors or make sure dsn is always present when the platform is - # not known - dsn = kwargs.pop("dsn") - ixmp4.conf.settings.toml.add_platform(name, dsn) + info["_backend"] = kwargs["_backend"] + except KeyError: + raise ValueError(f"Missing key '_backend' for backend=ixmp4; got {kwargs}") - # Instantiate and store - self._platform = Platform(name) + return info + + # def close_db(self) -> None: + # self._backend.close() + + def add_scenario_name(self, name: str) -> None: + self._platform.scenarios.create(name) + + # TODO clarify: ixmp4.Run doesn't have a name, but is the new ixmp.Scenario + # should it have a name or are these scenario names okay? + def get_scenario_names(self) -> Generator[str, None, None]: + for scenario in self._platform.scenarios.list(): + yield scenario.name + + def add_model_name(self, name: str) -> None: + self._platform.models.create(name) + + def get_model_names(self) -> Generator[str, None, None]: + for model in self._platform.models.list(): + yield model.name def get_scenarios(self, default, model, scenario): return self._platform.runs.list() + def set_unit(self, name: str, comment: str) -> None: + self._platform.units.create(name).docs = comment + + def get_units(self) -> list[str]: + return [unit.name for unit in self._platform.units.list()] + # The below methods of base.Backend are not yet implemented def _ni(self, *args, **kwargs): raise NotImplementedError - add_model_name = _ni - add_scenario_name = _ni cat_get_elements = _ni cat_list = _ni cat_set_elements = _ni @@ -64,11 +102,8 @@ def _ni(self, *args, **kwargs): get_doc = _ni get_geo = _ni get_meta = _ni - get_model_names = _ni get_nodes = _ni - get_scenario_names = _ni get_timeslices = _ni - get_units = _ni has_solution = _ni init = _ni init_item = _ni @@ -88,4 +123,3 @@ def _ni(self, *args, **kwargs): set_meta = _ni set_node = _ni set_timeslice = _ni - set_unit = _ni diff --git a/ixmp/core/platform.py b/ixmp/core/platform.py index bc5c7686a..7107773e0 100644 --- a/ixmp/core/platform.py +++ b/ixmp/core/platform.py @@ -81,9 +81,6 @@ def __init__( # Overwrite any platform config with explicit keyword arguments kwargs.update(backend_args) - if backend == "ixmp4": - kwargs["name"] = self.name - # Retrieve the Backend class try: backend_class_name = kwargs.pop("class") diff --git a/ixmp/testing/__init__.py b/ixmp/testing/__init__.py index 63080c66b..2f5afae6c 100644 --- a/ixmp/testing/__init__.py +++ b/ixmp/testing/__init__.py @@ -410,7 +410,7 @@ def _platform_fixture( platform_name, backend, "hsqldb", url=f"jdbc:hsqldb:mem:{platform_name}" ) elif backend == "ixmp4": - import ixmp4.conf + # import ixmp4.conf # Setup ixmp4 backend and run DB migrations sqlite = SqliteTestBackend( @@ -418,13 +418,8 @@ def _platform_fixture( ) sqlite.setup() - # Add DB to ixmp4 config - ixmp4.conf.settings.toml.add_platform( - name=platform_name, dsn="sqlite:///:memory:" - ) - # Add ixmp4 backend to ixmp platforms - ixmp_config.add_platform(platform_name, backend) + ixmp_config.add_platform(platform_name, backend, _backend=sqlite) # Launch Platform mp = Platform(name=platform_name, backend=backend) @@ -444,4 +439,3 @@ def _platform_fixture( # Close DB connection and remove platform sqlite.close() sqlite.teardown() - ixmp4.conf.settings.toml.remove_platform(platform_name) diff --git a/ixmp/tests/core/test_platform.py b/ixmp/tests/core/test_platform.py index 6a069a1c1..cf6c7e7fc 100644 --- a/ixmp/tests/core/test_platform.py +++ b/ixmp/tests/core/test_platform.py @@ -35,8 +35,7 @@ def test_init0(self): "backend, backend_args", ( ("jdbc", dict(driver="hsqldb", url="jdbc:hsqldb:mem:TestPlatform")), - # TODO use this/default name for ixmp4 platforms without passing it manually - ("ixmp4", dict(name="ixmp4-local")), + ("ixmp4", dict()), ), ) def test_init1(self, backend, backend_args): From b336b58773210f05902fbe0aa712914e07cba512 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 21 Jan 2025 13:50:09 +0100 Subject: [PATCH 14/72] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20Avoid=20?= =?UTF-8?q?installing=20ixmp4=20for=20Python=203.9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1faa78e66..de1bd1e8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,7 +61,8 @@ ixmp4 = ["ixmp4"] report = ["genno[compat,graphviz]"] tutorial = ["jupyter"] tests = [ - "ixmp[ixmp4,report,tutorial]", + "ixmp[report,tutorial]", + "ixmp[ixmp4]; python_version > '3.9'", "memory_profiler", "nbclient >= 0.5", "pytest >= 5", From 8a923377bec2bda0719c08b01c24200ba5f4e353 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 21 Jan 2025 14:08:41 +0100 Subject: [PATCH 15/72] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20Avoid=20?= =?UTF-8?q?running=20ixmp4=20tests=20on=20Python=203.9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ixmp/backend/ixmp4.py | 14 +++++++------- ixmp/testing/__init__.py | 16 +++++++++++----- ixmp/tests/core/test_platform.py | 9 +++++++-- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/ixmp/backend/ixmp4.py b/ixmp/backend/ixmp4.py index 729efcf27..5fecf47fc 100644 --- a/ixmp/backend/ixmp4.py +++ b/ixmp/backend/ixmp4.py @@ -2,15 +2,11 @@ from collections.abc import Generator, MutableMapping, Sequence from typing import TYPE_CHECKING, Any, Optional -from ixmp4 import Platform -from ixmp4.conf.base import PlatformInfo -from ixmp4.data.backend import SqliteTestBackend -from ixmp4.data.backend.base import Backend as ixmp4_backend - from ixmp.backend.base import CachingBackend if TYPE_CHECKING: import ixmp4 + from ixmp4.data.backend.base import Backend as ixmp4_backend log = logging.getLogger(__name__) @@ -19,9 +15,13 @@ class IXMP4Backend(CachingBackend): """Backend using :mod:`ixmp4`.""" _platform: "ixmp4.Platform" - _backend: ixmp4_backend + _backend: "ixmp4_backend" + + def __init__(self, _backend: Optional["ixmp4_backend"] = None) -> None: + from ixmp4 import Platform + from ixmp4.conf.base import PlatformInfo + from ixmp4.data.backend import SqliteTestBackend - def __init__(self, _backend: Optional[ixmp4_backend] = None) -> None: # Create a default backend if None is provided if not _backend: log.warning("Falling back to default SqliteBackend 'ixmp4-local'") diff --git a/ixmp/testing/__init__.py b/ixmp/testing/__init__.py index 2f5afae6c..3b1dd230e 100644 --- a/ixmp/testing/__init__.py +++ b/ixmp/testing/__init__.py @@ -35,6 +35,7 @@ import logging import os import shutil +import sys from collections.abc import Generator from contextlib import contextmanager, nullcontext from copy import deepcopy @@ -45,8 +46,6 @@ import pint import pytest from click.testing import CliRunner -from ixmp4.conf.base import PlatformInfo -from ixmp4.data.backend import SqliteTestBackend from ixmp import BACKENDS, Platform, cli from ixmp import config as ixmp_config @@ -89,6 +88,12 @@ "tmp_env", ] +# Parametrize platforms for jdbc and ixmp4 +backends = list(BACKENDS.keys()) +# ixmp4 is not published for Python 3.9 +if sys.version_info < (3, 10): + backends.remove("ixmp4") + # Pytest hooks @@ -158,7 +163,7 @@ def test_data_path() -> Path: return Path(__file__).parents[1].joinpath("tests", "data") -@pytest.fixture(scope="module", params=list(BACKENDS.keys())) +@pytest.fixture(scope="module", params=backends) def test_mp( request: pytest.FixtureRequest, tmp_env, test_data_path ) -> Generator[Platform, Any, None]: @@ -252,7 +257,7 @@ def protect_rename_dims(): RENAME_DIMS.update(saved) -@pytest.fixture(scope="function", params=list(BACKENDS.keys())) +@pytest.fixture(scope="function", params=backends) def test_mp_f(request: pytest.FixtureRequest, tmp_env, test_data_path): """An empty :class:`Platform` connected to a temporary, in-memory database. @@ -410,7 +415,8 @@ def _platform_fixture( platform_name, backend, "hsqldb", url=f"jdbc:hsqldb:mem:{platform_name}" ) elif backend == "ixmp4": - # import ixmp4.conf + from ixmp4.conf.base import PlatformInfo + from ixmp4.data.backend import SqliteTestBackend # Setup ixmp4 backend and run DB migrations sqlite = SqliteTestBackend( diff --git a/ixmp/tests/core/test_platform.py b/ixmp/tests/core/test_platform.py index cf6c7e7fc..b806f4969 100644 --- a/ixmp/tests/core/test_platform.py +++ b/ixmp/tests/core/test_platform.py @@ -2,6 +2,7 @@ import logging import re +import sys from sys import getrefcount from typing import TYPE_CHECKING from weakref import getweakrefcount @@ -18,12 +19,16 @@ if TYPE_CHECKING: from ixmp import Platform +min_ixmp4_version = pytest.mark.skipif( + sys.version_info < (3, 10), reason="ixmp4 requires Python 3.10 or higher" +) + class TestPlatform: def test_init0(self): with pytest.raises( ValueError, - match=re.escape("backend class 'foo' not among ['ixmp4', 'jdbc']"), + match=re.escape("backend class 'foo' not among"), ): ixmp.Platform(backend="foo") @@ -35,7 +40,7 @@ def test_init0(self): "backend, backend_args", ( ("jdbc", dict(driver="hsqldb", url="jdbc:hsqldb:mem:TestPlatform")), - ("ixmp4", dict()), + pytest.param("ixmp4", dict(), marks=min_ixmp4_version), ), ) def test_init1(self, backend, backend_args): From 6fd393a595c1e1bb366ae185482caa6bac664fd4 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 28 Jan 2025 08:40:14 +0100 Subject: [PATCH 16/72] =?UTF-8?q?=E2=9C=A8=20feat:=20Use=20specific=20ixmp?= =?UTF-8?q?4=20branch=20as=20dependency?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index de1bd1e8d..5cc1ca911 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,8 @@ docs = [ "sphinx_rtd_theme", "sphinxcontrib-bibtex", ] -ixmp4 = ["ixmp4"] +# Change ixmp4 to new release after https://github.com/iiasa/ixmp4/pull/153 is merged +ixmp4 = ["ixmp4 @ git+https://github.com/iiasa/ixmp4@enh/expose-run-is-default"] report = ["genno[compat,graphviz]"] tutorial = ["jupyter"] tests = [ From dd04c732a1f06a304f456aaa9f6234485c65cc9b Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 28 Jan 2025 08:41:42 +0100 Subject: [PATCH 17/72] =?UTF-8?q?=E2=9C=A8=20feat:=20Make=20type=20hints?= =?UTF-8?q?=20more=20concise?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ixmp/backend/base.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/ixmp/backend/base.py b/ixmp/backend/base.py index fd8ab0e54..56b2a022f 100644 --- a/ixmp/backend/base.py +++ b/ixmp/backend/base.py @@ -204,12 +204,13 @@ def set_node( """ @abstractmethod - def get_nodes(self) -> Iterable[tuple[str, Optional[str], str, str]]: + def get_nodes(self) -> Iterable[tuple[str, Optional[str], Optional[str], str]]: """Iterate over all nodes stored on the Platform. Yields ------- tuple + The members of each tuple are: ========= =========== === @@ -820,7 +821,7 @@ def has_solution(self, s: Scenario) -> bool: """ @abstractmethod - def list_items(self, s: Scenario, type: str) -> list[str]: + def list_items(self, s: Scenario, type: Literal["set", "par", "equ"]) -> list[str]: """Return a list of names of items of `type`. Parameters @@ -832,7 +833,7 @@ def list_items(self, s: Scenario, type: str) -> list[str]: def init_item( self, s: Scenario, - type: str, + type: Literal["set", "par", "equ", "var"], name: str, idx_sets: Sequence[str], idx_names: Optional[Sequence[str]], @@ -871,7 +872,9 @@ def delete_item(self, s: Scenario, type: str, name: str) -> None: """ @abstractmethod - def item_index(self, s: Scenario, name: str, sets_or_names: str) -> list[str]: + def item_index( + self, s: Scenario, name: str, sets_or_names: Literal["sets", "names"] + ) -> list[str]: """Return the index sets or names of item `name`. Parameters @@ -932,7 +935,7 @@ def item_get_elements( def item_set_elements( self, s: Scenario, - type: str, + type: Literal["par", "set"], name: str, elements: Iterable[tuple[Any, Optional[float], Optional[str], Optional[str]]], ) -> None: @@ -974,7 +977,9 @@ def item_set_elements( """ @abstractmethod - def item_delete_elements(self, s: Scenario, type: str, name: str, keys) -> None: + def item_delete_elements( + self, s: Scenario, type: Literal["par", "set"], name: str, keys + ) -> None: """Remove elements of item `name`. Parameters From 8e36620e6f4c8a473a07fcd9c7bcf1970588baf6 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 28 Jan 2025 08:42:04 +0100 Subject: [PATCH 18/72] =?UTF-8?q?=E2=9C=A8=20feat:=20Add=20various=20platf?= =?UTF-8?q?orm=20functions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ixmp/backend/ixmp4.py | 350 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 329 insertions(+), 21 deletions(-) diff --git a/ixmp/backend/ixmp4.py b/ixmp/backend/ixmp4.py index 5fecf47fc..372c9aa07 100644 --- a/ixmp/backend/ixmp4.py +++ b/ixmp/backend/ixmp4.py @@ -1,8 +1,12 @@ import logging -from collections.abc import Generator, MutableMapping, Sequence -from typing import TYPE_CHECKING, Any, Optional +from collections.abc import Generator, Iterable, MutableMapping, Sequence +from typing import TYPE_CHECKING, Any, Literal, Optional, Union, cast + +import pandas as pd from ixmp.backend.base import CachingBackend +from ixmp.core.scenario import Scenario +from ixmp.core.timeseries import TimeSeries if TYPE_CHECKING: import ixmp4 @@ -17,6 +21,10 @@ class IXMP4Backend(CachingBackend): _platform: "ixmp4.Platform" _backend: "ixmp4_backend" + # Mapping from ixmp.TimeSeries object to the underlying ixmp4.Run object (or + # subclasses of either) + index: dict[TimeSeries, "ixmp4.Run"] = {} + def __init__(self, _backend: Optional["ixmp4_backend"] = None) -> None: from ixmp4 import Platform from ixmp4.conf.base import PlatformInfo @@ -56,8 +64,10 @@ def handle_config(cls, args: Sequence, kwargs: MutableMapping) -> dict[str, Any] # def close_db(self) -> None: # self._backend.close() + # Modifying the Platform + def add_scenario_name(self, name: str) -> None: - self._platform.scenarios.create(name) + self._platform.scenarios.create(name=name) # TODO clarify: ixmp4.Run doesn't have a name, but is the new ixmp.Scenario # should it have a name or are these scenario names okay? @@ -66,7 +76,7 @@ def get_scenario_names(self) -> Generator[str, None, None]: yield scenario.name def add_model_name(self, name: str) -> None: - self._platform.models.create(name) + self._platform.models.create(name=name) def get_model_names(self) -> Generator[str, None, None]: for model in self._platform.models.list(): @@ -76,11 +86,325 @@ def get_scenarios(self, default, model, scenario): return self._platform.runs.list() def set_unit(self, name: str, comment: str) -> None: - self._platform.units.create(name).docs = comment + self._platform.units.create(name=name).docs = comment def get_units(self) -> list[str]: return [unit.name for unit in self._platform.units.list()] + def set_node( + self, + name: str, + parent: Optional[str] = None, + hierarchy: Optional[str] = None, + synonym: Optional[str] = None, + ) -> None: + if parent: + log.warning(f"Discarding parent parameter {parent}; unused in ixmp4.") + if synonym: + log.warning(f"Discarding synonym parameter {synonym}; unused in ixmp4.") + if hierarchy is None: + raise ValueError("IXMP4Backend.set_node() requires to specify 'hierarchy'!") + self._platform.regions.create(name=name, hierarchy=hierarchy) + + def get_nodes(self) -> list[tuple[str, str, None, str]]: + return [ + (region.name, region.name, None, region.hierarchy) + for region in self._platform.regions.list() + ] + + # Modifying the Run object + + def _index_and_set_attrs(self, run: "ixmp4.Run", ts: TimeSeries) -> None: + """Add *run* to index and update attributes of *ts*. + + Helper for init and get. + """ + # Add to index + self.index[ts] = run + + # Retrieve the version of the ixmp4.Run object + v = run.version + if ts.version is None: + # The default version was requested; update the attribute + ts.version = v + elif v != ts.version: # pragma: no cover + # Something went wrong on the ixmp4 side + raise RuntimeError(f"got version {v} instead of {ts.version}") + + def init(self, ts: TimeSeries, annotation: str) -> None: + run = self._platform.runs.create(model=ts.model, scenario=ts.scenario) + # TODO either do log.warning that annotation is unused or + # run.docs = annotation + self._index_and_set_attrs(run, ts) + + def get(self, ts: TimeSeries) -> None: + v = int(ts.version) if ts.version else None + run = self._platform.runs.get(model=ts.model, scenario=ts.scenario, version=v) + self._index_and_set_attrs(run, ts) + + def check_out(self, ts: TimeSeries, timeseries_only: bool) -> None: + log.warning("ixmp4 backed Scenarios/Runs don't need to be checked out!") + + def discard_changes(self, ts: TimeSeries) -> None: + log.warning( + "ixmp4 backed Scenarios/Runs are changed immediately, can't " + "discard changes. To avoid the need, be sure to start work on " + "fresh clones." + ) + + def commit(self, ts: TimeSeries, comment: str) -> None: + log.warning( + "ixmp4 backed Scenarios/Runs are changed immediately, no need for " + "a commit!" + ) + + def clear_solution(self, s: Scenario, from_year: Optional[int] = None) -> None: + if from_year: + log.warning( + "ixmp4 does not support removing the solution only after a " + "certain year" + ) + self.index[s].optimization.remove_solution() + + def set_as_default(self, ts: TimeSeries) -> None: + # TODO are we okay with always loading a Run for this or should we adapt + # TimeSeries to have a `_run` attribute that has the Run loaded? + self.index[ts].set_as_default() + + # Information about the Run + + def run_id(self, ts: TimeSeries) -> int: + # TODO is the Run.version really what this function is after? + return self.index[ts].version + + def is_default(self, ts: TimeSeries) -> bool: + return self.index[ts].is_default() + + # Handle various items + + # TODO: type hints + def _get_repo( + self, + s: Scenario, + type: Literal["scalar", "indexset", "set", "par", "equ", "var"], + ): + if type == "scalar": + return self.index[s].optimization.scalars + if type == "indexset": + return self.index[s].optimization.indexsets + if type == "set": + return self.index[s].optimization.tables + elif type == "par": + return self.index[s].optimization.parameters + elif type == "equ": + return self.index[s].optimization.equations + else: # "var" + return self.index[s].optimization.variables + + def init_item( + self, + s: Scenario, + type: Literal["set", "par", "equ", "var"], + name: str, + idx_sets: Sequence[str], + idx_names: Optional[Sequence[str]], + ) -> None: + # TODO how are scalars created? Scalars require a value in ixmp4 + # In base::item_get_elements, it sounds like "equ" and "var" can also target + # scalars, whereas below, inspired from jdbc, I'm only linking "par" to scalars + if type == "set" and len(idx_sets) == 0: + repo = self._get_repo(s=s, type="indexset") + repo.create(name=name) + else: + repo = self._get_repo(s=s, type=type) + repo.create( + name=name, constrained_to_indexsets=idx_sets, column_names=idx_names + ) + + def list_items(self, s: Scenario, type: Literal["set", "par", "equ"]) -> list[str]: + if type == "set": + indexset_repo = self._get_repo(s=s, type="indexset") + set_repo = self._get_repo(s=s, type=type) + return [item.name for item in indexset_repo.list()] + [ + item.name for item in set_repo.list() + ] + else: + repo = self._get_repo(s=s, type=type) + return [item.name for item in repo.list()] + + # TODO type hints + def _find_item(self, s: Scenario, name: str): + # NOTE this currently assumes that `name` will only be present once among + # Tables, Parameters, Equations, Variables. This is in line with the assumption + # made in the Java backend, but ixmp4 enforces no such constraint. + for type in ("scalar", "indexset", "set", "par", "equ", "var"): + repo = self._get_repo(s=s, type=type) + item_list = repo.list(name=name) + if ( + len(item_list) == 1 + ): # ixmp4 enforces names to be unique among individual item classes + return (type, item_list[0]) + + def _get_item( + self, + s: Scenario, + name: str, + type: Literal["scalar", "indexset", "set", "par", "equ", "var"], + ): + # TODO add try except here to always return using IndexError from repo.list()[0] + repo = self._get_repo(s=s, type=type) + item_list = repo.list(name=name) + if ( + len(item_list) == 1 + ): # ixmp4 enforces names to be unique among individual item classes + return (type, item_list[0]) + + # TODO mypy says this function is missing a return statement. But why? + # AFAICT, items.items() will always contain something, so we will always return + # something (unless we raise). Am I missing something? + def item_index( # type: ignore[return] + self, s: Scenario, name: str, sets_or_names: Literal["sets", "names"] + ) -> list[str]: + type, item = self._find_item(s=s, name=name) + if item is None: + raise LookupError(f"No item called {name} found on this Scenario!") + if type == "indexset": + return cast(list[str], []) + else: + return ( + [column.name for column in item.columns] + if sets_or_names == "names" + else item.constrained_to_indexsets + ) + + def _add_data_to_set( + self, s: Scenario, name: str, key: str | list[str], comment: Optional[str] + ) -> None: + if comment: + log.warning( + "`comment` currently unused with ixmp4 when adding data to Tables." + ) + # Assumption: if key is just one value, we're dealing with an IndexSet + if isinstance(key, str): + self.index[s].optimization.indexsets.get(name=name).add(key) + else: + table = self.index[s].optimization.tables.get(name=name) + data_to_add = { + table.constrained_to_indexsets[i]: key[i] for i in range(len(key)) + } + table.add(data=data_to_add) + + def _create_scalar( + self, + s: Scenario, + name: str, + value: float, + unit: Optional[str], + comment: Optional[str], + ) -> None: + scalar = self.index[s].optimization.scalars.create( + name=name, value=value, unit=unit + ) + if comment: + scalar.docs = comment + + ### TODO + ### + ### Currently, tests fail because they expect 'cases' to be predefined on the test_mp + + def _add_data_to_parameter( + self, + s: Scenario, + name: str, + key: str | list[str], + value: float, + unit: str, + comment: Optional[str], + ) -> None: + if comment: + log.warning( + "`comment` currently unused with ixmp4 when adding data to " + "Parameters." + ) + parameter = self.index[s].optimization.parameters.get(name=name) + # TODO there's got to be a better way for handling possible lists + if isinstance(key, str): + key = [key] + data_to_add: dict[str, Union[list[float], list[str]]] = { + parameter.constrained_to_indexsets[i]: [key[i]] for i in range(len(key)) + } + data_to_add["values"] = [value] + data_to_add["units"] = [unit] + parameter.add(data=data_to_add) + + def item_set_elements( + self, + s: Scenario, + type: Literal["par", "set"], + name: str, + elements: Iterable[tuple[Any, Optional[float], Optional[str], Optional[str]]], + ) -> None: + for key, value, unit, comment in elements: + if type == "set": + self._add_data_to_set(s=s, name=name, key=key, comment=comment) + else: + if key is None: + assert value, "Creating a Scalar requires a value!" + self._create_scalar( + s=s, name=name, value=value, unit=unit, comment=comment + ) + else: + assert value, "Adding data to a Parameter requires a value!" + assert unit, "Adding data to a Parameter requires a unit!" + self._add_data_to_parameter( + s=s, name=name, key=key, value=value, unit=unit, comment=comment + ) + + def _get_set_data( + self, s: Scenario, name: str, filters: Optional[dict[str, list[Any]]] = None + ) -> Union[pd.Series, pd.DataFrame]: + # TODO handle filters + from ixmp4.core import IndexSet, Table + from ixmp4.core.exceptions import NotFound + + item: Union[IndexSet, Table] + try: + repo = self._get_repo(s=s, type="indexset") + item = cast(IndexSet, repo.get(name=name)) + except NotFound: + repo = self._get_repo(s=s, type="set") + item = cast(Table, repo.get(name=name)) + + return ( + pd.DataFrame(item.data) if isinstance(item, Table) else pd.Series(item.data) + ) + + def item_get_elements( + self, + s: Scenario, + type: Literal["equ", "par", "set", "var"], + name: str, + filters: Optional[dict[str, list[Any]]] = None, + ) -> Union[dict[str, Any], pd.Series, pd.DataFrame]: + # TODO handle filters + if type == "set": + return self._get_set_data(s=s, name=name, filters=filters) + # TODO this is not handling scalars at the moment, but maybe try with type, + # except NotFound, try scalar? + else: + # TODO this can really only be Equation, Parameter, or Variable, so cast + # as such? + _, item = self._get_item(s=s, name=name, type=type) + data = pd.DataFrame(item.data) + print(data) + if type == "par": + data.rename(columns={"values": "value", "units": "unit"}, inplace=True) + else: + data.rename( + columns={"levels": "level", "marginals": "marginal"}, inplace=True + ) + return data + # The below methods of base.Backend are not yet implemented def _ni(self, *args, **kwargs): raise NotImplementedError @@ -88,38 +412,22 @@ def _ni(self, *args, **kwargs): cat_get_elements = _ni cat_list = _ni cat_set_elements = _ni - check_out = _ni - clear_solution = _ni clone = _ni - commit = _ni delete = _ni delete_geo = _ni delete_item = _ni delete_meta = _ni - discard_changes = _ni - get = _ni get_data = _ni get_doc = _ni get_geo = _ni get_meta = _ni - get_nodes = _ni get_timeslices = _ni has_solution = _ni - init = _ni - init_item = _ni - is_default = _ni item_delete_elements = _ni - item_get_elements = _ni - item_index = _ni - item_set_elements = _ni last_update = _ni - list_items = _ni remove_meta = _ni - run_id = _ni - set_as_default = _ni set_data = _ni set_doc = _ni set_geo = _ni set_meta = _ni - set_node = _ni set_timeslice = _ni From 8dee3f61305bf676501aaf2d9a5443d7e1e3f4f3 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 28 Jan 2025 08:50:35 +0100 Subject: [PATCH 19/72] =?UTF-8?q?=E2=9C=A8=20feat:=20Add=20type=20hints=20?= =?UTF-8?q?to=20platform=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ixmp/tests/core/test_platform.py | 48 +++++++++++++++++--------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/ixmp/tests/core/test_platform.py b/ixmp/tests/core/test_platform.py index b806f4969..f0b125cab 100644 --- a/ixmp/tests/core/test_platform.py +++ b/ixmp/tests/core/test_platform.py @@ -3,8 +3,10 @@ import logging import re import sys +from collections.abc import Generator +from pathlib import Path from sys import getrefcount -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from weakref import getweakrefcount import pandas as pd @@ -17,7 +19,7 @@ from ixmp.testing import DATA, assert_logs, models if TYPE_CHECKING: - from ixmp import Platform + pass min_ixmp4_version = pytest.mark.skipif( sys.version_info < (3, 10), reason="ixmp4 requires Python 3.10 or higher" @@ -25,7 +27,7 @@ class TestPlatform: - def test_init0(self): + def test_init0(self) -> None: with pytest.raises( ValueError, match=re.escape("backend class 'foo' not among"), @@ -43,11 +45,11 @@ def test_init0(self): pytest.param("ixmp4", dict(), marks=min_ixmp4_version), ), ) - def test_init1(self, backend, backend_args): + def test_init1(self, backend: str, backend_args: dict[str, str]) -> None: # Platform can be instantiated ixmp.Platform(backend=backend, **backend_args) - def test_getattr(self, test_mp): + def test_getattr(self, test_mp: ixmp.Platform) -> None: """Test __getattr__.""" with pytest.raises(AttributeError): test_mp.not_a_direct_backend_method @@ -58,7 +60,7 @@ def test_scenario_list(self, mp: ixmp.Platform) -> None: @pytest.fixture -def log_level_mp(test_mp): +def log_level_mp(test_mp: ixmp.Platform) -> Generator[ixmp.Platform, Any, None]: """A fixture that preserves the log level of *test_mp*.""" tmp = test_mp.get_log_level() yield test_mp @@ -78,7 +80,7 @@ def log_level_mp(test_mp): ("FOO", ValueError), ], ) -def test_log_level(log_level_mp, level, exc): +def test_log_level(log_level_mp: ixmp.Platform, level: str, exc) -> None: """Log level can be set and retrieved.""" if exc is None: log_level_mp.set_log_level(level) @@ -88,12 +90,12 @@ def test_log_level(log_level_mp, level, exc): log_level_mp.set_log_level(level) -def test_scenario_list(mp): +def test_scenario_list(mp: ixmp.Platform) -> None: scenario = mp.scenario_list(model="Douglas Adams")["scenario"] assert scenario[0] == "Hitchhiker" -def test_export_timeseries_data(mp: "Platform", tmp_path) -> None: +def test_export_timeseries_data(mp: ixmp.Platform, tmp_path: Path) -> None: path = tmp_path / "export.csv" mp.export_timeseries_data(path, model="Douglas Adams", unit="???", region="World") @@ -108,7 +110,7 @@ def test_export_timeseries_data(mp: "Platform", tmp_path) -> None: assert_frame_equal(exp, obs) -def test_export_ts_wrong_params(test_mp, tmp_path): +def test_export_ts_wrong_params(test_mp: ixmp.Platform, tmp_path: Path) -> None: """Platform.export_timeseries_data to raise error with wrong parameters.""" path = tmp_path / "export.csv" with raises(ValueError, match="Invalid arguments"): @@ -121,7 +123,7 @@ def test_export_ts_wrong_params(test_mp, tmp_path): ) -def test_export_ts_of_all_runs(mp, tmp_path): +def test_export_ts_of_all_runs(mp: ixmp.Platform, tmp_path: Path) -> None: """Export timeseries of all runs.""" path = tmp_path / "export.csv" @@ -154,7 +156,7 @@ def test_export_ts_of_all_runs(mp, tmp_path): assert 4 == len(obs) -def test_export_timeseries_data_empty(mp, tmp_path): +def test_export_timeseries_data_empty(mp: ixmp.Platform, tmp_path: Path) -> None: """Dont export data if given models/scenarios do not have any runs.""" path = tmp_path / "export.csv" model = "model-no-run" @@ -166,16 +168,16 @@ def test_export_timeseries_data_empty(mp, tmp_path): assert 0 == len(pd.read_csv(path, index_col=False, header=0)) -def test_unit_list(test_mp): +def test_unit_list(test_mp: ixmp.Platform) -> None: units = test_mp.units() assert ("cases" in units) is True -def test_add_unit(test_mp): +def test_add_unit(test_mp: ixmp.Platform) -> None: test_mp.add_unit("test", "just testing") -def test_regions(test_mp): +def test_regions(test_mp: ixmp.Platform) -> None: regions = test_mp.regions() # Result has the expected columns @@ -187,7 +189,7 @@ def test_regions(test_mp): assert all([list(obs.loc[0]) == ["World", None, "World", "common"]]) -def test_add_region(test_mp): +def test_add_region(test_mp: ixmp.Platform) -> None: # Region can be added test_mp.add_region("foo", "bar", "World") @@ -197,7 +199,7 @@ def test_add_region(test_mp): assert all([list(obs.loc[0]) == ["foo", None, "World", "bar"]]) -def test_add_region_synonym(test_mp): +def test_add_region_synonym(test_mp: ixmp.Platform) -> None: test_mp.add_region("foo", "bar", "World") test_mp.add_region_synonym("foo2", "foo") regions = test_mp.regions() @@ -213,7 +215,7 @@ def test_add_region_synonym(test_mp): assert_frame_equal(obs, exp) -def test_timeslices(test_mp): +def test_timeslices(test_mp: ixmp.Platform) -> None: timeslices = test_mp.timeslices() obs = timeslices[timeslices.category == "Common"] # result has all attributes of time slice @@ -222,7 +224,7 @@ def test_timeslices(test_mp): assert all([list(obs.iloc[0]) == ["Year", "Common", 1.0]]) -def test_add_timeslice(test_mp): +def test_add_timeslice(test_mp: ixmp.Platform) -> None: test_mp.add_timeslice("January, 1st", "Days", 1.0 / 366) timeslices = test_mp.timeslices() obs = timeslices[timeslices.category == "Days"] @@ -232,7 +234,7 @@ def test_add_timeslice(test_mp): assert all([list(obs.iloc[0]) == ["January, 1st", "Days", 1.0 / 366]]) -def test_add_timeslice_duplicate(caplog, test_mp): +def test_add_timeslice_duplicate(caplog, test_mp: ixmp.Platform) -> None: test_mp.add_timeslice("foo_slice", "foo_category", 0.2) # Adding same name with different duration raises an error @@ -245,7 +247,7 @@ def test_add_timeslice_duplicate(caplog, test_mp): test_mp.add_timeslice("foo_slice", "bar_category", 0.2) -def test_weakref(): +def test_weakref() -> None: """Weak references allow Platforms to be del'd while Scenarios live.""" mp = ixmp.Platform( backend="jdbc", @@ -288,11 +290,11 @@ def test_weakref(): # *s* is garbage-collected at this point -def test_add_model_name(test_mp): +def test_add_model_name(test_mp: ixmp.Platform) -> None: test_mp.add_model_name("new_model_name") assert "new_model_name" in test_mp.get_model_names() -def test_add_scenario_name(test_mp): +def test_add_scenario_name(test_mp: ixmp.Platform) -> None: test_mp.add_scenario_name("new_scenario_name") assert "new_scenario_name" in test_mp.get_scenario_names() From 00c7819a00fb039b1f99492a969af69ca37e8af6 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 28 Jan 2025 09:40:07 +0100 Subject: [PATCH 20/72] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20Update?= =?UTF-8?q?=20ixmp4-branch=20dependency?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5cc1ca911..14bb83484 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,8 +57,11 @@ docs = [ "sphinx_rtd_theme", "sphinxcontrib-bibtex", ] -# Change ixmp4 to new release after https://github.com/iiasa/ixmp4/pull/153 is merged -ixmp4 = ["ixmp4 @ git+https://github.com/iiasa/ixmp4@enh/expose-run-is-default"] +# Change ixmp4 to new release after https://github.com/iiasa/ixmp4/pull/164 is merged +ixmp4 = [ + "ixmp4 @ git+https://github.com/iiasa/ixmp4@enh/make-equation-indexsets-optional", + "gamsapi[core,transfer] >= 45.7.0", +] report = ["genno[compat,graphviz]"] tutorial = ["jupyter"] tests = [ From 046ea87ace067e5038d0401cfb61c2d545a90575 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 28 Jan 2025 09:40:49 +0100 Subject: [PATCH 21/72] =?UTF-8?q?=E2=9C=A8=20feat:=20Add=20setup=20functio?= =?UTF-8?q?n=20for=20ixmp4-backed=20Platform?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ixmp/testing/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ixmp/testing/__init__.py b/ixmp/testing/__init__.py index 3b1dd230e..bfa8917bb 100644 --- a/ixmp/testing/__init__.py +++ b/ixmp/testing/__init__.py @@ -401,6 +401,13 @@ def create_test_platform(tmp_path, data_path, name, **properties): # Private utilities +# TODO Check if this setup is necessary when running all tests (not just test_platform) +def _setup_ixmp4_platform(mp: Platform) -> None: + """Set up an ixmp4-backed Platform with things hardcoded in Java.""" + mp.add_unit("cases", comment="As pre-defined in Java.") + mp.add_unit("km", comment="As pre-defined in Java.") + + def _platform_fixture( request: pytest.FixtureRequest, tmp_env, test_data_path, backend: str ) -> Generator[Platform, Any, None]: @@ -429,6 +436,8 @@ def _platform_fixture( # Launch Platform mp = Platform(name=platform_name, backend=backend) + if backend == "ixmp4": + _setup_ixmp4_platform(mp=mp) yield mp # Teardown: don't show log messages when destroying the platform, even if From 815cafa6d1473db31fb5dd072ac6dbd169396668 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 28 Jan 2025 09:41:05 +0100 Subject: [PATCH 22/72] =?UTF-8?q?=E2=9C=A8=20feat:=20Add=20has=5Fsolution(?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ixmp/backend/ixmp4.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/ixmp/backend/ixmp4.py b/ixmp/backend/ixmp4.py index 372c9aa07..dd2c46c68 100644 --- a/ixmp/backend/ixmp4.py +++ b/ixmp/backend/ixmp4.py @@ -180,7 +180,10 @@ def run_id(self, ts: TimeSeries) -> int: def is_default(self, ts: TimeSeries) -> bool: return self.index[ts].is_default() - # Handle various items + def has_solution(self, s: Scenario) -> bool: + return self.index[s].optimization.has_solution() + + # Handle optimization items # TODO: type hints def _get_repo( @@ -308,10 +311,6 @@ def _create_scalar( if comment: scalar.docs = comment - ### TODO - ### - ### Currently, tests fail because they expect 'cases' to be predefined on the test_mp - def _add_data_to_parameter( self, s: Scenario, @@ -396,7 +395,6 @@ def item_get_elements( # as such? _, item = self._get_item(s=s, name=name, type=type) data = pd.DataFrame(item.data) - print(data) if type == "par": data.rename(columns={"values": "value", "units": "unit"}, inplace=True) else: @@ -405,6 +403,8 @@ def item_get_elements( ) return data + # Handle timeslices + # The below methods of base.Backend are not yet implemented def _ni(self, *args, **kwargs): raise NotImplementedError @@ -422,7 +422,6 @@ def _ni(self, *args, **kwargs): get_geo = _ni get_meta = _ni get_timeslices = _ni - has_solution = _ni item_delete_elements = _ni last_update = _ni remove_meta = _ni From 27422d4253a9ec606b55c94a8c097b9d01ec76e1 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 28 Jan 2025 11:17:36 +0100 Subject: [PATCH 23/72] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20Add=20Wo?= =?UTF-8?q?rld-region=20definition=20to=20ixmp4=20platform=20setup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ixmp/testing/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ixmp/testing/__init__.py b/ixmp/testing/__init__.py index bfa8917bb..9a9861799 100644 --- a/ixmp/testing/__init__.py +++ b/ixmp/testing/__init__.py @@ -406,6 +406,7 @@ def _setup_ixmp4_platform(mp: Platform) -> None: """Set up an ixmp4-backed Platform with things hardcoded in Java.""" mp.add_unit("cases", comment="As pre-defined in Java.") mp.add_unit("km", comment="As pre-defined in Java.") + mp.add_region(region="World", hierarchy="common") def _platform_fixture( From 49952cfaaac67ad42266cb434ff5c64540d1eba0 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 28 Jan 2025 11:17:56 +0100 Subject: [PATCH 24/72] =?UTF-8?q?=E2=9C=A8=20feat:=20Add=20more=20platform?= =?UTF-8?q?=20functions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ixmp/backend/ixmp4.py | 293 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 236 insertions(+), 57 deletions(-) diff --git a/ixmp/backend/ixmp4.py b/ixmp/backend/ixmp4.py index dd2c46c68..cc5c03b16 100644 --- a/ixmp/backend/ixmp4.py +++ b/ixmp/backend/ixmp4.py @@ -5,11 +5,18 @@ import pandas as pd from ixmp.backend.base import CachingBackend +from ixmp.core.platform import Platform from ixmp.core.scenario import Scenario from ixmp.core.timeseries import TimeSeries if TYPE_CHECKING: import ixmp4 + from ixmp4.core.optimization.equation import Equation + from ixmp4.core.optimization.indexset import IndexSet + from ixmp4.core.optimization.parameter import Parameter + from ixmp4.core.optimization.scalar import Scalar + from ixmp4.core.optimization.table import Table + from ixmp4.core.optimization.variable import Variable from ixmp4.data.backend.base import Backend as ixmp4_backend log = logging.getLogger(__name__) @@ -46,7 +53,16 @@ def __init__(self, _backend: Optional["ixmp4_backend"] = None) -> None: # def __del__(self) -> None: # self.close_db() + def set_log_level(self, level: int) -> None: + # Set the level of the 'ixmp.backend.ixmp4' logger. Messages are handled by the + # 'ixmp' logger; see ixmp/__init__.py. + log.setLevel(level) + + # def get_log_level(self): + # return super().get_log_level() + # Platform methods + @classmethod def handle_config(cls, args: Sequence, kwargs: MutableMapping) -> dict[str, Any]: msg = "Unhandled {} args to Backend.handle_config(): {!r}" @@ -103,12 +119,16 @@ def set_node( if synonym: log.warning(f"Discarding synonym parameter {synonym}; unused in ixmp4.") if hierarchy is None: - raise ValueError("IXMP4Backend.set_node() requires to specify 'hierarchy'!") + log.warning( + "IXMP4Backend.set_node() requires to specify 'hierarchy'! " + "Using 'None' as the meaningsless default." + ) + hierarchy = "None" self._platform.regions.create(name=name, hierarchy=hierarchy) - def get_nodes(self) -> list[tuple[str, str, None, str]]: + def get_nodes(self) -> list[tuple[str, None, str, str]]: return [ - (region.name, region.name, None, region.hierarchy) + (region.name, None, region.name, region.hierarchy) for region in self._platform.regions.list() ] @@ -137,6 +157,36 @@ def init(self, ts: TimeSeries, annotation: str) -> None: # run.docs = annotation self._index_and_set_attrs(run, ts) + def clone( + self, + s: Scenario, + platform_dest: Platform, + model: str, + scenario: str, + annotation: str, + keep_solution: bool, + first_model_year: Optional[int] = None, + ) -> Scenario: + # TODO either do log.warning that annotation is unused or + # run.docs = annotation + # TODO Should this be supported? + if first_model_year: + log.warning( + "ixmp4-backed Scenarios don't support cloning from " + "`first_model_year` only!" + ) + # TODO Is this enough? ixmp4 doesn't support cloning to a different platform at + # the moment, but maybe we can imitate this here? (Access + # platform_dest.backend._platform to create a new Run?) + cloned_s = Scenario( + mp=platform_dest, model=model, scenario=scenario, annotation=annotation + ) + cloned_run = self.index[s].clone( + model=model, scenario=scenario, keep_solution=keep_solution + ) + self._index_and_set_attrs(cloned_run, cloned_s) + return cloned_s + def get(self, ts: TimeSeries) -> None: v = int(ts.version) if ts.version else None run = self._platform.runs.get(model=ts.model, scenario=ts.scenario, version=v) @@ -154,21 +204,17 @@ def discard_changes(self, ts: TimeSeries) -> None: def commit(self, ts: TimeSeries, comment: str) -> None: log.warning( - "ixmp4 backed Scenarios/Runs are changed immediately, no need for " - "a commit!" + "ixmp4 backed Scenarios/Runs are changed immediately, no need for a commit!" ) def clear_solution(self, s: Scenario, from_year: Optional[int] = None) -> None: if from_year: log.warning( - "ixmp4 does not support removing the solution only after a " - "certain year" + "ixmp4 does not support removing the solution only after a certain year" ) self.index[s].optimization.remove_solution() def set_as_default(self, ts: TimeSeries) -> None: - # TODO are we okay with always loading a Run for this or should we adapt - # TimeSeries to have a `_run` attribute that has the Run loaded? self.index[ts].set_as_default() # Information about the Run @@ -253,35 +299,55 @@ def _get_item( s: Scenario, name: str, type: Literal["scalar", "indexset", "set", "par", "equ", "var"], - ): - # TODO add try except here to always return using IndexError from repo.list()[0] - repo = self._get_repo(s=s, type=type) - item_list = repo.list(name=name) - if ( - len(item_list) == 1 - ): # ixmp4 enforces names to be unique among individual item classes - return (type, item_list[0]) - - # TODO mypy says this function is missing a return statement. But why? - # AFAICT, items.items() will always contain something, so we will always return - # something (unless we raise). Am I missing something? - def item_index( # type: ignore[return] + ) -> Union[ + "Scalar", + "IndexSet", + "Table", + "Equation", + "Variable", + "Parameter", + ]: + return self._get_repo(s=s, type=type).get(name=name) + + def _get_indexset_or_table( + self, s: Scenario, name: str + ) -> Union["IndexSet", "Table"]: + from ixmp4.core import IndexSet, Table + from ixmp4.core.exceptions import NotFound + + try: + repo = self._get_repo(s=s, type="indexset") + return cast(IndexSet, repo.get(name=name)) + except NotFound: + repo = self._get_repo(s=s, type="set") + return cast(Table, repo.get(name=name)) + + def item_index( self, s: Scenario, name: str, sets_or_names: Literal["sets", "names"] ) -> list[str]: - type, item = self._find_item(s=s, name=name) - if item is None: - raise LookupError(f"No item called {name} found on this Scenario!") - if type == "indexset": + _, item = self._find_item(s=s, name=name) + # NOTE Using isinstance allows adequate attribute access + if ( + isinstance(item, IndexSet) + or isinstance(item, Scalar) + or ( + isinstance(item, Variable) + and sets_or_names == "names" + and item.column_names is None + ) + ): return cast(list[str], []) else: - return ( - [column.name for column in item.columns] - if sets_or_names == "names" - else item.constrained_to_indexsets + index_names = ( + item.column_names if sets_or_names == "names" else item.indexsets ) + assert index_names, ( + f"Requested {sets_or_names}, but these are None for item {item.name}" + ) + return index_names def _add_data_to_set( - self, s: Scenario, name: str, key: str | list[str], comment: Optional[str] + self, s: Scenario, name: str, key: Union[str, list[str]], comment: Optional[str] ) -> None: if comment: log.warning( @@ -292,9 +358,10 @@ def _add_data_to_set( self.index[s].optimization.indexsets.get(name=name).add(key) else: table = self.index[s].optimization.tables.get(name=name) - data_to_add = { - table.constrained_to_indexsets[i]: key[i] for i in range(len(key)) - } + # TODO should we enforce in ixmp4 that when constrained_to_indexsets + # contains duplicate values, column_names must be provided? + keys = table.column_names if table.column_names else table.indexsets + data_to_add = {keys[i]: [key[i]] for i in range(len(key))} table.add(data=data_to_add) def _create_scalar( @@ -315,25 +382,26 @@ def _add_data_to_parameter( self, s: Scenario, name: str, - key: str | list[str], + key: Union[str, list[str]], value: float, unit: str, comment: Optional[str], ) -> None: if comment: log.warning( - "`comment` currently unused with ixmp4 when adding data to " - "Parameters." + "`comment` currently unused with ixmp4 when adding data to Parameters." ) parameter = self.index[s].optimization.parameters.get(name=name) # TODO there's got to be a better way for handling possible lists if isinstance(key, str): key = [key] + keys = parameter.column_names if parameter.column_names else parameter.indexsets data_to_add: dict[str, Union[list[float], list[str]]] = { - parameter.constrained_to_indexsets[i]: [key[i]] for i in range(len(key)) + keys[i]: [key[i]] for i in range(len(key)) } data_to_add["values"] = [value] data_to_add["units"] = [unit] + parameter.add(data=data_to_add) def item_set_elements( @@ -363,20 +431,14 @@ def _get_set_data( self, s: Scenario, name: str, filters: Optional[dict[str, list[Any]]] = None ) -> Union[pd.Series, pd.DataFrame]: # TODO handle filters - from ixmp4.core import IndexSet, Table - from ixmp4.core.exceptions import NotFound - item: Union[IndexSet, Table] - try: - repo = self._get_repo(s=s, type="indexset") - item = cast(IndexSet, repo.get(name=name)) - except NotFound: - repo = self._get_repo(s=s, type="set") - item = cast(Table, repo.get(name=name)) + item = self._get_indexset_or_table(s=s, name=name) - return ( - pd.DataFrame(item.data) if isinstance(item, Table) else pd.Series(item.data) - ) + if isinstance(item, Table): + columns = item.column_names if item.column_names else item.indexsets + return pd.DataFrame(item.data, columns=columns) + else: + return pd.Series(item.data) def item_get_elements( self, @@ -393,8 +455,9 @@ def item_get_elements( else: # TODO this can really only be Equation, Parameter, or Variable, so cast # as such? - _, item = self._get_item(s=s, name=name, type=type) - data = pd.DataFrame(item.data) + item = self._get_item(s=s, name=name, type=type) + # TODO if item.data can be empty, set columns explicitly as for table above + data = pd.DataFrame(item.data) # type: ignore[union-attr] if type == "par": data.rename(columns={"values": "value", "units": "unit"}, inplace=True) else: @@ -403,16 +466,133 @@ def item_get_elements( ) return data + def item_delete_elements( + self, + s: Scenario, + type: Literal["par", "set"], + name: str, + keys: Iterable[Sequence[str]], + ) -> None: + if type == "set": + item = self._get_indexset_or_table(s=s, name=name) + + if isinstance(item, IndexSet): + # NOTE We might have to expose IndexSet._data_type to cast correctly + data = pd.DataFrame(keys, columns=[item.name]) + item.remove(data=cast(list[str], data[item.name].to_list())) + else: + # TODO can we assume that keys follow same order as indexsets/columns? + columns = item.column_names if item.column_names else item.indexsets + data = pd.DataFrame(keys, columns=columns) + item.remove(data=data) + else: + parameter = cast(Parameter, self._get_item(s=s, name=name, type="par")) + columns = ( + parameter.column_names + if parameter.column_names + else parameter.indexsets + ) + data = pd.DataFrame(keys, columns=columns) + parameter.remove(data=data) + + def cat_set_elements( + self, + ms: Scenario, + name: str, + cat: str, + keys: Union[str, Sequence[str]], + is_unique: bool, + ) -> None: + """Add data to a category mapping. + + For the ixmp4.Table or IndexSet `name`, define a category as a new IndexSet + called 'type_`name`' (if it doesn't exist already) and add `cat` to it. Then, + define a new Table 'category_`name`' storing one column for `keys` and one for + 'categories'. + + Parameters + ---------- + name : str + Name of the category mapping Table. + cat : str + Name of the category within `name`. + keys : iterable of str or list of str + Keys to add to `cat`. + is_unique : bool + If :obj:`True`: + + - `keys` **must** contain only one key. + - The Backend **must** remove any existing member of `cat`, so that it has + only one element. + """ + from ixmp4.core.exceptions import NotFound + + # Assume for now that only IndexSets are requested to be mapped to cats + indexset = self.index[ms].optimization.indexsets.get(name=name) + + # Get or create the 'type_name' indexset and 'category_name' table + try: + category_indexset = self.index[ms].optimization.indexsets.get( + name=f"type_{name}" + ) + except NotFound: + category_indexset = self.index[ms].optimization.indexsets.create( + name=f"type_{name}" + ) + + try: + category_table = self.index[ms].optimization.tables.get( + name=f"category_{name}" + ) + except NotFound: + category_table = self.index[ms].optimization.tables.create( + name=f"category_{name}", + constrained_to_indexsets=[indexset.name, category_indexset.name], + ) + + # Convert for convenience + if isinstance(keys, str): + keys = [keys] + + # Ensure proper treatment when is_unique is True + if is_unique: + if len(keys) > 1: + raise ValueError("One can only add one element to a unique category!") + # Ensure data contains no data except that which we're going to add + # NOTE if category_table contains data linked to elements existing now, this + # will lead to DataValidationErrors when adding data to the table. Also, + # ixmp4 might safeguard against this when implementing remove() functions + if category_indexset.data: + # TODO remove data once ixmp4 implements that + log.warning("Can't remove data from ixmp4 objects (categories) yet!") + + # Add data to both objects + category_indexset.add(data=cat) + data = {indexset.name: keys, category_indexset.name: [cat] * len(keys)} + category_table.add(data=data) + + def cat_get_elements(self, ms: Scenario, name: str, cat: str) -> list[str]: + data = pd.DataFrame( + self.index[ms].optimization.tables.get(name=f"category_{name}").data + ) + # This assumes there are only two columns + columns = data.columns.to_list() + columns.remove(f"type_{name}") + return data[data[f"type_{name}"] == cat, columns[0]].astype(str).to_list() + + def cat_list(self, ms: Scenario, name: str) -> list[str]: + category_indexset = self.index[ms].optimization.indexsets.get(f"type_{name}") + return cast(list[str], category_indexset.data) + # Handle timeslices + # def set_timeslice(self, name: str, category: str, duration: float) -> None: + # return super().set_timeslice(name, category, duration) + # The below methods of base.Backend are not yet implemented def _ni(self, *args, **kwargs): raise NotImplementedError - cat_get_elements = _ni - cat_list = _ni - cat_set_elements = _ni - clone = _ni delete = _ni delete_geo = _ni delete_item = _ni @@ -422,7 +602,6 @@ def _ni(self, *args, **kwargs): get_geo = _ni get_meta = _ni get_timeslices = _ni - item_delete_elements = _ni last_update = _ni remove_meta = _ni set_data = _ni From 6f2d6e1f72e287480ac1024e5c5d08c4ead539b9 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 28 Jan 2025 15:55:50 +0100 Subject: [PATCH 25/72] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20Make=20s?= =?UTF-8?q?ure=20to=20handle=20defined=20variables?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ixmp/backend/ixmp4.py | 44 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/ixmp/backend/ixmp4.py b/ixmp/backend/ixmp4.py index cc5c03b16..871a0fbea 100644 --- a/ixmp/backend/ixmp4.py +++ b/ixmp/backend/ixmp4.py @@ -10,7 +10,8 @@ from ixmp.core.timeseries import TimeSeries if TYPE_CHECKING: - import ixmp4 + from ixmp4 import Platform as ixmp4_platform + from ixmp4.core import Run from ixmp4.core.optimization.equation import Equation from ixmp4.core.optimization.indexset import IndexSet from ixmp4.core.optimization.parameter import Parameter @@ -25,12 +26,12 @@ class IXMP4Backend(CachingBackend): """Backend using :mod:`ixmp4`.""" - _platform: "ixmp4.Platform" + _platform: "ixmp4_platform" _backend: "ixmp4_backend" # Mapping from ixmp.TimeSeries object to the underlying ixmp4.Run object (or # subclasses of either) - index: dict[TimeSeries, "ixmp4.Run"] = {} + index: dict[TimeSeries, "Run"] = {} def __init__(self, _backend: Optional["ixmp4_backend"] = None) -> None: from ixmp4 import Platform @@ -134,7 +135,7 @@ def get_nodes(self) -> list[tuple[str, None, str, str]]: # Modifying the Run object - def _index_and_set_attrs(self, run: "ixmp4.Run", ts: TimeSeries) -> None: + def _index_and_set_attrs(self, run: "Run", ts: TimeSeries) -> None: """Add *run* to index and update attributes of *ts*. Helper for init and get. @@ -281,18 +282,47 @@ def list_items(self, s: Scenario, type: Literal["set", "par", "equ"]) -> list[st repo = self._get_repo(s=s, type=type) return [item.name for item in repo.list()] - # TODO type hints - def _find_item(self, s: Scenario, name: str): + def _find_item( + self, s: Scenario, name: str + ) -> tuple[ + str, + Union[ + "Scalar", + "IndexSet", + "Table", + "Equation", + "Variable", + "Parameter", + ], + ]: # NOTE this currently assumes that `name` will only be present once among # Tables, Parameters, Equations, Variables. This is in line with the assumption # made in the Java backend, but ixmp4 enforces no such constraint. + _type: Optional[str] = None + _item: Optional[ + Union[ + "Scalar", + "IndexSet", + "Table", + "Equation", + "Variable", + "Parameter", + ] + ] = None for type in ("scalar", "indexset", "set", "par", "equ", "var"): repo = self._get_repo(s=s, type=type) item_list = repo.list(name=name) if ( len(item_list) == 1 ): # ixmp4 enforces names to be unique among individual item classes - return (type, item_list[0]) + _type = type + _item = item_list[0] + break + + if _item is None or _type is None: + raise KeyError(f"No item called {name} found on this Scenario!") + else: + return (_type, _item) def _get_item( self, From 080719e6693de5ccbed1cebee6cfa719eeda9a28 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Thu, 6 Feb 2025 10:14:54 +0100 Subject: [PATCH 26/72] Fix item_index return value and type hints --- ixmp/backend/ixmp4.py | 44 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/ixmp/backend/ixmp4.py b/ixmp/backend/ixmp4.py index 871a0fbea..9a67c66fc 100644 --- a/ixmp/backend/ixmp4.py +++ b/ixmp/backend/ixmp4.py @@ -1,25 +1,23 @@ import logging from collections.abc import Generator, Iterable, MutableMapping, Sequence -from typing import TYPE_CHECKING, Any, Literal, Optional, Union, cast +from typing import Any, Literal, Optional, Union, cast import pandas as pd +from ixmp4 import Platform as ixmp4_platform +from ixmp4.core import Run +from ixmp4.core.optimization.equation import Equation +from ixmp4.core.optimization.indexset import IndexSet +from ixmp4.core.optimization.parameter import Parameter +from ixmp4.core.optimization.scalar import Scalar +from ixmp4.core.optimization.table import Table +from ixmp4.core.optimization.variable import Variable +from ixmp4.data.backend.base import Backend as ixmp4_backend from ixmp.backend.base import CachingBackend from ixmp.core.platform import Platform from ixmp.core.scenario import Scenario from ixmp.core.timeseries import TimeSeries -if TYPE_CHECKING: - from ixmp4 import Platform as ixmp4_platform - from ixmp4.core import Run - from ixmp4.core.optimization.equation import Equation - from ixmp4.core.optimization.indexset import IndexSet - from ixmp4.core.optimization.parameter import Parameter - from ixmp4.core.optimization.scalar import Scalar - from ixmp4.core.optimization.table import Table - from ixmp4.core.optimization.variable import Variable - from ixmp4.data.backend.base import Backend as ixmp4_backend - log = logging.getLogger(__name__) @@ -360,21 +358,21 @@ def item_index( if ( isinstance(item, IndexSet) or isinstance(item, Scalar) - or ( - isinstance(item, Variable) - and sets_or_names == "names" - and item.column_names is None - ) + or (isinstance(item, Variable) and item.indexsets is None) ): return cast(list[str], []) else: - index_names = ( - item.column_names if sets_or_names == "names" else item.indexsets - ) - assert index_names, ( - f"Requested {sets_or_names}, but these are None for item {item.name}" + if sets_or_names == "names" and item.column_names is None: + log.warning( + f"Requested {sets_or_names}, but these are None for item " + f"{item.name}, falling back on (Index)Set names!" + ) + assert item.indexsets # Could only be None for Variables + return ( + item.column_names + if item.column_names and sets_or_names == "names" + else item.indexsets ) - return index_names def _add_data_to_set( self, s: Scenario, name: str, key: Union[str, list[str]], comment: Optional[str] From aeccf61e66e11db81222c73dc51c68cf5c02f5a9 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Wed, 12 Feb 2025 13:58:48 +0100 Subject: [PATCH 27/72] Improve type hints --- ixmp/backend/base.py | 70 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 59 insertions(+), 11 deletions(-) diff --git a/ixmp/backend/base.py b/ixmp/backend/base.py index 56b2a022f..78927cd33 100644 --- a/ixmp/backend/base.py +++ b/ixmp/backend/base.py @@ -9,6 +9,9 @@ import pandas as pd +# TODO Import this from typing when dropping Python 3.11 +from typing_extensions import TypedDict, Unpack + from ixmp.backend import ItemType from ixmp.backend.io import s_read_excel, s_write_excel, ts_read_file from ixmp.core.platform import Platform @@ -16,6 +19,35 @@ from ixmp.core.timeseries import TimeSeries +# These are based on existing calls within ixmp +class WriteFiltersKwargs(TypedDict, total=False): + scenario: Scenario | list[str] + model: list[str] + variable: list[str] + unit: list[str] + region: list[str] + default: bool + export_all_runs: bool + + +class WriteKwargs(TypedDict, total=False): + filters: WriteFiltersKwargs + record_version_packages: list[str] + + +class ReadKwargs(TypedDict, total=False): + filters: WriteFiltersKwargs + firstyear: Optional[Any] + lastyear: Optional[Any] + add_units: bool + init_items: bool + commit_steps: bool + check_solution: bool + comment: str + equ_list: list[str] + var_list: list[str] + + class Backend(ABC): """Abstract base class for backends.""" @@ -379,7 +411,9 @@ def get_units(self) -> list[str]: set_unit """ - def read_file(self, path: PathLike, item_type: ItemType, **kwargs) -> None: + def read_file( + self, path: PathLike, item_type: ItemType, **kwargs: Unpack[ReadKwargs] + ) -> None: """OPTIONAL: Read Platform, TimeSeries, or Scenario data from file. A backend **may** implement read_file for one or more combinations of the `path` @@ -419,16 +453,20 @@ def read_file(self, path: PathLike, item_type: ItemType, **kwargs) -> None: -------- write_file """ - s, filters = self._handle_rw_filters(kwargs.pop("filters", {})) + s, _ = self._handle_rw_filters(kwargs["filters"]) + _kwargs = {k: v for (k, v) in kwargs.items() if k != "filters"} + path = Path(path) if path.suffix in (".csv", ".xlsx") and item_type is ItemType.TS and s: - ts_read_file(s, path, **kwargs) + ts_read_file(s, path, **_kwargs) elif path.suffix == ".xlsx" and item_type is ItemType.MODEL and s: - s_read_excel(self, s, path, **kwargs) + s_read_excel(self, s, path, **_kwargs) else: raise NotImplementedError - def write_file(self, path: PathLike, item_type: ItemType, **kwargs) -> None: + def write_file( + self, path: PathLike, item_type: ItemType, **kwargs: Unpack[WriteKwargs] + ) -> None: """OPTIONAL: Write Platform, TimeSeries, or Scenario data to file. A backend **may** implement write_file for one or more combinations of the @@ -461,12 +499,18 @@ def write_file(self, path: PathLike, item_type: ItemType, **kwargs) -> None: """ # Use the "scenario" filter to retrieve the Scenario `s` to be written; reappend # any other filters - s, kwargs["filters"] = self._handle_rw_filters(kwargs.pop("filters", {})) + s, kwargs["filters"] = self._handle_rw_filters(kwargs["filters"]) xlsx_types = (ItemType.SET | ItemType.PAR, ItemType.MODEL) path = Path(path) if path.suffix == ".xlsx" and item_type in xlsx_types and s: - s_write_excel(self, s, path, item_type, **kwargs) + s_write_excel( + be=self, + s=s, + path=path, + item_type=item_type, + filters=kwargs["filters"], + ) else: # All other combinations of arguments raise NotImplementedError @@ -582,18 +626,20 @@ def preload(self, ts: TimeSeries) -> None: """OPTIONAL: Load `ts` data into memory.""" @staticmethod - def _handle_rw_filters(filters: dict) -> tuple[Optional[TimeSeries], dict]: + def _handle_rw_filters( + filters: WriteFiltersKwargs, + ) -> tuple[Optional[TimeSeries], WriteFiltersKwargs]: """Helper for :meth:`read_file` and :meth:`write_file`. The `filters` argument is unpacked if the 'scenarios' key is a single :class:`.TimeSeries` object. A 2-tuple is returned of the object (or :obj:`None`) and the remaining filters. """ - ts = None + ts: Optional[TimeSeries] = None filters = copy(filters) try: if isinstance(filters["scenario"], TimeSeries): - ts = filters.pop("scenario") + ts = filters["scenario"] except KeyError: pass # Don't modify filters at all @@ -861,7 +907,9 @@ def init_item( """ @abstractmethod - def delete_item(self, s: Scenario, type: str, name: str) -> None: + def delete_item( + self, s: Scenario, type: Literal["set", "par", "equ"], name: str + ) -> None: """Remove an item `name` of `type`. Parameters From 20fec5bdff3b3295fb39748731b84dd6eb7dbe47 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Wed, 12 Feb 2025 13:59:48 +0100 Subject: [PATCH 28/72] Introduce GAMS I/O for IXMP4Backend --- ixmp/backend/io.py | 259 ++++++++++++++++++++++++++++++++++++++++++ ixmp/backend/ixmp4.py | 205 ++++++++++++++++++++++++++++++--- 2 files changed, 446 insertions(+), 18 deletions(-) diff --git a/ixmp/backend/io.py b/ixmp/backend/io.py index 92b6e2c1a..ec342b1c4 100644 --- a/ixmp/backend/io.py +++ b/ixmp/backend/io.py @@ -1,8 +1,22 @@ import logging from collections import deque +from collections.abc import Iterable +from pathlib import Path +import gams.transfer as gt import pandas as pd +# from gams import GamsWorkspace +from ixmp4.core import Run +from ixmp4.core.optimization.equation import Equation +from ixmp4.core.optimization.indexset import IndexSet +from ixmp4.core.optimization.parameter import Parameter +from ixmp4.core.optimization.scalar import Scalar +from ixmp4.core.optimization.table import Table +from ixmp4.core.optimization.variable import Variable +from ixmp4.data.abstract.optimization.equation import Equation as AbstractEquation +from ixmp4.data.abstract.optimization.variable import Variable as AbstractVariable + from ixmp.util import as_str_list, maybe_check_out, maybe_commit from . import ItemType @@ -313,3 +327,248 @@ def parse_item_sheets(name): maybe_commit(s, commit_steps, f"Loaded {ix_type} {repr(name)} from {path}") maybe_commit(s, not commit_steps, f"Import from {path}") + + +def _add_scalars_to_container(container: gt.Container, scalars: list[Scalar]) -> None: + for scalar in scalars: + gt.Parameter( + container=container, + name=scalar.name, + records=scalar.value, + description=scalar.docs if scalar.docs else "", + ) + + +# TODO instead of checking len(indexset.data), make ixmp4 return None if it's empty? +def _add_indexsets_to_container( + container: gt.Container, indexsets: list[IndexSet] +) -> list[gt.Set]: + """Add Indexsets to Container and return them for dependent functions.""" + return [ + gt.Set( + container=container, + name=indexset.name, + records=indexset.data if len(indexset.data) else None, + description=indexset.docs + if indexset.docs + else "", # description is "optional", but must be str + ) + for indexset in indexsets + ] + + +def _add_tables_to_container( + container: gt.Container, tables: list[Table], indexsets: list[gt.Set] +) -> None: + for table in tables: + gt.Set( + container=container, + name=table.name, + domain=table.indexsets, + records=table.data if len(table.data.items()) else None, + description=table.docs if table.docs else "", + ) + + +def _add_parameters_to_container( + container: gt.Container, parameters: list[Parameter], indexsets: list[gt.Set] +) -> None: + for parameter in parameters: + records = parameter.data if len(parameter.data.items()) else None + if records: + records.pop("units") + + gt.Parameter( + container=container, + name=parameter.name, + domain=parameter.indexsets, + records=records, + description=parameter.docs if parameter.docs else "", + ) + + +def _add_variables_to_container( + container: gt.Container, variables: list[Variable], indexsets: list[gt.Set] +) -> None: + for variable in variables: + records = variable.data if len(variable.data.items()) else None + if records: + records.pop("levels") + records.pop("marginals") + + gt.Variable( + container=container, + name=variable.name, + domain=variable.indexsets, + records=records, + description=variable.docs if variable.docs else "", + ) + + +def _add_equations_to_container( + container: gt.Container, equations: list[Equation], indexsets: list[gt.Set] +) -> None: + for equation in equations: + records = equation.data if len(equation.data.items()) else None + if records: + records.pop("levels") + records.pop("marginals") + # The gams documentation confuses me: The docstring says `type` is required, the + # example says no. It seems to work fine like this, but if we do need a value, + # maybe we could guess based on + # https://github.com/iiasa/ixmp_source/blob/master/src/main/java/at/ac/iiasa/ixmp/objects/Scenario.java#L1926, + gt.Equation( + container=container, + name=equation.name, + domain=equation.indexsets, + records=records, + description=equation.docs if equation.docs else "", + ) + + +def _record_versions(container: gt.Container, packages: list[str]) -> None: + """Store Python package versions as set elements to be written to GDX. + + The values are stored in a 2-dimensional set named ``ixmp_version``, where the + first element is the package name, and the second is its version according to + :func:`importlib.metadata.version`). If the package is not installed, the + string "(not installed)" is stored. + """ + from importlib.metadata import PackageNotFoundError, version + + name = "ixmp_version" + + # Each tuple consists of (package_name, package_version) + versions: list[tuple[str, str]] = [] + + # Handle each identified package + for package in packages: + try: + # Retrieve the version; replace characters not supported by GAMS + package_version = version(package).replace(".", "-") + except PackageNotFoundError: + package_version = "(not installed)" # Not installed + + versions.append((package, package_version)) + + # Add Set to the container + container.addSet(name=name, domain=["*", "*"], records=versions) + + +def write_run_to_gdx( + run: Run, + file_name: Path, + record_version_packages: list[str], + include_variables_and_equations: bool = False, +) -> None: + """Writes scenario data from the Run to a GAMS container.""" + + # TODO How to deal with [*] Set? + + container = gt.Container() + _add_scalars_to_container( + container=container, scalars=run.optimization.scalars.list() + ) + indexsets = _add_indexsets_to_container( + container=container, indexsets=run.optimization.indexsets.list() + ) + _add_tables_to_container( + container=container, + tables=run.optimization.tables.list(), + indexsets=indexsets, + ) + _add_parameters_to_container( + container=container, + parameters=run.optimization.parameters.list(), + indexsets=indexsets, + ) + + if include_variables_and_equations: + _add_variables_to_container( + container=container, + variables=run.optimization.variables.list(), + indexsets=indexsets, + ) + _add_equations_to_container( + container=container, + equations=run.optimization.equations.list(), + indexsets=indexsets, + ) + + _record_versions(container=container, packages=record_version_packages) + + container.write(write_to=file_name) + + +def _read_variables_to_run( + container: gt.Container, run: Run, variables: Iterable[AbstractVariable] +) -> None: + for variable in variables: + # Prepare columns to select from container.data + # DF also includes lower, upper, scale + columns = ["level", "marginal"] + variable_columns = variable.column_names or variable.indexsets + if variable_columns: + columns += variable_columns + + run.backend.optimization.variables.add_data( + variable_id=variable.id, + data=( + container.data[variable.name] + .records[columns] + .rename(columns={"level": "levels", "marginal": "marginals"}) + ), + ) + + +def _read_equations_to_run( + container: gt.Container, run: Run, equations: Iterable[AbstractEquation] +) -> None: + for equation in equations: + # Prepare columns to select from container.data + # DF also includes lower, upper, scale + columns = ["level", "marginal"] + equation_columns = equation.column_names or equation.indexsets + if equation_columns: + columns += equation_columns + + run.backend.optimization.equations.add_data( + equation_id=equation.id, + data=( + container.data[equation.name] + .records[columns] + .rename(columns={"level": "levels", "marginal": "marginals"}) + ), + ) + + +def read_gdx_to_run( + run: Run, + result_file: Path, + equ_list: list[str], + var_list: list[str], + comment: str, + check_solution: bool, +) -> None: + if comment: + log.warning(f"Ignoring comment {comment} for now; unused by ixmp4!") + if check_solution: + log.warning( + f"Ignoring check_solution={check_solution} for now; unused by ixmp4!" + ) + + variables = ( + run.backend.optimization.variables.list(name__in=var_list) + if len(var_list) + else run.backend.optimization.variables.list() + ) + equations = ( + run.backend.optimization.equations.list(name__in=equ_list) + if len(equ_list) + else run.backend.optimization.equations.list() + ) + + container = gt.Container(load_from=result_file) + + _read_variables_to_run(container=container, run=run, variables=variables) + _read_equations_to_run(container=container, run=run, equations=equations) diff --git a/ixmp/backend/ixmp4.py b/ixmp/backend/ixmp4.py index 9a67c66fc..733891968 100644 --- a/ixmp/backend/ixmp4.py +++ b/ixmp/backend/ixmp4.py @@ -1,5 +1,7 @@ import logging from collections.abc import Generator, Iterable, MutableMapping, Sequence +from os import PathLike +from pathlib import Path from typing import Any, Literal, Optional, Union, cast import pandas as pd @@ -13,7 +15,12 @@ from ixmp4.core.optimization.variable import Variable from ixmp4.data.backend.base import Backend as ixmp4_backend -from ixmp.backend.base import CachingBackend +# TODO Import this from typing when dropping Python 3.11 +from typing_extensions import Unpack + +from ixmp.backend import ItemType +from ixmp.backend.base import CachingBackend, ReadKwargs, WriteKwargs +from ixmp.backend.io import read_gdx_to_run, write_run_to_gdx from ixmp.core.platform import Platform from ixmp.core.scenario import Scenario from ixmp.core.timeseries import TimeSeries @@ -263,6 +270,10 @@ def init_item( if type == "set" and len(idx_sets) == 0: repo = self._get_repo(s=s, type="indexset") repo.create(name=name) + elif type == "par" and len(idx_sets) == 0: + # NOTE ixmp4 requires scalars to get value and unit upon creation, so the + # shim does that when `item_set_elements()` targets a scalar + return None else: repo = self._get_repo(s=s, type=type) repo.create( @@ -281,9 +292,19 @@ def list_items(self, s: Scenario, type: Literal["set", "par", "equ"]) -> list[st return [item.name for item in repo.list()] def _find_item( - self, s: Scenario, name: str + self, + s: Scenario, + name: str, + types: tuple[Literal["scalar", "indexset", "set", "par", "equ", "var"], ...] = ( + "scalar", + "indexset", + "set", + "par", + "equ", + "var", + ), ) -> tuple[ - str, + Literal["scalar", "indexset", "set", "par", "equ", "var"], Union[ "Scalar", "IndexSet", @@ -296,7 +317,9 @@ def _find_item( # NOTE this currently assumes that `name` will only be present once among # Tables, Parameters, Equations, Variables. This is in line with the assumption # made in the Java backend, but ixmp4 enforces no such constraint. - _type: Optional[str] = None + _type: Optional[Literal["scalar", "indexset", "set", "par", "equ", "var"]] = ( + None + ) _item: Optional[ Union[ "Scalar", @@ -307,7 +330,7 @@ def _find_item( "Parameter", ] ] = None - for type in ("scalar", "indexset", "set", "par", "equ", "var"): + for type in types: repo = self._get_repo(s=s, type=type) item_list = repo.list(name=name) if ( @@ -340,7 +363,6 @@ def _get_item( def _get_indexset_or_table( self, s: Scenario, name: str ) -> Union["IndexSet", "Table"]: - from ixmp4.core import IndexSet, Table from ixmp4.core.exceptions import NotFound try: @@ -388,7 +410,7 @@ def _add_data_to_set( table = self.index[s].optimization.tables.get(name=name) # TODO should we enforce in ixmp4 that when constrained_to_indexsets # contains duplicate values, column_names must be provided? - keys = table.column_names if table.column_names else table.indexsets + keys = table.column_names or table.indexsets data_to_add = {keys[i]: [key[i]] for i in range(len(key))} table.add(data=data_to_add) @@ -423,7 +445,8 @@ def _add_data_to_parameter( # TODO there's got to be a better way for handling possible lists if isinstance(key, str): key = [key] - keys = parameter.column_names if parameter.column_names else parameter.indexsets + + keys = parameter.column_names or parameter.indexsets data_to_add: dict[str, Union[list[float], list[str]]] = { keys[i]: [key[i]] for i in range(len(key)) } @@ -463,7 +486,7 @@ def _get_set_data( item = self._get_indexset_or_table(s=s, name=name) if isinstance(item, Table): - columns = item.column_names if item.column_names else item.indexsets + columns = item.column_names or item.indexsets return pd.DataFrame(item.data, columns=columns) else: return pd.Series(item.data) @@ -510,19 +533,28 @@ def item_delete_elements( item.remove(data=cast(list[str], data[item.name].to_list())) else: # TODO can we assume that keys follow same order as indexsets/columns? - columns = item.column_names if item.column_names else item.indexsets + columns = item.column_names or item.indexsets data = pd.DataFrame(keys, columns=columns) item.remove(data=data) else: parameter = cast(Parameter, self._get_item(s=s, name=name, type="par")) - columns = ( - parameter.column_names - if parameter.column_names - else parameter.indexsets - ) + columns = parameter.column_names or parameter.indexsets data = pd.DataFrame(keys, columns=columns) parameter.remove(data=data) + def delete_item( + self, + s: Scenario, + type: Literal["set", "par", "equ", "indexset"], + name: str, + ) -> None: + if type == "set": + # type will always be either "indexset" or "set" + type, _ = self._find_item(s=s, name=name, types=("indexset", "set")) # type: ignore[assignment] + + repo = self._get_repo(s=s, type=type) + repo.delete(item=name) + def cat_set_elements( self, ms: Scenario, @@ -591,8 +623,9 @@ def cat_set_elements( # will lead to DataValidationErrors when adding data to the table. Also, # ixmp4 might safeguard against this when implementing remove() functions if category_indexset.data: - # TODO remove data once ixmp4 implements that - log.warning("Can't remove data from ixmp4 objects (categories) yet!") + # Remove all existing data so that only the single provided element will + # be stored in the indexset + category_indexset.remove(category_indexset.data) # Add data to both objects category_indexset.add(data=cat) @@ -617,13 +650,149 @@ def cat_list(self, ms: Scenario, name: str) -> list[str]: # def set_timeslice(self, name: str, category: str, duration: float) -> None: # return super().set_timeslice(name, category, duration) + # Handle I/O + + def write_file( + self, path: PathLike, item_type: ItemType, **kwargs: Unpack[WriteKwargs] + ) -> None: + """Write Platform, TimeSeries, or Scenario data to file. + + IXMP4Backend supports writing to: + + - ``path='*.gdx', item_type=ItemType.SET | ItemType.PAR``. + + IXMP4Backend does not yet support: + - ``path='*.csv', item_type=TS``. The `default` keyword argument is + **required**. + + Other parameters + ---------------- + filters : dict of dict of str + Restrict items written. The following filters may be used: + + - model : list of str + - scenario : list of str | Scenario + - variable : list of str + - region: list of str + - unit: list of str + - default : bool + If :obj:`True`, only data from TimeSeries versions with + :meth:`.TimeSeries.set_as_default` are written. + - export_all_runs: bool + Whether to export all existing model+scenario run combinations. + + See also + -------- + .Backend.write_file + """ + try: + # Call the default implementation, e.g. for .xlsx + super().write_file(path, item_type, **kwargs) + except NotImplementedError: + pass + else: + return + + # NOTE Would like to use proper paths in signature already, but comply with base + # class for now + _path = Path(path) + + ts, filters = self._handle_rw_filters(kwargs["filters"]) + if _path.suffix == ".gdx" and item_type is ItemType.SET | ItemType.PAR: + # NOTE if we keep the TypedDicts and can't pop items, we might have to + # adjust more checks like this. Alternatively, use explicit keyword + # parameters for functions or convert kwargs to dicts in here to enable pop. + if len(filters) > 1: # pragma: no cover + raise NotImplementedError("write to GDX with filters") + elif not isinstance(ts, Scenario): # pragma: no cover + raise ValueError("write to GDX requires a Scenario object") + + write_run_to_gdx( + run=self.index[ts], + file_name=_path, + record_version_packages=kwargs["record_version_packages"], + ) + else: + raise NotImplementedError + + def read_file( + self, path: PathLike, item_type: ItemType, **kwargs: Unpack[ReadKwargs] + ) -> None: + """Read Platform, TimeSeries, or Scenario data from file. + + IXMP4Backend supports reading from: + + - ``path='*.gdx', item_type=ItemType.MODEL``. The keyword arguments + `check_solution`, `comment`, `equ_list`, and `var_list` are **required**. + + Other parameters + ---------------- + check_solution : bool + If True, raise an exception if the GAMS solver did not reach optimality. + (Only for MESSAGE-scheme Scenarios.) + comment : str + Comment added to Scenario when importing the solution. + equ_list : list of str + Equations to be imported. + var_list : list of str + Variables to be imported. + filters : dict of dict of str + Restrict items read. + + See also + -------- + .Backend.read_file + """ + try: + # Call the default implementation, e.g. for .xlsx + super().read_file(path, item_type, **kwargs) + except NotImplementedError: + pass + else: + return + + _path = Path(path) + + # TODO handle case when filters is not present + ts, _ = self._handle_rw_filters(kwargs["filters"]) + _kwargs = {k: v for (k, v) in kwargs.items() if k != "filters"} + + if _path.suffix == ".gdx" and item_type is ItemType.MODEL: + kw = {"check_solution", "comment", "equ_list", "var_list"} + + if not isinstance(ts, Scenario): # pragma: no cover + raise ValueError("read from GDX requires a Scenario object") + elif set(_kwargs.keys()) != kw: + raise ValueError( + f"keyword arguments {_kwargs.keys()} do not match required {kw}" + ) + + check_solution = bool(_kwargs.pop("check_solution")) + comment = str(_kwargs.pop("comment")) + equ_list = cast(list[str], _kwargs.pop("equ_list")) + var_list = cast(list[str], _kwargs.pop("var_list")) + + if len(_kwargs): + raise ValueError(f"extra keyword arguments {_kwargs}") + + read_gdx_to_run( + run=self.index[ts], + result_file=_path, + equ_list=equ_list, + var_list=var_list, + comment=comment, + check_solution=check_solution, + ) + + else: + raise NotImplementedError(path, item_type) + # The below methods of base.Backend are not yet implemented def _ni(self, *args, **kwargs): raise NotImplementedError delete = _ni delete_geo = _ni - delete_item = _ni delete_meta = _ni get_data = _ni get_doc = _ni From c9d41ea978b67d4ffc80560ebf8de3c1a84cba6d Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Wed, 12 Feb 2025 14:00:47 +0100 Subject: [PATCH 29/72] Adjust existing logic to ixmp4 gams i/o --- ixmp/backend/jdbc.py | 8 ++++--- ixmp/model/gams.py | 52 ++++++++++++++++++++++++++++++++++++-------- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/ixmp/backend/jdbc.py b/ixmp/backend/jdbc.py index ac8c864dc..659d7205f 100644 --- a/ixmp/backend/jdbc.py +++ b/ixmp/backend/jdbc.py @@ -10,7 +10,7 @@ from functools import lru_cache from pathlib import Path, PurePosixPath from types import SimpleNamespace -from typing import Optional +from typing import Optional, cast from weakref import WeakKeyDictionary import jpype @@ -605,7 +605,7 @@ def write_file(self, path, item_type: ItemType, **kwargs): ts, filters = self._handle_rw_filters(kwargs.pop("filters", {})) if path.suffix == ".gdx" and item_type is ItemType.SET | ItemType.PAR: - if len(filters): # pragma: no cover + if len(filters) > 1: # pragma: no cover raise NotImplementedError("write to GDX with filters") elif not isinstance(ts, Scenario): # pragma: no cover raise ValueError("write to GDX requires a Scenario object") @@ -614,7 +614,9 @@ def write_file(self, path, item_type: ItemType, **kwargs): self.jindex[ts].toGDX(str(path.parent), path.name, False) elif path.suffix == ".csv" and item_type is ItemType.TS: models = set(filters.pop("model")) - scenarios = set(filters.pop("scenario")) + # NOTE this is what we get for not differentiating e.g. scenario vs + # scenarios in filters... + scenarios = set(cast(list[str], filters.pop("scenario"))) variables = filters.pop("variable") units = filters.pop("unit") regions = filters.pop("region") diff --git a/ixmp/model/gams.py b/ixmp/model/gams.py index 819516319..a005deff8 100644 --- a/ixmp/model/gams.py +++ b/ixmp/model/gams.py @@ -11,6 +11,8 @@ from typing import Any, Optional from ixmp.backend import ItemType +from ixmp.backend.jdbc import JDBCBackend +from ixmp.core.scenario import Scenario from ixmp.model.base import Model, ModelError from ixmp.util import as_str_list @@ -180,6 +182,21 @@ class GAMSModel(Model): See :meth:`record_versions`. """ # noqa: E501 + # Make attributes known to self + model_file: str + case: str + in_file: os.PathLike + out_file: os.PathLike + solve_args: list[str] + gams_args: list[str] + check_solution: bool + comment: Optional[str] + equ_list: Optional[list[str]] + var_list: Optional[list[str]] + quiet: bool + use_temp_dir: bool + record_version_packages: list[str] + #: Model name. name = "default" @@ -308,7 +325,7 @@ def __del__(self): # This appears to still fail on Windows. self.remove_temp_dir("at GAMSModel teardown") - def run(self, scenario): + def run(self, scenario: Scenario) -> None: """Execute the model. Among other steps: @@ -324,8 +341,18 @@ def run(self, scenario): # Store the scenario so its attributes can be referenced by format() self.scenario = scenario - # Record versions of packages listed in `record_version_packages` - self.record_versions() + # NOTE workaround delattr(self, "scenario"); differentiate backend types without + # moving that call + # If we get more backend types, we'll need to adjust this + backend_type = ( + "jdbc" + if isinstance(self.scenario.platform._backend, JDBCBackend) + else "ixmp4" + ) + + if backend_type == "jdbc": + # Record versions of packages listed in `record_version_packages` + self.record_versions() # Format or retrieve the model file option model_file = Path(self.format_option("model_file")) @@ -348,13 +375,17 @@ def run(self, scenario): if os.name == "nt": # Windows: join the commands to a single string - command = " ".join(command) + command = " ".join(command) # type: ignore[assignment] # Remove stored reference to the Scenario to allow it to be GC'd later delattr(self, "scenario") # Common argument for write_file and read_file - s_arg = dict(filters=dict(scenario=scenario)) + s_arg: dict[str, object] = dict(filters=dict(scenario=scenario)) + + # Instruct ixmp4 to record package versions + if backend_type == "ixmp4": + s_arg["record_version_packages"] = self.record_version_packages try: # Write model data to file @@ -373,10 +404,13 @@ def run(self, scenario): "JDBCBackend" ) else: - # Remove ixmp_version set entirely - with scenario.transact(): - scenario.remove_set("ixmp_version") - + if backend_type == "jdbc": + # Remove ixmp_version set entirely + with scenario.transact(): + scenario.remove_set("ixmp_version") + else: # backend_type == "ixmp4" + # Clean up s_arg for use in read_file() + s_arg.pop("record_version_packages") try: # Invoke GAMS run(command, shell=os.name == "nt", cwd=self.cwd, check=True) From dc493512b3b6d00e21c57399ab3368e761fe8935 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 12 Feb 2025 16:16:08 +0100 Subject: [PATCH 30/72] Merge .backend.io._add_*_to_container() functions --- ixmp/backend/io.py | 209 +++++++++++++++++++-------------------------- 1 file changed, 88 insertions(+), 121 deletions(-) diff --git a/ixmp/backend/io.py b/ixmp/backend/io.py index ec342b1c4..8e99b1a53 100644 --- a/ixmp/backend/io.py +++ b/ixmp/backend/io.py @@ -2,18 +2,15 @@ from collections import deque from collections.abc import Iterable from pathlib import Path +from typing import TYPE_CHECKING, Optional, Union import gams.transfer as gt import pandas as pd # from gams import GamsWorkspace from ixmp4.core import Run -from ixmp4.core.optimization.equation import Equation from ixmp4.core.optimization.indexset import IndexSet -from ixmp4.core.optimization.parameter import Parameter from ixmp4.core.optimization.scalar import Scalar -from ixmp4.core.optimization.table import Table -from ixmp4.core.optimization.variable import Variable from ixmp4.data.abstract.optimization.equation import Equation as AbstractEquation from ixmp4.data.abstract.optimization.variable import Variable as AbstractVariable @@ -21,6 +18,22 @@ from . import ItemType +if TYPE_CHECKING: + from typing import TypeVar + + from ixmp4.core.optimization.equation import Equation + + # from ixmp4.core.optimization.indexset import IndexSet + from ixmp4.core.optimization.parameter import Parameter + + # from ixmp4.core.optimization.scalar import Scalar + from ixmp4.core.optimization.table import Table + from ixmp4.core.optimization.variable import Variable + + # Type variable that can be any one of these 6 types, but not a union of 2+ of them + Item4 = TypeVar("Item4", Equation, IndexSet, Parameter, Scalar, Table, Variable) + + log = logging.getLogger(__name__) #: Maximum number of rows supported by the Excel file format. See @@ -329,100 +342,68 @@ def parse_item_sheets(name): maybe_commit(s, not commit_steps, f"Import from {path}") -def _add_scalars_to_container(container: gt.Container, scalars: list[Scalar]) -> None: - for scalar in scalars: - gt.Parameter( - container=container, - name=scalar.name, - records=scalar.value, - description=scalar.docs if scalar.docs else "", - ) - - -# TODO instead of checking len(indexset.data), make ixmp4 return None if it's empty? -def _add_indexsets_to_container( - container: gt.Container, indexsets: list[IndexSet] -) -> list[gt.Set]: - """Add Indexsets to Container and return them for dependent functions.""" - return [ - gt.Set( - container=container, - name=indexset.name, - records=indexset.data if len(indexset.data) else None, - description=indexset.docs - if indexset.docs - else "", # description is "optional", but must be str - ) - for indexset in indexsets - ] - - -def _add_tables_to_container( - container: gt.Container, tables: list[Table], indexsets: list[gt.Set] -) -> None: - for table in tables: - gt.Set( - container=container, - name=table.name, - domain=table.indexsets, - records=table.data if len(table.data.items()) else None, - description=table.docs if table.docs else "", - ) - - -def _add_parameters_to_container( - container: gt.Container, parameters: list[Parameter], indexsets: list[gt.Set] -) -> None: - for parameter in parameters: - records = parameter.data if len(parameter.data.items()) else None - if records: - records.pop("units") - - gt.Parameter( - container=container, - name=parameter.name, - domain=parameter.indexsets, - records=records, - description=parameter.docs if parameter.docs else "", - ) - - -def _add_variables_to_container( - container: gt.Container, variables: list[Variable], indexsets: list[gt.Set] -) -> None: - for variable in variables: - records = variable.data if len(variable.data.items()) else None - if records: - records.pop("levels") - records.pop("marginals") - - gt.Variable( - container=container, - name=variable.name, - domain=variable.indexsets, - records=records, - description=variable.docs if variable.docs else "", - ) - - -def _add_equations_to_container( - container: gt.Container, equations: list[Equation], indexsets: list[gt.Set] -) -> None: - for equation in equations: - records = equation.data if len(equation.data.items()) else None - if records: - records.pop("levels") - records.pop("marginals") +def _add_to_container(container: gt.Container, items: list[Item4]) -> None: + """Add ixmp4 data `items` to `container`.""" + if not items: + return # Nothing to be done + + # Identify the ixmp4 class name and corresponding gams.transfer class + kind = type(items[0]).__name__ + klass = { + "IndexSet": gt.Set, + "Parameter": gt.Parameter, + "Scalar": gt.Parameter, + "Table": gt.Set, + "Variable": gt.Variable, + "Equation": gt.Equation, + }[kind] + + def domain(item: Item4) -> Optional[list[str]]: + if isinstance(item, (IndexSet, Scalar)): + return None + else: + return item.indexsets + + def records( + item: Item4, + ) -> Union[ + float, + Union[list[float], list[int], list[str]], + dict[str, Union[list[float], list[int], list[str]]], + None, + ]: + if isinstance(item, Scalar): + return item.value + elif not len(item.data): + return None + + if isinstance(item, IndexSet): + return item.data + + result = item.data + + # Pop items not to be stored + for name in { + "Equation": ["levels", "marginals"], + "Parameter": ["units"], + "Variable": ["levels", "marginals"], + }[kind]: + result.pop(name) + + return result + + for item in items: # The gams documentation confuses me: The docstring says `type` is required, the # example says no. It seems to work fine like this, but if we do need a value, # maybe we could guess based on # https://github.com/iiasa/ixmp_source/blob/master/src/main/java/at/ac/iiasa/ixmp/objects/Scenario.java#L1926, - gt.Equation( + klass( container=container, - name=equation.name, - domain=equation.indexsets, - records=records, - description=equation.docs if equation.docs else "", + name=item.name, + domain=domain(item), + records=records(item), + # Optional, but must be str + description=item.docs if item.docs else "", ) @@ -461,39 +442,25 @@ def write_run_to_gdx( record_version_packages: list[str], include_variables_and_equations: bool = False, ) -> None: - """Writes scenario data from the Run to a GAMS container.""" + """Write scenario data from the `run` to a GDX file via GAMS container.""" # TODO How to deal with [*] Set? container = gt.Container() - _add_scalars_to_container( - container=container, scalars=run.optimization.scalars.list() - ) - indexsets = _add_indexsets_to_container( - container=container, indexsets=run.optimization.indexsets.list() - ) - _add_tables_to_container( - container=container, - tables=run.optimization.tables.list(), - indexsets=indexsets, - ) - _add_parameters_to_container( - container=container, - parameters=run.optimization.parameters.list(), - indexsets=indexsets, - ) - if include_variables_and_equations: - _add_variables_to_container( - container=container, - variables=run.optimization.variables.list(), - indexsets=indexsets, - ) - _add_equations_to_container( - container=container, - equations=run.optimization.equations.list(), - indexsets=indexsets, - ) + repository = [ + run.optimization.indexsets, + run.optimization.scalars, + run.optimization.tables, + run.optimization.parameters, + run.optimization.variables, + run.optimization.equations, + ] + idx = slice(None) if include_variables_and_equations else slice(-2) + for r in repository[idx]: + # FIXME Type of 'r' is inferred as BaseFacade, the narrowest common supertype of + # IndexSetRepository etc.; BaseFacade.list() is not defined. + _add_to_container(container, r.list()) # type: ignore [attr-defined] _record_versions(container=container, packages=record_version_packages) From e210bea93d6548f7697c2392a3d7348dfbd94ec3 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Mon, 3 Mar 2025 12:27:40 +0100 Subject: [PATCH 31/72] Move TypedDict-Kwargs to dedicated file --- ixmp/backend/base.py | 32 ++------------------------------ ixmp/core/platform.py | 19 ++++++++++--------- ixmp/util/ixmp4.py | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 39 deletions(-) create mode 100644 ixmp/util/ixmp4.py diff --git a/ixmp/backend/base.py b/ixmp/backend/base.py index 78927cd33..276631b19 100644 --- a/ixmp/backend/base.py +++ b/ixmp/backend/base.py @@ -10,42 +10,14 @@ import pandas as pd # TODO Import this from typing when dropping Python 3.11 -from typing_extensions import TypedDict, Unpack +from typing_extensions import Unpack from ixmp.backend import ItemType from ixmp.backend.io import s_read_excel, s_write_excel, ts_read_file from ixmp.core.platform import Platform from ixmp.core.scenario import Scenario from ixmp.core.timeseries import TimeSeries - - -# These are based on existing calls within ixmp -class WriteFiltersKwargs(TypedDict, total=False): - scenario: Scenario | list[str] - model: list[str] - variable: list[str] - unit: list[str] - region: list[str] - default: bool - export_all_runs: bool - - -class WriteKwargs(TypedDict, total=False): - filters: WriteFiltersKwargs - record_version_packages: list[str] - - -class ReadKwargs(TypedDict, total=False): - filters: WriteFiltersKwargs - firstyear: Optional[Any] - lastyear: Optional[Any] - add_units: bool - init_items: bool - commit_steps: bool - check_solution: bool - comment: str - equ_list: list[str] - var_list: list[str] +from ixmp.util.ixmp4 import ReadKwargs, WriteFiltersKwargs, WriteKwargs class Backend(ABC): diff --git a/ixmp/core/platform.py b/ixmp/core/platform.py index 7107773e0..4fe07ddce 100644 --- a/ixmp/core/platform.py +++ b/ixmp/core/platform.py @@ -9,6 +9,7 @@ from ixmp._config import config from ixmp.backend import BACKENDS, FIELDS, ItemType from ixmp.util import as_str_list +from ixmp.util.ixmp4 import WriteFiltersKwargs if TYPE_CHECKING: from ixmp.backend.base import Backend @@ -236,15 +237,15 @@ def export_timeseries_data( "Invalid arguments: export_all_runs cannot be used when providing a " "model or scenario." ) - filters = { - "model": as_str_list(model), - "scenario": as_str_list(scenario), - "variable": as_str_list(variable), - "unit": as_str_list(unit), - "region": as_str_list(region), - "default": default, - "export_all_runs": export_all_runs, - } + filters = WriteFiltersKwargs( + scenario=as_str_list(scenario), + model=as_str_list(model), + variable=as_str_list(variable), + unit=as_str_list(unit), + region=as_str_list(region), + default=default, + export_all_runs=export_all_runs, + ) self._backend.write_file(path, ItemType.TS, filters=filters) diff --git a/ixmp/util/ixmp4.py b/ixmp/util/ixmp4.py new file mode 100644 index 000000000..af4f448c3 --- /dev/null +++ b/ixmp/util/ixmp4.py @@ -0,0 +1,36 @@ +# TODO Import this from typing when dropping Python 3.11 +from typing import TYPE_CHECKING, Any, Optional + +from typing_extensions import TypedDict + +if TYPE_CHECKING: + from ixmp.core.scenario import Scenario + + +# These are based on existing calls within ixmp +class WriteFiltersKwargs(TypedDict, total=False): + scenario: "Scenario | list[str]" + model: list[str] + variable: list[str] + unit: list[str] + region: list[str] + default: bool + export_all_runs: bool + + +class WriteKwargs(TypedDict, total=False): + filters: WriteFiltersKwargs + record_version_packages: list[str] + + +class ReadKwargs(TypedDict, total=False): + filters: WriteFiltersKwargs + firstyear: Optional[Any] + lastyear: Optional[Any] + add_units: bool + init_items: bool + commit_steps: bool + check_solution: bool + comment: str + equ_list: list[str] + var_list: list[str] From f949537656e30b0fb4b5eb21bc3f34ca2eaa518a Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Mon, 3 Mar 2025 12:30:03 +0100 Subject: [PATCH 32/72] Fix imports for TypeVar/type hints --- ixmp/backend/io.py | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/ixmp/backend/io.py b/ixmp/backend/io.py index 8e99b1a53..1f7504a4a 100644 --- a/ixmp/backend/io.py +++ b/ixmp/backend/io.py @@ -2,15 +2,19 @@ from collections import deque from collections.abc import Iterable from pathlib import Path -from typing import TYPE_CHECKING, Optional, Union +from typing import Optional, TypeVar, Union import gams.transfer as gt import pandas as pd # from gams import GamsWorkspace from ixmp4.core import Run +from ixmp4.core.optimization.equation import Equation from ixmp4.core.optimization.indexset import IndexSet +from ixmp4.core.optimization.parameter import Parameter from ixmp4.core.optimization.scalar import Scalar +from ixmp4.core.optimization.table import Table +from ixmp4.core.optimization.variable import Variable from ixmp4.data.abstract.optimization.equation import Equation as AbstractEquation from ixmp4.data.abstract.optimization.variable import Variable as AbstractVariable @@ -18,20 +22,8 @@ from . import ItemType -if TYPE_CHECKING: - from typing import TypeVar - - from ixmp4.core.optimization.equation import Equation - - # from ixmp4.core.optimization.indexset import IndexSet - from ixmp4.core.optimization.parameter import Parameter - - # from ixmp4.core.optimization.scalar import Scalar - from ixmp4.core.optimization.table import Table - from ixmp4.core.optimization.variable import Variable - - # Type variable that can be any one of these 6 types, but not a union of 2+ of them - Item4 = TypeVar("Item4", Equation, IndexSet, Parameter, Scalar, Table, Variable) +# Type variable that can be any one of these 6 types, but not a union of 2+ of them +Item4 = TypeVar("Item4", Equation, IndexSet, Parameter, Scalar, Table, Variable) log = logging.getLogger(__name__) @@ -393,9 +385,9 @@ def records( return result for item in items: - # The gams documentation confuses me: The docstring says `type` is required, the - # example says no. It seems to work fine like this, but if we do need a value, - # maybe we could guess based on + # The gams documentation confuses me: The docstring says `type` is required for + # Equations, the example says no. It seems to work fine like this, but if we do + # need a value, maybe we could guess based on # https://github.com/iiasa/ixmp_source/blob/master/src/main/java/at/ac/iiasa/ixmp/objects/Scenario.java#L1926, klass( container=container, From eb8519eb9dd36b2e2524c03f9fa949d8d05786bb Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Mon, 3 Mar 2025 12:30:23 +0100 Subject: [PATCH 33/72] Add more backend functions: * Add get_scenarios() * Add get_data() * Add set_data() * Ensure correct versions are used when cloning --- ixmp/backend/ixmp4.py | 133 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 124 insertions(+), 9 deletions(-) diff --git a/ixmp/backend/ixmp4.py b/ixmp/backend/ixmp4.py index 733891968..cbf32c83d 100644 --- a/ixmp/backend/ixmp4.py +++ b/ixmp/backend/ixmp4.py @@ -5,6 +5,7 @@ from typing import Any, Literal, Optional, Union, cast import pandas as pd +from ixmp4 import DataPoint from ixmp4 import Platform as ixmp4_platform from ixmp4.core import Run from ixmp4.core.optimization.equation import Equation @@ -104,8 +105,36 @@ def get_model_names(self) -> Generator[str, None, None]: for model in self._platform.models.list(): yield model.name - def get_scenarios(self, default, model, scenario): - return self._platform.runs.list() + def get_scenarios( + self, default: bool, model: Optional[str], scenario: Optional[str] + ): + runs = self._platform.runs.list( + default_only=default, + model={"name": model} if model else None, + scenario={"name": scenario} if scenario else None, + ) + + for run in runs: + yield [ + run.model.name, + run.scenario.name, + # TODO What are we going to use for scheme in ixmp4? + "IXMP4Run", + run.is_default(), + # TODO Change this from being hardcoded + False, + # TODO Expose the creation, update and lock info in ixmp4 + # (if we get lock info) + "Some user", + "Some date", + "Some user", + "Some date", + "Some user", + "Some date", + # TODO Should Runs get .docs? + "Some docs", + run.version, + ] def set_unit(self, name: str, comment: str) -> None: self._platform.units.create(name=name).docs = comment @@ -184,12 +213,16 @@ def clone( # TODO Is this enough? ixmp4 doesn't support cloning to a different platform at # the moment, but maybe we can imitate this here? (Access # platform_dest.backend._platform to create a new Run?) - cloned_s = Scenario( - mp=platform_dest, model=model, scenario=scenario, annotation=annotation - ) cloned_run = self.index[s].clone( model=model, scenario=scenario, keep_solution=keep_solution ) + # Instantiate same class as the original object + cloned_s = s.__class__( + platform_dest, + model, + scenario, + version=cloned_run.version, + ) self._index_and_set_attrs(cloned_run, cloned_s) return cloned_s @@ -645,6 +678,68 @@ def cat_list(self, ms: Scenario, name: str) -> list[str]: category_indexset = self.index[ms].optimization.indexsets.get(f"type_{name}") return cast(list[str], category_indexset.data) + def set_data( + self, + ts: TimeSeries, + region: str, + variable: str, + data: dict[int, float], + unit: str, + subannual: str, + meta: bool, + ) -> None: + log.warning("Parameter `meta` for set_data() currently unused by ixmp4!") + log.warning("Parameter `subannual` for set_data() currently unused by ixmp4!") + + # Construct dataframe as ixmp4 expects it + years = data.keys() + values = data.values() + number_of_years = len(years) + regions = [region] * number_of_years + variables = [variable] * number_of_years + units = [unit] * number_of_years + _data = list(zip(years, values, regions, variables, units)) + + # Add timeseries dataframe + # TODO Is this really the only type we're interested in? + self.index[ts].iamc.add( + pd.DataFrame( + _data, columns=["step_year", "value", "region", "variable", "unit"] + ), + type=DataPoint.Type.ANNUAL, + ) + + def get_data( + self, + ts: TimeSeries, + region: Sequence[str], + variable: Sequence[str], + unit: Sequence[str], + year: Sequence[str], + ) -> Generator[tuple, Any, None]: + data = self.index[ts].iamc.tabulate( + region={"name__in": region}, + variable={"name__in": variable}, + unit={"name__in": unit}, + ) + + # TODO depending on data["type"], a different column name will contain the + # "year" values: + # step_year if type is ANNUAL or CATEGORICAL + # step_category if type is CATEGORICAL + # step_datetime if type is DATETIME + # We're only adding ANNUAL above for now, so let's assume that + data = data.loc[data["step_year"].isin(year)] + + # Select only the columns we're interested in + # TODO the ixmp4 docstrings sounds like region, variable, and unit are not part + # of the df? + data = data[["region", "variable", "unit", "step_year", "value"]] + + # TODO Why would we iterate and yield tuples instead of returning the whole df? + for row in data.itertuples(index=False, name=None): + yield row + # Handle timeslices # def set_timeslice(self, name: str, category: str, duration: float) -> None: @@ -660,8 +755,6 @@ def write_file( IXMP4Backend supports writing to: - ``path='*.gdx', item_type=ItemType.SET | ItemType.PAR``. - - IXMP4Backend does not yet support: - ``path='*.csv', item_type=TS``. The `default` keyword argument is **required**. @@ -712,6 +805,30 @@ def write_file( file_name=_path, record_version_packages=kwargs["record_version_packages"], ) + elif _path.suffix == ".csv" and item_type is ItemType.TS: + models = filters.pop("model") + # NOTE this is what we get for not differentiating e.g. scenario vs + # scenarios in filters... + scenarios = set(cast(list[str], filters.pop("scenario"))) + variables = filters.pop("variable") + units = filters.pop("unit") + regions = filters.pop("region") + default = filters.pop("default") + # TODO (How) Should we include this? + # export_all_runs = filters.pop("export_all_runs") + + data = self._backend.runs.tabulate( + model={"name__in": models}, + scenario={"name__in": scenarios}, + iamc={ + "region": {"name__in": regions}, + "variable": {"name__in": variables}, + "unit": {"name__in": units}, + }, + default_only=default, + ) + data.to_csv(path_or_buf=_path) + else: raise NotImplementedError @@ -794,14 +911,12 @@ def _ni(self, *args, **kwargs): delete = _ni delete_geo = _ni delete_meta = _ni - get_data = _ni get_doc = _ni get_geo = _ni get_meta = _ni get_timeslices = _ni last_update = _ni remove_meta = _ni - set_data = _ni set_doc = _ni set_geo = _ni set_meta = _ni From c5edf10c5f6358e713167fbd9afb930e5c35ee23 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Mon, 3 Mar 2025 12:32:57 +0100 Subject: [PATCH 34/72] Add required units to ixmp4-backed test platforms --- ixmp/testing/data.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ixmp/testing/data.py b/ixmp/testing/data.py index f00043550..888e1208d 100644 --- a/ixmp/testing/data.py +++ b/ixmp/testing/data.py @@ -11,6 +11,7 @@ from ixmp import Platform, Scenario, TimeSeries from ixmp.backend import IAMC_IDX +from ixmp.backend.ixmp4 import IXMP4Backend if TYPE_CHECKING: from typing import TypedDict @@ -182,6 +183,11 @@ def add_test_data(scen: Scenario): return t, t_foo, t_bar, x +def _add_required_units(mp: Platform) -> None: + mp.add_unit("USD") + mp.add_unit("???") + + def make_dantzig( mp: Platform, solve: bool = False, @@ -209,6 +215,9 @@ def make_dantzig( -------- .DantzigModel """ + if isinstance(mp._backend, IXMP4Backend): + _add_required_units(mp=mp) + # Add custom units and region for time series data try: mp.add_unit("USD/km") From 9f0e7f256259ea112a538f9a79465b0200cbbdfa Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Mon, 24 Mar 2025 11:38:42 +0100 Subject: [PATCH 35/72] Add ContainerData to pass data structuredly to GAMS Container --- ixmp/util/ixmp4.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/ixmp/util/ixmp4.py b/ixmp/util/ixmp4.py index af4f448c3..927126480 100644 --- a/ixmp/util/ixmp4.py +++ b/ixmp/util/ixmp4.py @@ -1,6 +1,8 @@ # TODO Import this from typing when dropping Python 3.11 -from typing import TYPE_CHECKING, Any, Optional +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Literal, Optional, Union +import pandas as pd from typing_extensions import TypedDict if TYPE_CHECKING: @@ -18,9 +20,28 @@ class WriteFiltersKwargs(TypedDict, total=False): export_all_runs: bool +@dataclass +class ContainerData: + name: str + kind: Literal["IndexSet", "Table", "Scalar", "Parameter", "Equation", "Variable"] + records: Optional[ + Union[ + float, + list[int], + list[float], + list[str], + dict[str, Union[list[float], list[int], list[str]]], + pd.DataFrame, + ] + ] + domain: Optional[list[str]] = None + docs: Optional[str] = None + + class WriteKwargs(TypedDict, total=False): filters: WriteFiltersKwargs record_version_packages: list[str] + container_data: list[ContainerData] class ReadKwargs(TypedDict, total=False): From affedc5bad46e7ff52bc71c8756058c5bdde8a52 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Mon, 24 Mar 2025 12:38:56 +0100 Subject: [PATCH 36/72] Handle `container_data` in GAMSModel class --- ixmp/model/gams.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ixmp/model/gams.py b/ixmp/model/gams.py index a005deff8..ec7584de3 100644 --- a/ixmp/model/gams.py +++ b/ixmp/model/gams.py @@ -15,6 +15,7 @@ from ixmp.core.scenario import Scenario from ixmp.model.base import Model, ModelError from ixmp.util import as_str_list +from ixmp.util.ixmp4 import ContainerData log = logging.getLogger(__name__) @@ -180,6 +181,9 @@ class GAMSModel(Model): record_version_packages : list of str, optional Names of Python packages to record versions. Default: :py:`["ixmp"]`. See :meth:`record_versions`. + container_data : list of :class:`ixmp.util.ixmp4.ContainerData`, optional + List of data to add to the GAMS Container used by the IXMP4Backend for GAMS I/O. + Default: empty list. """ # noqa: E501 # Make attributes known to self @@ -196,6 +200,7 @@ class GAMSModel(Model): quiet: bool use_temp_dir: bool record_version_packages: list[str] + container_data: list[ContainerData] #: Model name. name = "default" @@ -216,6 +221,7 @@ class GAMSModel(Model): "quiet": False, "use_temp_dir": True, "record_version_packages": ["ixmp"], + "container_data": [], } def __init__(self, name_=None, **model_options): @@ -386,6 +392,7 @@ def run(self, scenario: Scenario) -> None: # Instruct ixmp4 to record package versions if backend_type == "ixmp4": s_arg["record_version_packages"] = self.record_version_packages + s_arg["container_data"] = self.container_data try: # Write model data to file @@ -411,6 +418,7 @@ def run(self, scenario: Scenario) -> None: else: # backend_type == "ixmp4" # Clean up s_arg for use in read_file() s_arg.pop("record_version_packages") + s_arg.pop("container_data") try: # Invoke GAMS run(command, shell=os.name == "nt", cwd=self.cwd, check=True) From 41adec0704738f11a20a9e38b57df546e2539601 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Mon, 24 Mar 2025 12:40:39 +0100 Subject: [PATCH 37/72] Fix various things to enable message_ix.Scenario.solve() * Handle ContainerData passed from message_ix * Ensure items are processed in the correct order * Handle missing column names when reading the solution back to a Run * Handle the special case of "technology"/"tec" --- ixmp/backend/io.py | 384 +++++++++++++++++++++++++++++++----------- ixmp/backend/ixmp4.py | 64 +++++-- 2 files changed, 336 insertions(+), 112 deletions(-) diff --git a/ixmp/backend/io.py b/ixmp/backend/io.py index 1f7504a4a..80865f4d2 100644 --- a/ixmp/backend/io.py +++ b/ixmp/backend/io.py @@ -2,15 +2,14 @@ from collections import deque from collections.abc import Iterable from pathlib import Path -from typing import Optional, TypeVar, Union +from typing import Literal, Optional, TypeVar, Union, cast import gams.transfer as gt import pandas as pd - -# from gams import GamsWorkspace from ixmp4.core import Run +from ixmp4.core.optimization.base import Lister from ixmp4.core.optimization.equation import Equation -from ixmp4.core.optimization.indexset import IndexSet +from ixmp4.core.optimization.indexset import IndexSet, IndexSetRepository from ixmp4.core.optimization.parameter import Parameter from ixmp4.core.optimization.scalar import Scalar from ixmp4.core.optimization.table import Table @@ -19,6 +18,7 @@ from ixmp4.data.abstract.optimization.variable import Variable as AbstractVariable from ixmp.util import as_str_list, maybe_check_out, maybe_commit +from ixmp.util.ixmp4 import ContainerData from . import ItemType @@ -334,70 +334,172 @@ def parse_item_sheets(name): maybe_commit(s, not commit_steps, f"Import from {path}") -def _add_to_container(container: gt.Container, items: list[Item4]) -> None: - """Add ixmp4 data `items` to `container`.""" +def _domain(item: Item4) -> Optional[list[str]]: + """Return domain for `item`. + + For IndexSets and Scalars, this is :obj:`None`. + + For all others, this is `item.indexsets`. + """ + if isinstance(item, (IndexSet, Scalar)): + return None + else: + return item.indexsets + + +def _records( + item: Item4, + kind: str, +) -> Union[ + float, + Union[list[float], list[int], list[str]], + dict[str, Union[list[float], list[int], list[str]]], + None, +]: + """Return records for `item`. + + For Scalars, this is `item.value`. + + For empty other items, this is :obj:`None`. + + For IndexSets and Tables, this is `item.data`. + + For Parameters, this is `item.data` without the column 'units'. + + For Equations and Variables, this is `item.data` + without the columns 'levels' and 'marginals'. + """ + if isinstance(item, Scalar): + return item.value + elif not len(item.data): + return None + + if isinstance(item, (IndexSet, Table)): + return item.data + + result = item.data + + # Pop items not to be stored + for name in { + "Equation": ["levels", "marginals"], + "Parameter": ["units"], + "Variable": ["levels", "marginals"], + }[kind]: + result.pop(name) + + return result + + +def _ensure_correct_item_order(items: list[Item4], repo: Lister) -> list[Item4]: + """Reorder items to ensure the GDX file is written correctly. + + gamsapi stores all unique elements of all items in a single list internally. If item + A adds elements that item B shares, item B is not adding them again. If B's elements + are then requested, they are returned starting with those in A in their order, + disrupting expectations. + """ + if isinstance(repo, IndexSetRepository): + # Move all indexsets called 'type_*' to the end of the list + for type_indexset in list( + filter(lambda item: item.name.startswith("type_"), items) + ): + items.remove(type_indexset) + items.append(type_indexset) + + return items + + +def _align_records_and_domain( + item: Item4, records: dict[str, Union[list[float], list[int], list[str]]] +) -> dict[str, Union[list[float], list[int], list[str]]]: + """Align the order of `records.keys()` with domain of `item`.""" + # This function will only be called for these types + assert isinstance(item, (Table, Parameter, Variable, Equation)) + + # `records` may contain keys that are column_names, but not indexsets + domain_order = item.column_names or item.indexsets + + if not domain_order: + # This could happen for Variables or Equations + return records + else: + # Parameters contain values that we want to keep + if isinstance(item, Parameter): + domain_order.append("values") + + return {key: records[key] for key in domain_order} + + +def _add_items_to_container( + container: gt.Container, items: list[ContainerData] +) -> None: + """Add `items` to `container`.""" if not items: return # Nothing to be done - # Identify the ixmp4 class name and corresponding gams.transfer class - kind = type(items[0]).__name__ - klass = { - "IndexSet": gt.Set, - "Parameter": gt.Parameter, - "Scalar": gt.Parameter, - "Table": gt.Set, - "Variable": gt.Variable, - "Equation": gt.Equation, - }[kind] - - def domain(item: Item4) -> Optional[list[str]]: - if isinstance(item, (IndexSet, Scalar)): - return None - else: - return item.indexsets - - def records( - item: Item4, - ) -> Union[ - float, - Union[list[float], list[int], list[str]], - dict[str, Union[list[float], list[int], list[str]]], - None, - ]: - if isinstance(item, Scalar): - return item.value - elif not len(item.data): - return None - - if isinstance(item, IndexSet): - return item.data - - result = item.data - - # Pop items not to be stored - for name in { - "Equation": ["levels", "marginals"], - "Parameter": ["units"], - "Variable": ["levels", "marginals"], - }[kind]: - result.pop(name) - - return result + for item in items: + # Identify the corresponding gams.transfer class + klass = { + "IndexSet": gt.Set, + "Parameter": gt.Parameter, + "Scalar": gt.Parameter, + "Table": gt.Set, + "Variable": gt.Variable, + "Equation": gt.Equation, + }[item.kind] + + # NOTE A container may already contain some data from the MESSAGE-specific setup + # This seems to imply a UniqueConstraint over all item names, though. + if not container.hasSymbols(item.name): + # The gams documentation confuses me: The docstring says `type` is required + # for Equations, the example says no. It seems to work fine like this, but + # if we do need a value, maybe we could guess based on + # https://github.com/iiasa/ixmp_source/blob/master/src/main/java/at/ac/iiasa/ixmp/objects/Scenario.java#L1926, + + # Create an item instance linked to the container + klass( + container=container, + name=item.name, + domain=item.domain, + records=item.records, + # Optional, but must be str + description=item.docs or "", + ) + + +def _convert_ixmp4_items_to_containerdata(items: list[Item4]) -> list[ContainerData]: + """Convert list of ixmp4 `items` to ContainerData.""" + if not items: + return [] # Nothing to be done + + # Identify the ixmp4 class name + kind = cast( + Literal["Scalar", "IndexSet", "Table", "Parameter", "Equation", "Variable"], + type(items[0]).__name__, + ) + + container_items: list[ContainerData] = [] for item in items: - # The gams documentation confuses me: The docstring says `type` is required for - # Equations, the example says no. It seems to work fine like this, but if we do - # need a value, maybe we could guess based on - # https://github.com/iiasa/ixmp_source/blob/master/src/main/java/at/ac/iiasa/ixmp/objects/Scenario.java#L1926, - klass( - container=container, - name=item.name, - domain=domain(item), - records=records(item), - # Optional, but must be str - description=item.docs if item.docs else "", + records = _records(item=item, kind=kind) + # The order of keys in 'domain' is used by gamsapi to overwrite the order of + # keys in 'records', so make sure they are aligned: + # NOTE ixmp4 ensures that all _records.keys() are in _domain and vice-versa + if isinstance(records, dict): + records = _align_records_and_domain(item=item, records=records) + + container_items.append( + ContainerData( + name=item.name, + kind=kind, + records=records, + domain=_domain(item), + docs=item.docs, + ) ) + return container_items + def _record_versions(container: gt.Container, packages: list[str]) -> None: """Store Python package versions as set elements to be written to GDX. @@ -431,16 +533,35 @@ def _record_versions(container: gt.Container, packages: list[str]) -> None: def write_run_to_gdx( run: Run, file_name: Path, + container_data: list[ContainerData], record_version_packages: list[str], include_variables_and_equations: bool = False, ) -> None: - """Write scenario data from the `run` to a GDX file via GAMS container.""" + """Write data from the `run` to a GDX file at `file_name` via a GAMS Container. + + Parameters + ---------- + run : :class:`ixmp4.core.Run` + The Run in which the Scenario data is stored. + file_name: :obj:`Path` + The location at which the GDX file should be written. + container_data : list of :class:`ixmp.util.ixmp4.ContainerData`. + Data that should also be written to the GDX file + in the form of ContainerData. + record_version_packages : list of str + A list of package names for which versions will be stored in the GDX file. + include_variables_and_equations: bool, optional + Flag to indicate whether Variabels and Equations should be written to the GDX + file. + Default: :obj:`False`. + """ - # TODO How to deal with [*] Set? + # TODO How to deal with [*] Set? That seems to be handled by GAMS automatically. + # Define the container container = gt.Container() - repository = [ + repository: list[Lister] = [ run.optimization.indexsets, run.optimization.scalars, run.optimization.tables, @@ -450,55 +571,88 @@ def write_run_to_gdx( ] idx = slice(None) if include_variables_and_equations else slice(-2) for r in repository[idx]: - # FIXME Type of 'r' is inferred as BaseFacade, the narrowest common supertype of - # IndexSetRepository etc.; BaseFacade.list() is not defined. - _add_to_container(container, r.list()) # type: ignore [attr-defined] + # Reorder items if necessary for GAMS to successfully read the GDX + ixmp4_items = _ensure_correct_item_order(items=r.list(), repo=r) + + # Convert ixmp4 items to ContainerData to streamline adding to container + container_items = _convert_ixmp4_items_to_containerdata(items=ixmp4_items) + + _add_items_to_container(container, container_items) + + # Add additional data *after* the required items to avoid confusing GAMS' internal + # Unique Element List + _add_items_to_container(container, container_data) _record_versions(container=container, packages=record_version_packages) container.write(write_to=file_name) +# NOTE since we currently only read Variables and Equations, this function only covers +# these cases +def _set_columns_to_read_from_records( + item: Union[AbstractVariable, AbstractEquation], +) -> list[str]: + """Gather all columns for `item` to read from GDX records.""" + # Prepare columns to select from container.data + # DF also includes lower, upper, scale + columns = ["levels", "marginals"] + item_columns = item.column_names or item.indexsets + if item_columns: + item_columns.extend(columns) + + return item_columns if item_columns else columns + + +# NOTE not sure we only need equations and variables; if we need others, abstracting one +# function for reading would not be as easy, since we might need different details, so +# I'm keeping them separate for now + + def _read_variables_to_run( container: gt.Container, run: Run, variables: Iterable[AbstractVariable] ) -> None: + """Read `variables` from `container` and store them in `run`.""" for variable in variables: - # Prepare columns to select from container.data - # DF also includes lower, upper, scale - columns = ["level", "marginal"] - variable_columns = variable.column_names or variable.indexsets - if variable_columns: - columns += variable_columns - - run.backend.optimization.variables.add_data( - variable_id=variable.id, - data=( - container.data[variable.name] - .records[columns] - .rename(columns={"level": "levels", "marginal": "marginals"}) - ), - ) + columns_of_interest = _set_columns_to_read_from_records(item=variable) + + records = pd.DataFrame(container.data[variable.name].records) + + # Avoid touching the DB for empty data + if not records.empty: + # NOTE Overwriting the columns changes GAMS' "level" and "marginal" to the + # ixmp4-expected "levels" and "marginals" + # NOTE This assumes that the order the columns are defined in the gams code + # is equal to the order in item.column_names and item.indexsets + records.columns = pd.Index( + columns_of_interest + ["lower", "upper", "scale"] + ) + run.backend.optimization.variables.add_data( + variable_id=variable.id, data=(records[columns_of_interest]) + ) def _read_equations_to_run( container: gt.Container, run: Run, equations: Iterable[AbstractEquation] ) -> None: + """Read `equations` from `container` and store them in `run`.""" for equation in equations: - # Prepare columns to select from container.data - # DF also includes lower, upper, scale - columns = ["level", "marginal"] - equation_columns = equation.column_names or equation.indexsets - if equation_columns: - columns += equation_columns - - run.backend.optimization.equations.add_data( - equation_id=equation.id, - data=( - container.data[equation.name] - .records[columns] - .rename(columns={"level": "levels", "marginal": "marginals"}) - ), - ) + columns_of_interest = _set_columns_to_read_from_records(item=equation) + + records = pd.DataFrame(container.data[equation.name].records) + + # Avoid touching the DB for empty data + if not records.empty: + # NOTE Overwriting the columns changes GAMS' "level" and "marginal" to the + # ixmp4-expected "levels" and "marginals" + # NOTE This assumes that the order the columns are defined in the gams code + # is equal to the order in item.column_names and item.indexsets + records.columns = pd.Index( + columns_of_interest + ["lower", "upper", "scale"] + ) + run.backend.optimization.equations.add_data( + equation_id=equation.id, data=(records[columns_of_interest]) + ) def read_gdx_to_run( @@ -509,6 +663,27 @@ def read_gdx_to_run( comment: str, check_solution: bool, ) -> None: + """Read data from `result_file` to `run` via a GAMS Container. + + Parameters + ---------- + run : :class:`ixmp4.core.Run` + The Run in which the Scenario data is to be stored. + result_file: :obj:`Path` + The location from which the GDX file should be read. + equ_list : list of str + Names of Equations to read from `result_file`. + var_list : list of str + Names of Variables to read from `result_file`. + comment : str + Unused by ixmp4. + A comment to store when adding the data to `run`. + check_solution : bool + Unused by ixmp4. + A flag to indicate whether the solution should be checked + (for consistency, maybe?). + """ + # Warn about unused parameters if comment: log.warning(f"Ignoring comment {comment} for now; unused by ixmp4!") if check_solution: @@ -516,18 +691,25 @@ def read_gdx_to_run( f"Ignoring check_solution={check_solution} for now; unused by ixmp4!" ) + # Create a GAMS Container from the `result_file` + container = gt.Container(load_from=result_file) + + # Load requested Variables and read them to `run` + # NOTE This handles empty `var_list`, too, + # which is not necessary as long as any Variables are required in message_ix variables = ( run.backend.optimization.variables.list(name__in=var_list) if len(var_list) else run.backend.optimization.variables.list() ) + _read_variables_to_run(container=container, run=run, variables=variables) + + # Load requested Equations and read them to `run` + # NOTE This handles empty `equ_list`, too, + # which is not necessary as long as any Equations are required in message_ix equations = ( run.backend.optimization.equations.list(name__in=equ_list) if len(equ_list) else run.backend.optimization.equations.list() ) - - container = gt.Container(load_from=result_file) - - _read_variables_to_run(container=container, run=run, variables=variables) _read_equations_to_run(container=container, run=run, equations=equations) diff --git a/ixmp/backend/ixmp4.py b/ixmp/backend/ixmp4.py index cbf32c83d..d9529256a 100644 --- a/ixmp/backend/ixmp4.py +++ b/ixmp/backend/ixmp4.py @@ -413,7 +413,7 @@ def item_index( if ( isinstance(item, IndexSet) or isinstance(item, Scalar) - or (isinstance(item, Variable) and item.indexsets is None) + or (isinstance(item, (Variable, Equation)) and item.indexsets is None) ): return cast(list[str], []) else: @@ -524,6 +524,22 @@ def _get_set_data( else: return pd.Series(item.data) + def _align_dtypes_for_filters( + self, filters: dict[str, list[Any]], data: pd.DataFrame + ) -> dict[str, list[Any]]: + """Convert `filters` values to types enabling `data.isin()`.""" + # TODO Is this really the proper way to handle these types? + TYPE_MAP = {"object": str, "int64": int} + for column_name in filters.keys(): + if not isinstance( + filters[column_name][0], TYPE_MAP[str(data.dtypes[column_name])] + ): + filters[column_name] = [ + TYPE_MAP[str(data.dtypes[column_name])](value) + for value in filters[column_name] + ] + return filters + def item_get_elements( self, s: Scenario, @@ -542,6 +558,14 @@ def item_get_elements( item = self._get_item(s=s, name=name, type=type) # TODO if item.data can be empty, set columns explicitly as for table above data = pd.DataFrame(item.data) # type: ignore[union-attr] + + # Apply filters if requested + if filters: + # isin() won't consider int(700) to be in ['700'], etc + filters = self._align_dtypes_for_filters(filters=filters, data=data) + + data = data[data.isin(values=filters)[filters.keys()].all(axis=1)] + if type == "par": data.rename(columns={"values": "value", "units": "unit"}, inplace=True) else: @@ -588,6 +612,9 @@ def delete_item( repo = self._get_repo(s=s, type=type) repo.delete(item=name) + # NOTE The name 'cat_`name`' is used for backward compatibility with the JDBC, where + # such names are hardcoded. 'cat' means 'category' and should be expanded for + # clarity in the future. def cat_set_elements( self, ms: Scenario, @@ -600,7 +627,7 @@ def cat_set_elements( For the ixmp4.Table or IndexSet `name`, define a category as a new IndexSet called 'type_`name`' (if it doesn't exist already) and add `cat` to it. Then, - define a new Table 'category_`name`' storing one column for `keys` and one for + define a new Table 'cat_`name`' storing one column for `keys` and one for 'categories'. Parameters @@ -623,7 +650,11 @@ def cat_set_elements( # Assume for now that only IndexSets are requested to be mapped to cats indexset = self.index[ms].optimization.indexsets.get(name=name) - # Get or create the 'type_name' indexset and 'category_name' table + # Special treatment for 'technology' for historical reasons: + if name == "technology": + name = "tec" + + # Get or create the 'type_name' indexset and 'cat_name' table try: category_indexset = self.index[ms].optimization.indexsets.get( name=f"type_{name}" @@ -634,12 +665,10 @@ def cat_set_elements( ) try: - category_table = self.index[ms].optimization.tables.get( - name=f"category_{name}" - ) + category_table = self.index[ms].optimization.tables.get(name=f"cat_{name}") except NotFound: category_table = self.index[ms].optimization.tables.create( - name=f"category_{name}", + name=f"cat_{name}", constrained_to_indexsets=[indexset.name, category_indexset.name], ) @@ -661,19 +690,27 @@ def cat_set_elements( category_indexset.remove(category_indexset.data) # Add data to both objects - category_indexset.add(data=cat) + if cat not in category_indexset.data: + category_indexset.add(data=cat) data = {indexset.name: keys, category_indexset.name: [cat] * len(keys)} category_table.add(data=data) + # TODO In cat_set_elements, we change e.g. cat_technology to cat_tec. Do we need the + # same here or do we expect user code to call this with name == "tec" if they're + # interested in "technology"? def cat_get_elements(self, ms: Scenario, name: str, cat: str) -> list[str]: data = pd.DataFrame( - self.index[ms].optimization.tables.get(name=f"category_{name}").data + self.index[ms].optimization.tables.get(name=f"cat_{name}").data ) - # This assumes there are only two columns + # This assumes there are only two columns: 'type_{name}' and the category name columns = data.columns.to_list() columns.remove(f"type_{name}") - return data[data[f"type_{name}"] == cat, columns[0]].astype(str).to_list() + return data[data[f"type_{name}"] == cat][columns[0]].astype(str).to_list() + + # TODO In cat_set_elements, we change e.g. cat_technology to cat_tec. Do we need the + # same here or do we expect user code to call this with name == "tec" if they're + # interested in "technology"? def cat_list(self, ms: Scenario, name: str) -> list[str]: category_indexset = self.index[ms].optimization.indexsets.get(f"type_{name}") return cast(list[str], category_indexset.data) @@ -804,6 +841,7 @@ def write_file( run=self.index[ts], file_name=_path, record_version_packages=kwargs["record_version_packages"], + container_data=kwargs["container_data"], ) elif _path.suffix == ".csv" and item_type is ItemType.TS: models = filters.pop("model") @@ -872,6 +910,7 @@ def read_file( # TODO handle case when filters is not present ts, _ = self._handle_rw_filters(kwargs["filters"]) + # Convert to normal dict to allow removal of keys _kwargs = {k: v for (k, v) in kwargs.items() if k != "filters"} if _path.suffix == ".gdx" and item_type is ItemType.MODEL: @@ -886,6 +925,9 @@ def read_file( check_solution = bool(_kwargs.pop("check_solution")) comment = str(_kwargs.pop("comment")) + + # message_ix/ixmp4 uses these lists to set the default items to read, so + # these will always be true equ_list = cast(list[str], _kwargs.pop("equ_list")) var_list = cast(list[str], _kwargs.pop("var_list")) From ffda46ad8814837b4658fdc5026b27f76c379526 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Mon, 24 Mar 2025 15:27:07 +0100 Subject: [PATCH 38/72] Improve type hints --- ixmp/report/reporter.py | 4 ++-- ixmp/report/util.py | 18 +++++++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/ixmp/report/reporter.py b/ixmp/report/reporter.py index 3eff09dd4..cab0ebe95 100644 --- a/ixmp/report/reporter.py +++ b/ixmp/report/reporter.py @@ -1,5 +1,5 @@ from itertools import chain, repeat -from typing import Union, cast +from typing import Literal, Union, cast import dask import pandas as pd @@ -53,7 +53,7 @@ def from_scenario(cls, scenario: Scenario, **kwargs) -> "Reporter": all_keys: list[Union[str, Key]] = [] # List of parameters, equations, and variables - quantities = chain( + quantities: chain[tuple[Literal["par", "equ", "var"], str]] = chain( zip(repeat("par"), sorted(scenario.par_list())), zip(repeat("equ"), sorted(scenario.equ_list())), zip(repeat("var"), sorted(scenario.var_list())), diff --git a/ixmp/report/util.py b/ixmp/report/util.py index 3a2430933..dd6e7622b 100644 --- a/ixmp/report/util.py +++ b/ixmp/report/util.py @@ -1,12 +1,18 @@ from functools import lru_cache, partial +from typing import TYPE_CHECKING, Literal, Union import pandas as pd from genno import Key from ixmp.report import common +if TYPE_CHECKING: + from genno.types import AnyQuantity -def dims_for_qty(data): + from ixmp.core.scenario import Scenario + + +def dims_for_qty(data: Union[list[str], pd.DataFrame]) -> list[str]: """Return the list of dimensions for *data*. If *data* is a :class:`pandas.DataFrame`, its columns are processed; @@ -28,7 +34,9 @@ def dims_for_qty(data): return [common.RENAME_DIMS.get(d, d) for d in dims] -def keys_for_quantity(ix_type, name, scenario): +def keys_for_quantity( + ix_type: Literal["par", "equ", "var"], name: str, scenario: "Scenario" +) -> list[tuple[Key, partial["AnyQuantity"], str, str]]: """Return keys for *name* in *scenario*.""" from .operator import data_for_quantity @@ -36,7 +44,7 @@ def keys_for_quantity(ix_type, name, scenario): dims = dims_for_qty(scenario.idx_names(name)) # Column for retrieving data - column = "value" if ix_type == "par" else "lvl" + column: Literal["mrg", "lvl", "value"] = "value" if ix_type == "par" else "lvl" # A computation to retrieve the data result = [ @@ -63,11 +71,11 @@ def keys_for_quantity(ix_type, name, scenario): @lru_cache(1) -def get_reversed_rename_dims(): +def get_reversed_rename_dims() -> dict[str, str]: return {v: k for k, v in common.RENAME_DIMS.items()} -def __getattr__(name: str): +def __getattr__(name: str) -> dict[str, str]: if name == "RENAME_DIMS": return common.RENAME_DIMS else: From f07bb7ee8ef5aae232673191d2d57fbb88c03a44 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Mon, 24 Mar 2025 15:27:44 +0100 Subject: [PATCH 39/72] Rename columns as expected by IXMPReporter --- ixmp/backend/ixmp4.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ixmp/backend/ixmp4.py b/ixmp/backend/ixmp4.py index d9529256a..f3ffba33f 100644 --- a/ixmp/backend/ixmp4.py +++ b/ixmp/backend/ixmp4.py @@ -570,7 +570,10 @@ def item_get_elements( data.rename(columns={"values": "value", "units": "unit"}, inplace=True) else: data.rename( - columns={"levels": "level", "marginals": "marginal"}, inplace=True + # NOTE the ixmp.report(er) expects these names, unfortunately; see + # ixmp.report.util::keys_for_quantity() + columns={"levels": "lvl", "marginals": "mrg"}, + inplace=True, ) return data From ae1cc62335e03bd794ed59f68ba6a63d23072f59 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 25 Mar 2025 15:35:00 +0100 Subject: [PATCH 40/72] Add unit list requiring warnings to Platform --- ixmp/core/platform.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ixmp/core/platform.py b/ixmp/core/platform.py index 4fe07ddce..3cf6eef3c 100644 --- a/ixmp/core/platform.py +++ b/ixmp/core/platform.py @@ -60,6 +60,8 @@ class Platform: "set_meta", ] + _units_to_warn_about: Optional[list[str]] = None + def __init__( self, name: Optional[str] = None, backend: Optional[str] = None, **backend_args ): From 45af85f1bd5028580610a927a6ca8c6bf61072a4 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 25 Mar 2025 15:35:37 +0100 Subject: [PATCH 41/72] Fix annotation for Scenario.scheme --- ixmp/core/scenario.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ixmp/core/scenario.py b/ixmp/core/scenario.py index fd81ef2e2..9581ae00a 100644 --- a/ixmp/core/scenario.py +++ b/ixmp/core/scenario.py @@ -41,7 +41,7 @@ class in :data:`.MODELS` is used to initialize items in the Scenario. """ #: Scheme of the Scenario. - scheme = None + scheme: Optional[str] = None def __init__( self, From b38e11bed0b89a2997fb733e8bb3d4e52cfd1976 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 25 Mar 2025 15:36:22 +0100 Subject: [PATCH 42/72] Use persistent local ixmp4 DB per default --- ixmp/_config.py | 6 +++++- ixmp/backend/ixmp4.py | 17 ++++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/ixmp/_config.py b/ixmp/_config.py index b5cc50149..761e09580 100644 --- a/ixmp/_config.py +++ b/ixmp/_config.py @@ -70,7 +70,11 @@ def _platform_default(): }, "ixmp4-local": { "class": "ixmp4", - "dsn": "sqlite:///:memory:", + "dsn": ( + "sqlite:///" + f"{Path.home().joinpath('.local', 'share', 'ixmp4', 'databases')}" + "/ixmp4-local.sqlite3" + ), }, } diff --git a/ixmp/backend/ixmp4.py b/ixmp/backend/ixmp4.py index f3ffba33f..a1459a31b 100644 --- a/ixmp/backend/ixmp4.py +++ b/ixmp/backend/ixmp4.py @@ -40,7 +40,6 @@ class IXMP4Backend(CachingBackend): index: dict[TimeSeries, "Run"] = {} def __init__(self, _backend: Optional["ixmp4_backend"] = None) -> None: - from ixmp4 import Platform from ixmp4.conf.base import PlatformInfo from ixmp4.data.backend import SqliteTestBackend @@ -48,14 +47,21 @@ def __init__(self, _backend: Optional["ixmp4_backend"] = None) -> None: if not _backend: log.warning("Falling back to default SqliteBackend 'ixmp4-local'") sqlite = SqliteTestBackend( - PlatformInfo(name="ixmp4-local", dsn="sqlite:///:memory:") + PlatformInfo( + name="ixmp4-local", + dsn=( + "sqlite:///" + f"{Path.home().joinpath('.local', 'share', 'ixmp4', 'databases')}" # noqa: E501 + "/ixmp4-local.sqlite3" + ), + ) ) sqlite.setup() self._backend = _backend if _backend else sqlite # Instantiate and store - self._platform = Platform(_backend=self._backend) + self._platform = ixmp4_platform(_backend=self._backend) # def __del__(self) -> None: # self.close_db() @@ -186,6 +192,11 @@ def _index_and_set_attrs(self, run: "Run", ts: TimeSeries) -> None: # Something went wrong on the ixmp4 side raise RuntimeError(f"got version {v} instead of {ts.version}") + # NOTE ixmp4 doesn't store 'scheme', which could in principle be anything. + # Since this is a shim between message_ix/ixmp4, I'm hardcoding this for now. + if isinstance(ts, Scenario): + ts.scheme = "MESSAGE" + def init(self, ts: TimeSeries, annotation: str) -> None: run = self._platform.runs.create(model=ts.model, scenario=ts.scenario) # TODO either do log.warning that annotation is unused or From 813698bb54763b619deb5293f2637bb36f7dc7fb Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Wed, 9 Apr 2025 11:13:33 +0200 Subject: [PATCH 43/72] Enable tab completion for platform.__init__(backend=...) --- ixmp/core/platform.py | 7 +++++-- ixmp/testing/__init__.py | 7 +++++-- ixmp/tests/core/test_platform.py | 9 ++++++--- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/ixmp/core/platform.py b/ixmp/core/platform.py index 3cf6eef3c..b3142001c 100644 --- a/ixmp/core/platform.py +++ b/ixmp/core/platform.py @@ -1,7 +1,7 @@ import logging from collections.abc import Sequence from os import PathLike -from typing import TYPE_CHECKING, Optional, Union +from typing import TYPE_CHECKING, Literal, Optional, Union import numpy as np import pandas as pd @@ -63,7 +63,10 @@ class Platform: _units_to_warn_about: Optional[list[str]] = None def __init__( - self, name: Optional[str] = None, backend: Optional[str] = None, **backend_args + self, + name: Optional[str] = None, + backend: Optional[Literal["ixmp4", "jdbc"]] = None, + **backend_args, ): if name is None: if backend is None and not len(backend_args): diff --git a/ixmp/testing/__init__.py b/ixmp/testing/__init__.py index 9a9861799..dc5488892 100644 --- a/ixmp/testing/__init__.py +++ b/ixmp/testing/__init__.py @@ -41,7 +41,7 @@ from copy import deepcopy from itertools import chain from pathlib import Path -from typing import Any +from typing import Any, Literal import pint import pytest @@ -410,7 +410,10 @@ def _setup_ixmp4_platform(mp: Platform) -> None: def _platform_fixture( - request: pytest.FixtureRequest, tmp_env, test_data_path, backend: str + request: pytest.FixtureRequest, + tmp_env, + test_data_path, + backend: Literal["jdbc", "ixmp4"], ) -> Generator[Platform, Any, None]: """Helper for :func:`test_mp` and other fixtures.""" # Long, unique name for the platform. diff --git a/ixmp/tests/core/test_platform.py b/ixmp/tests/core/test_platform.py index f0b125cab..23af1fce4 100644 --- a/ixmp/tests/core/test_platform.py +++ b/ixmp/tests/core/test_platform.py @@ -6,7 +6,7 @@ from collections.abc import Generator from pathlib import Path from sys import getrefcount -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Literal from weakref import getweakrefcount import pandas as pd @@ -32,7 +32,8 @@ def test_init0(self) -> None: ValueError, match=re.escape("backend class 'foo' not among"), ): - ixmp.Platform(backend="foo") + # Testing the wrong type on purpose + ixmp.Platform(backend="foo") # type: ignore[arg-type] # name="default" is used, referring to "local" mp = ixmp.Platform() @@ -45,7 +46,9 @@ def test_init0(self) -> None: pytest.param("ixmp4", dict(), marks=min_ixmp4_version), ), ) - def test_init1(self, backend: str, backend_args: dict[str, str]) -> None: + def test_init1( + self, backend: Literal["jdbc", "ixmp4"], backend_args: dict[str, str] + ) -> None: # Platform can be instantiated ixmp.Platform(backend=backend, **backend_args) From 1c33c1597d21c8c02dbff7fae3e5577814509e6e Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Wed, 9 Apr 2025 11:15:29 +0200 Subject: [PATCH 44/72] Fix read-in of Equations & Variables and gams container creation --- ixmp/backend/io.py | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/ixmp/backend/io.py b/ixmp/backend/io.py index 80865f4d2..2fb99c6b0 100644 --- a/ixmp/backend/io.py +++ b/ixmp/backend/io.py @@ -430,6 +430,22 @@ def _align_records_and_domain( return {key: records[key] for key in domain_order} +def _update_item_in_container(container: gt.Container, item: ContainerData) -> None: + """Update `item` in `container`.""" + # NOTE This is not updating the comment + + kwargs = dict(name=item.name, domain=item.domain, records=item.records) + + if item.kind in ("IndexSet", "Table"): + container.addSet(**kwargs) + elif item.kind in ("Scalar", "Parameter"): + container.addParameter(**kwargs) + elif item.kind == "Equation": + container.addEquation(**kwargs) + else: # Variable + container.addVariable(**kwargs) + + def _add_items_to_container( container: gt.Container, items: list[ContainerData] ) -> None: @@ -465,6 +481,8 @@ def _add_items_to_container( # Optional, but must be str description=item.docs or "", ) + else: + _update_item_in_container(container=container, item=item) def _convert_ixmp4_items_to_containerdata(items: list[Item4]) -> list[ContainerData]: @@ -628,7 +646,7 @@ def _read_variables_to_run( columns_of_interest + ["lower", "upper", "scale"] ) run.backend.optimization.variables.add_data( - variable_id=variable.id, data=(records[columns_of_interest]) + variable_id=variable.id, data=records[columns_of_interest] ) @@ -651,7 +669,7 @@ def _read_equations_to_run( columns_of_interest + ["lower", "upper", "scale"] ) run.backend.optimization.equations.add_data( - equation_id=equation.id, data=(records[columns_of_interest]) + equation_id=equation.id, data=records[columns_of_interest] ) @@ -698,9 +716,9 @@ def read_gdx_to_run( # NOTE This handles empty `var_list`, too, # which is not necessary as long as any Variables are required in message_ix variables = ( - run.backend.optimization.variables.list(name__in=var_list) + run.backend.optimization.variables.list(run_id=run.id, name__in=var_list) if len(var_list) - else run.backend.optimization.variables.list() + else run.backend.optimization.variables.list(run_id=run.id) ) _read_variables_to_run(container=container, run=run, variables=variables) @@ -708,8 +726,8 @@ def read_gdx_to_run( # NOTE This handles empty `equ_list`, too, # which is not necessary as long as any Equations are required in message_ix equations = ( - run.backend.optimization.equations.list(name__in=equ_list) + run.backend.optimization.equations.list(run_id=run.id, name__in=equ_list) if len(equ_list) - else run.backend.optimization.equations.list() + else run.backend.optimization.equations.list(run_id=run.id) ) _read_equations_to_run(container=container, run=run, equations=equations) From 22e19d21f6b75cf2014dec4329a74843d9841f00 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Wed, 9 Apr 2025 11:17:34 +0200 Subject: [PATCH 45/72] Update ixmp4-branch dependency --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 14bb83484..eb2301131 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ docs = [ ] # Change ixmp4 to new release after https://github.com/iiasa/ixmp4/pull/164 is merged ixmp4 = [ - "ixmp4 @ git+https://github.com/iiasa/ixmp4@enh/make-equation-indexsets-optional", + "ixmp4 @ git+https://github.com/iiasa/ixmp4@enh/remove-linked-items-when-removing-indexset-items", "gamsapi[core,transfer] >= 45.7.0", ] report = ["genno[compat,graphviz]"] From 0f4e6d554820ae5eb5d55cd8efe4ec6b6eb113a0 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Wed, 9 Apr 2025 11:17:51 +0200 Subject: [PATCH 46/72] Enable all message_ix tutorials * Handle adding duplicate data to items * Fix handling of empty data * Handle special case of 'addon' category * Handle filters when retrieving Table data --- ixmp/backend/ixmp4.py | 136 +++++++++++++++++++++++++++++++----------- ixmp/core/scenario.py | 1 + 2 files changed, 102 insertions(+), 35 deletions(-) diff --git a/ixmp/backend/ixmp4.py b/ixmp/backend/ixmp4.py index a1459a31b..36cebf5b6 100644 --- a/ixmp/backend/ixmp4.py +++ b/ixmp/backend/ixmp4.py @@ -429,7 +429,7 @@ def item_index( return cast(list[str], []) else: if sets_or_names == "names" and item.column_names is None: - log.warning( + log.debug( f"Requested {sets_or_names}, but these are None for item " f"{item.name}, falling back on (Index)Set names!" ) @@ -448,14 +448,28 @@ def _add_data_to_set( "`comment` currently unused with ixmp4 when adding data to Tables." ) # Assumption: if key is just one value, we're dealing with an IndexSet + # NOTE E.g. westeros_addon_technologies in message_ix calls + # `scenario.add_set("addon", "po_turbine")` for a 1-D Table called "addon". This + # only works now because ixmp.Scenario.add_set() converts str keys for Tables to + # [key] before adding them. We would need to replicate that or adapt the + # decision logic here should we drop ixmp.Scenario. if isinstance(key, str): - self.index[s].optimization.indexsets.get(name=name).add(key) + # NOTE ixmp_source silently ignores duplicate data; replicate here + # This could be improved by adding data without loading the indexset first, + # but this requires users to ensure their data are valid + indexset = self.index[s].optimization.indexsets.get(name=name) + if key not in indexset.data: + indexset.add(key) else: table = self.index[s].optimization.tables.get(name=name) # TODO should we enforce in ixmp4 that when constrained_to_indexsets # contains duplicate values, column_names must be provided? keys = table.column_names or table.indexsets - data_to_add = {keys[i]: [key[i]] for i in range(len(key))} + data_to_add = pd.DataFrame({keys[i]: [key[i]] for i in range(len(key))}) + + # Silently ignore duplicate data, see NOTE above + data_to_add = data_to_add[~data_to_add.isin(values=table.data).all(axis=1)] + table.add(data=data_to_add) def _create_scalar( @@ -511,13 +525,19 @@ def item_set_elements( self._add_data_to_set(s=s, name=name, key=key, comment=comment) else: if key is None: - assert value, "Creating a Scalar requires a value!" + assert isinstance(value, float), ( + "Creating a Scalar requires a value!" + ) self._create_scalar( s=s, name=name, value=value, unit=unit, comment=comment ) else: - assert value, "Adding data to a Parameter requires a value!" - assert unit, "Adding data to a Parameter requires a unit!" + assert isinstance(value, float), ( + "Adding data to a Parameter requires a value!" + ) + assert isinstance(unit, str), ( + "Adding data to a Parameter requires a unit!" + ) self._add_data_to_parameter( s=s, name=name, key=key, value=value, unit=unit, comment=comment ) @@ -525,19 +545,27 @@ def item_set_elements( def _get_set_data( self, s: Scenario, name: str, filters: Optional[dict[str, list[Any]]] = None ) -> Union[pd.Series, pd.DataFrame]: - # TODO handle filters - item = self._get_indexset_or_table(s=s, name=name) if isinstance(item, Table): columns = item.column_names or item.indexsets - return pd.DataFrame(item.data, columns=columns) + df = pd.DataFrame(item.data, columns=columns) + return df[df.isin(values=filters).all(axis=1)] if filters else df else: - return pd.Series(item.data) + series = pd.Series(item.data) + return series[series.isin(values=filters[name])] if filters else series + + def _ensure_filters_values_are_lists( + self, filters: dict[str, Union[Any, list[Any]]] + ) -> None: + """Convert singular values in `filters` to lists.""" + for k, v in filters.items(): + if not isinstance(v, list): + filters[k] = [v] def _align_dtypes_for_filters( self, filters: dict[str, list[Any]], data: pd.DataFrame - ) -> dict[str, list[Any]]: + ) -> None: """Convert `filters` values to types enabling `data.isin()`.""" # TODO Is this really the proper way to handle these types? TYPE_MAP = {"object": str, "int64": int} @@ -549,43 +577,58 @@ def _align_dtypes_for_filters( TYPE_MAP[str(data.dtypes[column_name])](value) for value in filters[column_name] ] - return filters def item_get_elements( self, s: Scenario, type: Literal["equ", "par", "set", "var"], name: str, - filters: Optional[dict[str, list[Any]]] = None, + filters: Optional[dict[str, Any]] = None, ) -> Union[dict[str, Any], pd.Series, pd.DataFrame]: - # TODO handle filters if type == "set": return self._get_set_data(s=s, name=name, filters=filters) # TODO this is not handling scalars at the moment, but maybe try with type, # except NotFound, try scalar? else: - # TODO this can really only be Equation, Parameter, or Variable, so cast - # as such? item = self._get_item(s=s, name=name, type=type) - # TODO if item.data can be empty, set columns explicitly as for table above - data = pd.DataFrame(item.data) # type: ignore[union-attr] + + # TODO What about Scalars? How does message_ix load them? + # This should always be the case + assert isinstance(item, (Parameter, Equation, Variable)) + + data = pd.DataFrame(item.data) + + if data.empty: + # Set correct columns + columns = item.column_names or item.indexsets + unindexed_columns = ( + ["value", "unit"] if type == "par" else ["lvl", "mrg"] + ) + if columns: + columns.extend(unindexed_columns) + + return pd.DataFrame( + columns=columns if bool(columns) else unindexed_columns + ) # Apply filters if requested if filters: # isin() won't consider int(700) to be in ['700'], etc - filters = self._align_dtypes_for_filters(filters=filters, data=data) + self._ensure_filters_values_are_lists(filters=filters) + self._align_dtypes_for_filters(filters=filters, data=data) data = data[data.isin(values=filters)[filters.keys()].all(axis=1)] - if type == "par": - data.rename(columns={"values": "value", "units": "unit"}, inplace=True) - else: - data.rename( - # NOTE the ixmp.report(er) expects these names, unfortunately; see - # ixmp.report.util::keys_for_quantity() - columns={"levels": "lvl", "marginals": "mrg"}, - inplace=True, - ) + # Rename columns + # NOTE the ixmp.report(er) expects these names, unfortunately; see + # ixmp.report.util::keys_for_quantity() + renames = ( + {"values": "value", "units": "unit"} + if type == "par" + else {"levels": "lvl", "marginals": "mrg"} + ) + data.rename(columns=renames, inplace=True) + return data def item_delete_elements( @@ -659,31 +702,52 @@ def cat_set_elements( - The Backend **must** remove any existing member of `cat`, so that it has only one element. """ - from ixmp4.core.exceptions import NotFound + column_name: Optional[str] = None - # Assume for now that only IndexSets are requested to be mapped to cats - indexset = self.index[ms].optimization.indexsets.get(name=name) + # Categories can be based on IndexSets directly or on 1-d Tables + try: + # Most should be based on IndexSets, try that first + indexset = self.index[ms].optimization.indexsets.get(name=name) + indexset_name = indexset.name + except IndexSet.NotFound: + # We're dealing with a Table + table = self.index[ms].optimization.tables.get(name=name) + + # Ensure the Table's dimensions are correct before setting variables + assert len(table.indexsets) == 1 + indexset_name = table.indexsets[0] + column_name = table.name # Special treatment for 'technology' for historical reasons: if name == "technology": name = "tec" + # NOTE Of the default items, all categories except 'cat_addon' are based on + # IndexSets. 'cat_addon' is based on the 1-d Table 'addon', which is why it also + # doesn't follow the naming convention. Usually, the column names are + # 'type_' and '', but for 'cat_addon', it's 'type_addon' and + # 'technology_addon'. + elif name == "addon": + column_name = "technology_addon" # Get or create the 'type_name' indexset and 'cat_name' table try: category_indexset = self.index[ms].optimization.indexsets.get( name=f"type_{name}" ) - except NotFound: + except IndexSet.NotFound: category_indexset = self.index[ms].optimization.indexsets.create( name=f"type_{name}" ) try: category_table = self.index[ms].optimization.tables.get(name=f"cat_{name}") - except NotFound: + except Table.NotFound: category_table = self.index[ms].optimization.tables.create( name=f"cat_{name}", - constrained_to_indexsets=[indexset.name, category_indexset.name], + constrained_to_indexsets=[indexset_name, category_indexset.name], + column_names=[column_name, category_indexset.name] + if column_name + else None, ) # Convert for convenience @@ -706,7 +770,9 @@ def cat_set_elements( # Add data to both objects if cat not in category_indexset.data: category_indexset.add(data=cat) - data = {indexset.name: keys, category_indexset.name: [cat] * len(keys)} + data = {column_name: keys} if column_name else {indexset_name: keys} + data[category_indexset.name] = [cat] * len(keys) + category_table.add(data=data) # TODO In cat_set_elements, we change e.g. cat_technology to cat_tec. Do we need the diff --git a/ixmp/core/scenario.py b/ixmp/core/scenario.py index 9581ae00a..4cb1c89fe 100644 --- a/ixmp/core/scenario.py +++ b/ixmp/core/scenario.py @@ -705,6 +705,7 @@ def remove_par(self, name: str, key=None) -> None: else: self._backend("item_delete_elements", "par", name, self._keys(name, key)) + # FIXME What ensures that filters has the correct type? def var(self, name: str, filters=None, **kwargs): """Return a dataframe of (filtered) elements for a specific variable. From 461945b93bc1d59c8268085bb6d2a8c7d34d2343 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Wed, 9 Apr 2025 12:17:31 +0200 Subject: [PATCH 47/72] Use specific ixmp4 branch for pre-commit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 74dc91f22..8d3a762c1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: additional_dependencies: - genno - GitPython - - ixmp4 + - "ixmp4 @ git+https://github.com/iiasa/ixmp4@enh/remove-linked-items-when-removing-indexset-items" - nbclient - pandas-stubs - pytest From e51393e204b0474fd4f189426dace7699dc3ab3e Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Wed, 9 Apr 2025 13:43:45 +0200 Subject: [PATCH 48/72] Fix some docstrings and type hints --- ixmp/backend/io.py | 2 +- ixmp/backend/ixmp4.py | 202 +++++++++++++++++++++++++++++++++++------- ixmp/model/gams.py | 6 +- 3 files changed, 173 insertions(+), 37 deletions(-) diff --git a/ixmp/backend/io.py b/ixmp/backend/io.py index 2fb99c6b0..04ac6c39c 100644 --- a/ixmp/backend/io.py +++ b/ixmp/backend/io.py @@ -449,7 +449,7 @@ def _update_item_in_container(container: gt.Container, item: ContainerData) -> N def _add_items_to_container( container: gt.Container, items: list[ContainerData] ) -> None: - """Add `items` to `container`.""" + """Add or update `items` to/in `container`.""" if not items: return # Nothing to be done diff --git a/ixmp/backend/ixmp4.py b/ixmp/backend/ixmp4.py index 36cebf5b6..22f70fc2e 100644 --- a/ixmp/backend/ixmp4.py +++ b/ixmp/backend/ixmp4.py @@ -2,18 +2,18 @@ from collections.abc import Generator, Iterable, MutableMapping, Sequence from os import PathLike from pathlib import Path -from typing import Any, Literal, Optional, Union, cast +from typing import Any, Literal, Optional, TypeAlias, Union, cast, overload import pandas as pd from ixmp4 import DataPoint from ixmp4 import Platform as ixmp4_platform from ixmp4.core import Run -from ixmp4.core.optimization.equation import Equation -from ixmp4.core.optimization.indexset import IndexSet -from ixmp4.core.optimization.parameter import Parameter -from ixmp4.core.optimization.scalar import Scalar -from ixmp4.core.optimization.table import Table -from ixmp4.core.optimization.variable import Variable +from ixmp4.core.optimization.equation import Equation, EquationRepository +from ixmp4.core.optimization.indexset import IndexSet, IndexSetRepository +from ixmp4.core.optimization.parameter import Parameter, ParameterRepository +from ixmp4.core.optimization.scalar import Scalar, ScalarRepository +from ixmp4.core.optimization.table import Table, TableRepository +from ixmp4.core.optimization.variable import Variable, VariableRepository from ixmp4.data.backend.base import Backend as ixmp4_backend # TODO Import this from typing when dropping Python 3.11 @@ -176,9 +176,9 @@ def get_nodes(self) -> list[tuple[str, None, str, str]]: # Modifying the Run object def _index_and_set_attrs(self, run: "Run", ts: TimeSeries) -> None: - """Add *run* to index and update attributes of *ts*. + """Add `run` to index and update attributes of `ts`. - Helper for init and get. + Helper for `init()` and `get()`. """ # Add to index self.index[ts] = run @@ -281,12 +281,40 @@ def has_solution(self, s: Scenario) -> bool: # Handle optimization items - # TODO: type hints + Ixmp4ItemType: TypeAlias = Literal["scalar", "indexset", "set", "par", "equ", "var"] + + @overload + def _get_repo(self, s: Scenario, type: Literal["scalar"]) -> ScalarRepository: ... + + @overload def _get_repo( - self, - s: Scenario, - type: Literal["scalar", "indexset", "set", "par", "equ", "var"], - ): + self, s: Scenario, type: Literal["indexset"] + ) -> IndexSetRepository: ... + + @overload + def _get_repo(self, s: Scenario, type: Literal["set"]) -> TableRepository: ... + + @overload + def _get_repo(self, s: Scenario, type: Literal["par"]) -> ParameterRepository: ... + + @overload + def _get_repo(self, s: Scenario, type: Literal["equ"]) -> EquationRepository: ... + + @overload + def _get_repo(self, s: Scenario, type: Literal["var"]) -> VariableRepository: ... + + # NOTE Type hints here ensure the function body is checked + def _get_repo( + self, s: Scenario, type: Ixmp4ItemType + ) -> Union[ + IndexSetRepository, + TableRepository, + ScalarRepository, + ParameterRepository, + EquationRepository, + VariableRepository, + ]: + """Return the repository of items of `type` belonging to `s`.""" if type == "scalar": return self.index[s].optimization.scalars if type == "indexset": @@ -312,8 +340,8 @@ def init_item( # In base::item_get_elements, it sounds like "equ" and "var" can also target # scalars, whereas below, inspired from jdbc, I'm only linking "par" to scalars if type == "set" and len(idx_sets) == 0: - repo = self._get_repo(s=s, type="indexset") - repo.create(name=name) + indexset_repo = self._get_repo(s=s, type="indexset") + indexset_repo.create(name=name) elif type == "par" and len(idx_sets) == 0: # NOTE ixmp4 requires scalars to get value and unit upon creation, so the # shim does that when `item_set_elements()` targets a scalar @@ -321,7 +349,9 @@ def init_item( else: repo = self._get_repo(s=s, type=type) repo.create( - name=name, constrained_to_indexsets=idx_sets, column_names=idx_names + name=name, + constrained_to_indexsets=list(idx_sets), + column_names=list(idx_names) if idx_names else None, ) def list_items(self, s: Scenario, type: Literal["set", "par", "equ"]) -> list[str]: @@ -339,7 +369,7 @@ def _find_item( self, s: Scenario, name: str, - types: tuple[Literal["scalar", "indexset", "set", "par", "equ", "var"], ...] = ( + types: tuple[Ixmp4ItemType, ...] = ( "scalar", "indexset", "set", @@ -348,7 +378,7 @@ def _find_item( "var", ), ) -> tuple[ - Literal["scalar", "indexset", "set", "par", "equ", "var"], + Ixmp4ItemType, Union[ "Scalar", "IndexSet", @@ -358,6 +388,28 @@ def _find_item( "Parameter", ], ]: + """Find item `name` in Scenario `s`. + + Parameters + ---------- + s : Scenario + The Scenario in which to search for `name`. + name : str + The name to search for. + types : tuple[`Ixmp4ItemType`, ...], optional + If provided, search only specific types for `name`. For example, + types=("indexset", "set"). Default: tuple of all types. + + Returns + ------- + tuple[`Ixmp4ItemType`, Scalar | IndexSet | Table | Parameter | Equation | Variable] + A tuple containing the string item type and the item itself. + + Raises + ------ + KeyError + If no item with `name` and type in `types` can be found in `s`. + """ # noqa: E501 # NOTE this currently assumes that `name` will only be present once among # Tables, Parameters, Equations, Variables. This is in line with the assumption # made in the Java backend, but ixmp4 enforces no such constraint. @@ -389,11 +441,28 @@ def _find_item( else: return (_type, _item) + @overload + def _get_item(self, s: Scenario, name: str, type: Literal["scalar"]) -> Scalar: ... + + @overload def _get_item( - self, - s: Scenario, - name: str, - type: Literal["scalar", "indexset", "set", "par", "equ", "var"], + self, s: Scenario, name: str, type: Literal["indexset"] + ) -> IndexSet: ... + + @overload + def _get_item(self, s: Scenario, name: str, type: Literal["set"]) -> Table: ... + + @overload + def _get_item(self, s: Scenario, name: str, type: Literal["par"]) -> Parameter: ... + + @overload + def _get_item(self, s: Scenario, name: str, type: Literal["equ"]) -> Equation: ... + + @overload + def _get_item(self, s: Scenario, name: str, type: Literal["var"]) -> Variable: ... + + def _get_item( + self, s: Scenario, name: str, type: Ixmp4ItemType ) -> Union[ "Scalar", "IndexSet", @@ -402,19 +471,28 @@ def _get_item( "Variable", "Parameter", ]: + """Retrieve item `name` with `type` from `s`. + + Returns + ------- + Scalar | IndexSet | Table | Parameter | Equation | Variable + Based on `type`. + """ return self._get_repo(s=s, type=type).get(name=name) def _get_indexset_or_table( self, s: Scenario, name: str ) -> Union["IndexSet", "Table"]: - from ixmp4.core.exceptions import NotFound + """Get `name` from Scenario `s`. + Try first if `name` is an IndexSet. Get it as a Table if it isn't. + """ try: - repo = self._get_repo(s=s, type="indexset") - return cast(IndexSet, repo.get(name=name)) - except NotFound: - repo = self._get_repo(s=s, type="set") - return cast(Table, repo.get(name=name)) + indexset_repo = self._get_repo(s=s, type="indexset") + return indexset_repo.get(name=name) + except IndexSet.NotFound: + table_repo = self._get_repo(s=s, type="set") + return table_repo.get(name=name) def item_index( self, s: Scenario, name: str, sets_or_names: Literal["sets", "names"] @@ -433,7 +511,7 @@ def item_index( f"Requested {sets_or_names}, but these are None for item " f"{item.name}, falling back on (Index)Set names!" ) - assert item.indexsets # Could only be None for Variables + assert item.indexsets # Could only be None for Variables & Equations return ( item.column_names if item.column_names and sets_or_names == "names" @@ -443,6 +521,22 @@ def item_index( def _add_data_to_set( self, s: Scenario, name: str, key: Union[str, list[str]], comment: Optional[str] ) -> None: + """Add data `key` to `name` in Scenario `s`. + + Parameters + ---------- + s : Scenario + The Scenario hosting the item. + name : str + The name of the item to add data to. + key : str | list[str] + The data to add. + + ATTENTION: if `key` is a str, we're assuming `name` means an IndexSet; + if `key` is a list, we're assuming `name` means a Table. + comment: str, optional + A message to store with the data addition. Unused by ixmp4. + """ if comment: log.warning( "`comment` currently unused with ixmp4 when adding data to Tables." @@ -480,6 +574,21 @@ def _create_scalar( unit: Optional[str], comment: Optional[str], ) -> None: + """Create the Scalar `name` in Scenario `s`. + + Parameters + ---------- + s : Scenario + The Scenario hosting the Scalar. + name : str + The name of the Scalar. + value : float + The value of the Scalar. + unit : str, optional + The unit of the Scalar. + comment: str, optional + A message to explain what this Scalar means. + """ scalar = self.index[s].optimization.scalars.create( name=name, value=value, unit=unit ) @@ -495,6 +604,23 @@ def _add_data_to_parameter( unit: str, comment: Optional[str], ) -> None: + """Add data `key` to the Parameter `name` in Scenario `s`. + + Parameters + ---------- + s : Scenario + The Scenario hosting the Parameter. + name : str + The name of the Parameter. + key : str | list[str] + The data to add to the Parameter. + value : float + The value of the Parameter. + unit : str + The unit of the Parameter. + comment: str, optional + A message to store with the data addition. Unused by ixmp4. + """ if comment: log.warning( "`comment` currently unused with ixmp4 when adding data to Parameters." @@ -545,6 +671,18 @@ def item_set_elements( def _get_set_data( self, s: Scenario, name: str, filters: Optional[dict[str, list[Any]]] = None ) -> Union[pd.Series, pd.DataFrame]: + """Get the data stored in `name` in `s`. + + Parameters + ---------- + s : Scenario + The Scenario hosting the item. + name : str + The name of the item. + filters : dict[str, list[Any]], optional + Filters to apply to the data. If present, return only matching data. + Default: None. + """ item = self._get_indexset_or_table(s=s, name=name) if isinstance(item, Table): @@ -593,8 +731,6 @@ def item_get_elements( item = self._get_item(s=s, name=name, type=type) # TODO What about Scalars? How does message_ix load them? - # This should always be the case - assert isinstance(item, (Parameter, Equation, Variable)) data = pd.DataFrame(item.data) @@ -651,7 +787,7 @@ def item_delete_elements( data = pd.DataFrame(keys, columns=columns) item.remove(data=data) else: - parameter = cast(Parameter, self._get_item(s=s, name=name, type="par")) + parameter = self._get_item(s=s, name=name, type="par") columns = parameter.column_names or parameter.indexsets data = pd.DataFrame(keys, columns=columns) parameter.remove(data=data) diff --git a/ixmp/model/gams.py b/ixmp/model/gams.py index ec7584de3..d098c383a 100644 --- a/ixmp/model/gams.py +++ b/ixmp/model/gams.py @@ -8,7 +8,7 @@ from pathlib import Path from subprocess import CalledProcessError, check_output, run from tempfile import TemporaryDirectory -from typing import Any, Optional +from typing import Any, Optional, Union from ixmp.backend import ItemType from ixmp.backend.jdbc import JDBCBackend @@ -373,7 +373,7 @@ def run(self, scenario: Scenario) -> None: # Assemble the full command: executable, model file, model-specific arguments, # and general GAMS arguments - command = ( + command: Union[str, list[str]] = ( ["gams", f'"{model_file}"'] + [self.format(arg) for arg in self.solve_args] + self.gams_args @@ -381,7 +381,7 @@ def run(self, scenario: Scenario) -> None: if os.name == "nt": # Windows: join the commands to a single string - command = " ".join(command) # type: ignore[assignment] + command = " ".join(command) # Remove stored reference to the Scenario to allow it to be GC'd later delattr(self, "scenario") From d0925f0d8d6a8d9a2883467a126bd8dac8d35b3f Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 15 Apr 2025 13:06:08 +0200 Subject: [PATCH 49/72] Handle missing kwargs["filters"] for base Backend file I/O --- ixmp/backend/base.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ixmp/backend/base.py b/ixmp/backend/base.py index 276631b19..2903618af 100644 --- a/ixmp/backend/base.py +++ b/ixmp/backend/base.py @@ -425,7 +425,8 @@ def read_file( -------- write_file """ - s, _ = self._handle_rw_filters(kwargs["filters"]) + filters = kwargs["filters"] if "filters" in kwargs else WriteFiltersKwargs() + s, _ = self._handle_rw_filters(filters=filters) _kwargs = {k: v for (k, v) in kwargs.items() if k != "filters"} path = Path(path) @@ -471,7 +472,8 @@ def write_file( """ # Use the "scenario" filter to retrieve the Scenario `s` to be written; reappend # any other filters - s, kwargs["filters"] = self._handle_rw_filters(kwargs["filters"]) + filters = kwargs["filters"] if "filters" in kwargs else WriteFiltersKwargs() + s, kwargs["filters"] = self._handle_rw_filters(filters=filters) xlsx_types = (ItemType.SET | ItemType.PAR, ItemType.MODEL) path = Path(path) From b1078cafc67126ba59cf502596eb19bc94f93303 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 15 Apr 2025 13:07:16 +0200 Subject: [PATCH 50/72] Provide GAMS-required Equation type --- ixmp/backend/io.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/ixmp/backend/io.py b/ixmp/backend/io.py index 04ac6c39c..d3ce8ee33 100644 --- a/ixmp/backend/io.py +++ b/ixmp/backend/io.py @@ -349,7 +349,6 @@ def _domain(item: Item4) -> Optional[list[str]]: def _records( item: Item4, - kind: str, ) -> Union[ float, Union[list[float], list[int], list[str]], @@ -381,10 +380,10 @@ def _records( # Pop items not to be stored for name in { - "Equation": ["levels", "marginals"], - "Parameter": ["units"], - "Variable": ["levels", "marginals"], - }[kind]: + Equation: ["levels", "marginals"], + Parameter: ["units"], + Variable: ["levels", "marginals"], + }[type(item)]: result.pop(name) return result @@ -431,7 +430,10 @@ def _align_records_and_domain( def _update_item_in_container(container: gt.Container, item: ContainerData) -> None: - """Update `item` in `container`.""" + """Update `item` in `container`. + + Note that this overwrites existing records of `item`. + """ # NOTE This is not updating the comment kwargs = dict(name=item.name, domain=item.domain, records=item.records) @@ -441,6 +443,9 @@ def _update_item_in_container(container: gt.Container, item: ContainerData) -> N elif item.kind in ("Scalar", "Parameter"): container.addParameter(**kwargs) elif item.kind == "Equation": + # NOTE This was hardcoded in ixmp_source the same way; not sure how the + # 'type' in the container affects anything + kwargs["type"] = "E" container.addEquation(**kwargs) else: # Variable container.addVariable(**kwargs) @@ -467,10 +472,9 @@ def _add_items_to_container( # NOTE A container may already contain some data from the MESSAGE-specific setup # This seems to imply a UniqueConstraint over all item names, though. if not container.hasSymbols(item.name): - # The gams documentation confuses me: The docstring says `type` is required - # for Equations, the example says no. It seems to work fine like this, but - # if we do need a value, maybe we could guess based on - # https://github.com/iiasa/ixmp_source/blob/master/src/main/java/at/ac/iiasa/ixmp/objects/Scenario.java#L1926, + # NOTE This was hardcoded in ixmp_source the same way; not sure how the + # 'type' in the container affects anything + kwargs = {"type": "E"} if item.kind == "Equation" else {} # Create an item instance linked to the container klass( @@ -480,6 +484,7 @@ def _add_items_to_container( records=item.records, # Optional, but must be str description=item.docs or "", + **kwargs, ) else: _update_item_in_container(container=container, item=item) @@ -499,7 +504,7 @@ def _convert_ixmp4_items_to_containerdata(items: list[Item4]) -> list[ContainerD container_items: list[ContainerData] = [] for item in items: - records = _records(item=item, kind=kind) + records = _records(item=item) # The order of keys in 'domain' is used by gamsapi to overwrite the order of # keys in 'records', so make sure they are aligned: # NOTE ixmp4 ensures that all _records.keys() are in _domain and vice-versa From 9057b46734e49215118fada690e153fe8ee548ac Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 15 Apr 2025 13:08:01 +0200 Subject: [PATCH 51/72] Fix IXMP4Backend for more passing tests * Guard against empty data * Extract functions not actually linked to the Backend * Set Scenario.scheme only if not already present --- ixmp/backend/ixmp4.py | 98 +++++++++++++++++++++++-------------------- 1 file changed, 53 insertions(+), 45 deletions(-) diff --git a/ixmp/backend/ixmp4.py b/ixmp/backend/ixmp4.py index 22f70fc2e..768f1d04a 100644 --- a/ixmp/backend/ixmp4.py +++ b/ixmp/backend/ixmp4.py @@ -2,7 +2,7 @@ from collections.abc import Generator, Iterable, MutableMapping, Sequence from os import PathLike from pathlib import Path -from typing import Any, Literal, Optional, TypeAlias, Union, cast, overload +from typing import Any, Literal, Optional, Union, cast, overload import pandas as pd from ixmp4 import DataPoint @@ -17,7 +17,7 @@ from ixmp4.data.backend.base import Backend as ixmp4_backend # TODO Import this from typing when dropping Python 3.11 -from typing_extensions import Unpack +from typing_extensions import TypeAlias, Unpack from ixmp.backend import ItemType from ixmp.backend.base import CachingBackend, ReadKwargs, WriteKwargs @@ -29,6 +29,30 @@ log = logging.getLogger(__name__) +def _ensure_filters_values_are_lists(filters: dict[str, Union[Any, list[Any]]]) -> None: + """Convert singular values in `filters` to lists.""" + for k, v in filters.items(): + if not isinstance(v, list): + filters[k] = [v] + + +def _align_dtypes_for_filters( + filters: dict[str, list[Any]], data: pd.DataFrame +) -> None: + """Convert `filters` values to types enabling `data.isin()`.""" + # TODO Is this really the proper way to handle these types? + TYPE_MAP = {"object": str, "int64": int} + + for column_name in filters.keys(): + if not isinstance( + filters[column_name][0], TYPE_MAP[str(data.dtypes[column_name])] + ): + filters[column_name] = [ + TYPE_MAP[str(data.dtypes[column_name])](value) + for value in filters[column_name] + ] + + class IXMP4Backend(CachingBackend): """Backend using :mod:`ixmp4`.""" @@ -193,9 +217,10 @@ def _index_and_set_attrs(self, run: "Run", ts: TimeSeries) -> None: raise RuntimeError(f"got version {v} instead of {ts.version}") # NOTE ixmp4 doesn't store 'scheme', which could in principle be anything. - # Since this is a shim between message_ix/ixmp4, I'm hardcoding this for now. + # Since this is a shim between message_ix/ixmp4, I'm hardcoding this as default. if isinstance(ts, Scenario): - ts.scheme = "MESSAGE" + if not ts.scheme: + ts.scheme = "IXMP4-MESSAGE" def init(self, ts: TimeSeries, annotation: str) -> None: run = self._platform.runs.create(model=ts.model, scenario=ts.scenario) @@ -437,7 +462,7 @@ def _find_item( break if _item is None or _type is None: - raise KeyError(f"No item called {name} found on this Scenario!") + raise KeyError(f"No item called '{name}' found on this Scenario!") else: return (_type, _item) @@ -519,7 +544,11 @@ def item_index( ) def _add_data_to_set( - self, s: Scenario, name: str, key: Union[str, list[str]], comment: Optional[str] + self, + s: Scenario, + name: str, + key: Union[str, list[str]], + comment: Optional[str] = None, ) -> None: """Add data `key` to `name` in Scenario `s`. @@ -572,7 +601,7 @@ def _create_scalar( name: str, value: float, unit: Optional[str], - comment: Optional[str], + comment: Optional[str] = None, ) -> None: """Create the Scalar `name` in Scenario `s`. @@ -602,7 +631,7 @@ def _add_data_to_parameter( key: Union[str, list[str]], value: float, unit: str, - comment: Optional[str], + comment: Optional[str] = None, ) -> None: """Add data `key` to the Parameter `name` in Scenario `s`. @@ -693,29 +722,6 @@ def _get_set_data( series = pd.Series(item.data) return series[series.isin(values=filters[name])] if filters else series - def _ensure_filters_values_are_lists( - self, filters: dict[str, Union[Any, list[Any]]] - ) -> None: - """Convert singular values in `filters` to lists.""" - for k, v in filters.items(): - if not isinstance(v, list): - filters[k] = [v] - - def _align_dtypes_for_filters( - self, filters: dict[str, list[Any]], data: pd.DataFrame - ) -> None: - """Convert `filters` values to types enabling `data.isin()`.""" - # TODO Is this really the proper way to handle these types? - TYPE_MAP = {"object": str, "int64": int} - for column_name in filters.keys(): - if not isinstance( - filters[column_name][0], TYPE_MAP[str(data.dtypes[column_name])] - ): - filters[column_name] = [ - TYPE_MAP[str(data.dtypes[column_name])](value) - for value in filters[column_name] - ] - def item_get_elements( self, s: Scenario, @@ -750,8 +756,8 @@ def item_get_elements( # Apply filters if requested if filters: # isin() won't consider int(700) to be in ['700'], etc - self._ensure_filters_values_are_lists(filters=filters) - self._align_dtypes_for_filters(filters=filters, data=data) + _ensure_filters_values_are_lists(filters=filters) + _align_dtypes_for_filters(filters=filters, data=data) data = data[data.isin(values=filters)[filters.keys()].all(axis=1)] @@ -976,18 +982,20 @@ def get_data( unit={"name__in": unit}, ) - # TODO depending on data["type"], a different column name will contain the - # "year" values: - # step_year if type is ANNUAL or CATEGORICAL - # step_category if type is CATEGORICAL - # step_datetime if type is DATETIME - # We're only adding ANNUAL above for now, so let's assume that - data = data.loc[data["step_year"].isin(year)] - - # Select only the columns we're interested in - # TODO the ixmp4 docstrings sounds like region, variable, and unit are not part - # of the df? - data = data[["region", "variable", "unit", "step_year", "value"]] + # Protect against empty data + if not data.empty: + # TODO depending on data["type"], a different column name will contain the + # "year" values: + # step_year if type is ANNUAL or CATEGORICAL + # step_category if type is CATEGORICAL + # step_datetime if type is DATETIME + # We're only adding ANNUAL above for now, so let's assume that + data = data.loc[data["step_year"].isin(year)] + + # Select only the columns we're interested in + # TODO the ixmp4 docstrings sounds like region, variable, and unit are not + # part of the df? + data = data[["region", "variable", "unit", "step_year", "value"]] # TODO Why would we iterate and yield tuples instead of returning the whole df? for row in data.itertuples(index=False, name=None): From 866b9305929a5ba59f46ba589621ff04a85c2287 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 15 Apr 2025 13:09:57 +0200 Subject: [PATCH 52/72] Add new pytest markers to distinguish backends --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index eb2301131..5a385f2a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -115,6 +115,8 @@ addopts = """ markers = [ "rixmp: test of the ixmp R interface.", "performance: ixmp performance test.", + "jdbc: tests exclusive to JDBCBackend.", + "ixmp4: tests exclusive to IXMP4Backend.", ] tmp_path_retention_policy = "none" From 81d769f54c37f6afe633ec273dfadc5495128269 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 15 Apr 2025 13:10:46 +0200 Subject: [PATCH 53/72] Parametrize tests based on new markers & add units for ixmp4 --- ixmp/testing/__init__.py | 56 ++++++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/ixmp/testing/__init__.py b/ixmp/testing/__init__.py index dc5488892..2996196fe 100644 --- a/ixmp/testing/__init__.py +++ b/ixmp/testing/__init__.py @@ -131,6 +131,20 @@ def pytest_report_header(config, start_path) -> str: return f"ixmp config: {repr(ixmp_config.values)}" +# NOTE https://docs.pytest.org/en/latest/example/markers.html#marking-platform-specific-tests-with-pytest +# sound like what we need, but I couldn't quite get it to work. Instead, this is more +# following https://pytest-with-eric.com/introduction/pytest-generate-tests/ +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + if "backend" in metafunc.fixturenames: + markers = [m.name for m in metafunc.definition.iter_markers()] + + requested_backends = [marker for marker in markers if marker in backends] + + _backends = requested_backends if bool(requested_backends) else backends + + metafunc.parametrize("backend", _backends, indirect=True) + + # Session-scoped fixtures @@ -163,18 +177,26 @@ def test_data_path() -> Path: return Path(__file__).parents[1].joinpath("tests", "data") -@pytest.fixture(scope="module", params=backends) +# NOTE We need to declare this as module-scope explicitly; otherwise, pytest creates +# backend for pytest_generate_tests as function-scoped fixture automatically +@pytest.fixture(scope="module") +def backend(request): + return request.param + + +@pytest.fixture(scope="module") def test_mp( - request: pytest.FixtureRequest, tmp_env, test_data_path + request: pytest.FixtureRequest, + tmp_env, + test_data_path, + backend: Literal["ixmp4", "jdbc"], ) -> Generator[Platform, Any, None]: """An empty :class:`.Platform` connected to a temporary, in-memory database. This fixture has **module** scope: the same Platform is reused for all tests in a module. """ - yield from _platform_fixture( - request, tmp_env, test_data_path, backend=request.param - ) + yield from _platform_fixture(request, tmp_env, test_data_path, backend=backend) @pytest.fixture(scope="session") @@ -257,8 +279,13 @@ def protect_rename_dims(): RENAME_DIMS.update(saved) -@pytest.fixture(scope="function", params=backends) -def test_mp_f(request: pytest.FixtureRequest, tmp_env, test_data_path): +@pytest.fixture(scope="function") +def test_mp_f( + request: pytest.FixtureRequest, + tmp_env, + test_data_path, + backend: Literal["ixmp4", "jdbc"], +) -> Generator[Platform, Any, None]: """An empty :class:`Platform` connected to a temporary, in-memory database. This fixture has **function** scope: the same Platform is reused for one test @@ -268,9 +295,7 @@ def test_mp_f(request: pytest.FixtureRequest, tmp_env, test_data_path): -------- test_mp """ - yield from _platform_fixture( - request, tmp_env, test_data_path, backend=request.param - ) + yield from _platform_fixture(request, tmp_env, test_data_path, backend=backend) # Assertions @@ -401,11 +426,16 @@ def create_test_platform(tmp_path, data_path, name, **properties): # Private utilities -# TODO Check if this setup is necessary when running all tests (not just test_platform) +# NOTE The test suite doesn't need all units/regions defined by default +# Only those marked with 'tutorials' are needed for the MESSAGE tutorials def _setup_ixmp4_platform(mp: Platform) -> None: """Set up an ixmp4-backed Platform with things hardcoded in Java.""" - mp.add_unit("cases", comment="As pre-defined in Java.") - mp.add_unit("km", comment="As pre-defined in Java.") + mp.add_unit("???", comment="As pre-defined in Java.") + mp.add_unit("GWa", comment="As pre-defined in Java.") + mp.add_unit("USD/kWa", comment="As pre-defined in Java.") + mp.add_unit("cases", comment="As pre-defined in Java.") # tutorials + mp.add_unit("kg", comment="As pre-defined in Java.") + mp.add_unit("km", comment="As pre-defined in Java.") # tutorials mp.add_region(region="World", hierarchy="common") From 3de8d42ce3d25b46e7b7a323887ad65df97e6b91 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 15 Apr 2025 13:12:09 +0200 Subject: [PATCH 54/72] Add tests for new/private ixmp4-related functions --- ixmp/tests/backend/test_io.py | 426 ++++++++++++++++++++++++++++++- ixmp/tests/backend/test_ixmp4.py | 226 ++++++++++++++++ 2 files changed, 651 insertions(+), 1 deletion(-) create mode 100644 ixmp/tests/backend/test_ixmp4.py diff --git a/ixmp/tests/backend/test_io.py b/ixmp/tests/backend/test_io.py index 65c9cc3fc..394e2185f 100644 --- a/ixmp/tests/backend/test_io.py +++ b/ixmp/tests/backend/test_io.py @@ -1,8 +1,31 @@ +from pathlib import Path +from typing import Union + +import gams.transfer as gt +import pytest + import ixmp +from ixmp.backend.io import ( + _add_items_to_container, + _align_records_and_domain, + _convert_ixmp4_items_to_containerdata, + _domain, + _ensure_correct_item_order, + _read_equations_to_run, + _read_variables_to_run, + _record_versions, + _records, + _set_columns_to_read_from_records, + _update_item_in_container, + read_gdx_to_run, + write_run_to_gdx, +) +from ixmp.backend.ixmp4 import IXMP4Backend from ixmp.testing import add_random_model_data, models +from ixmp.util.ixmp4 import ContainerData -def test_read_excel_big(test_mp, tmp_path): +def test_read_excel_big(test_mp: ixmp.Platform, tmp_path: Path) -> None: """Excel files with model items split across sheets can be read. https://github.com/iiasa/ixmp/pull/345. @@ -25,3 +48,404 @@ def test_read_excel_big(test_mp, tmp_path): scen_empty.read_excel(tmp_path) assert len(scen_empty.par("random_par")) == 25 + + +# TODO Should all tests handling scenarios get `request` and use unique scenario names? +# TODO Should we make the run-retrieval its own fixture? + + +@pytest.mark.ixmp4 +def test__domain(test_mp: ixmp.Platform) -> None: + scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") + + # TODO Remove after proper parametrization + assert isinstance(test_mp._backend, IXMP4Backend) + + run = test_mp._backend.index[scen] + indexset = run.optimization.indexsets.create("Indexset") + table = run.optimization.tables.create( + "Table", constrained_to_indexsets=[indexset.name] + ) + + assert _domain(indexset) is None + assert _domain(table) == [indexset.name] + + +@pytest.mark.ixmp4 +def test__records(test_mp: ixmp.Platform) -> None: + scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") + + test_mp.add_unit("unit") + + # TODO Remove after proper parametrization + assert isinstance(test_mp._backend, IXMP4Backend) + + run = test_mp._backend.index[scen] + + # Test records of a Scalar + scalar = run.optimization.scalars.create("Scalar", 3.14, "unit") + assert _records(scalar) == scalar.value + + # Test records of an empty Table + indexset = run.optimization.indexsets.create("Indexset") + table = run.optimization.tables.create( + "Table", constrained_to_indexsets=[indexset.name] + ) + assert _records(table) is None + + # Test records of an IndexSet + indexset.add(["foo", "bar"]) + assert _records(indexset) == indexset.data + + # Test records of a Parameter + parameter = run.optimization.parameters.create( + "Parameter", constrained_to_indexsets=[indexset.name] + ) + parameter.add( + {indexset.name: ["foo", "bar"], "values": [1, 2], "units": ["unit"] * 2} + ) + expected = parameter.data.copy() + expected.pop("units") + assert _records(parameter) == expected + + +@pytest.mark.ixmp4 +def test__ensure_correct_item_order(test_mp: ixmp.Platform) -> None: + scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") + + test_mp.add_unit("unit") + + # TODO Remove after proper parametrization + assert isinstance(test_mp._backend, IXMP4Backend) + + run = test_mp._backend.index[scen] + + years = run.optimization.indexsets.create("years") + nodes = run.optimization.indexsets.create("nodes") + type_years = run.optimization.indexsets.create("type_years") + type_nodes = run.optimization.indexsets.create("type_nodes") + + assert _ensure_correct_item_order( + items=[type_years, years, type_nodes, nodes], repo=run.optimization.indexsets + ) == [years, nodes, type_years, type_nodes] + + +@pytest.mark.ixmp4 +def test__align_records_and_domain(test_mp: ixmp.Platform) -> None: + scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") + + # TODO Remove after proper parametrization + assert isinstance(test_mp._backend, IXMP4Backend) + + run = test_mp._backend.index[scen] + + # Test item without domain order + equation = run.optimization.equations.create("Equation") + expected: dict[str, Union[list[float], list[int], list[str]]] = {"foo": [1, 2, 3]} + assert _align_records_and_domain(item=equation, records=expected) == expected + + # Test sorting of records + indexset_1 = run.optimization.indexsets.create("Indexset 1") + indexset_2 = run.optimization.indexsets.create("Indexset 2") + parameter = run.optimization.parameters.create( + "Parameter", constrained_to_indexsets=[indexset_1.name, indexset_2.name] + ) + records: dict[str, Union[list[float], list[int], list[str]]] = { + indexset_2.name: [1, 2, 3], + "values": [1.0, 2.0, 3.0], + indexset_1.name: ["1", "2", "3"], + } + expected = { + indexset_1.name: ["1", "2", "3"], + indexset_2.name: [1, 2, 3], + "values": [1.0, 2.0, 3.0], + } + + assert _align_records_and_domain(item=parameter, records=records) == expected + + +def test__update_item_in_container() -> None: + item_list = [ + ContainerData( + name="table", kind="Table", records=["foo", "bar"], domain=["indexset"] + ), + ContainerData(name="scalar", kind="Scalar", records=3.14), + ContainerData(name="equation", kind="Equation", records=None), + ContainerData(name="variable", kind="Variable", records=None), + ] + + container = gt.Container() + + for item in item_list: + _update_item_in_container(container=container, item=item) + + assert all(container.hasSymbols(symbols=[item.name for item in item_list])) + + +def test__add_items_to_container() -> None: + container = gt.Container() + + # Test handling of empty list + empty_list: list[ContainerData] = [] + _add_items_to_container(container=container, items=empty_list) + assert container.data == {} + + # Test adding new item + records = ["foo", "bar"] + item_list = [ContainerData(name="Indexset", kind="IndexSet", records=records)] + _add_items_to_container(container=container, items=item_list) + assert container.hasSymbols("Indexset") + indexsets = container.getSets() + assert len(indexsets) == 1 + indexset = indexsets[0] + assert indexset.records["uni"].to_list() == records + + # Test updating existing item + records = ["baz"] + item_list = [ContainerData(name="Indexset", kind="IndexSet", records=records)] + _add_items_to_container(container=container, items=item_list) + indexsets = container.getSets() + assert len(indexsets) == 1 + indexset = indexsets[0] + assert indexset.records["uni"].to_list() == records + + +@pytest.mark.ixmp4 +def test__convert_ixmp4_items_to_containerdata(test_mp: ixmp.Platform) -> None: + scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") + + # TODO Remove after proper parametrization + assert isinstance(test_mp._backend, IXMP4Backend) + + run = test_mp._backend.index[scen] + + # Test handling of empty list + empty_list = _convert_ixmp4_items_to_containerdata(items=[]) + assert len(empty_list) == 0 + + # Test adding an Indexset (covers no new cases, but we need to index a Table) + indexset = run.optimization.indexsets.create("Indexset") + indexset.add(data=["foo", "bar", "baz"]) + container_list = _convert_ixmp4_items_to_containerdata(items=[indexset]) + assert len(container_list) == 1 + indexset_data = container_list[0] + assert indexset.name == indexset_data.name + assert indexset.data == indexset_data.records + + # Test adding an item where records are a dict + table = run.optimization.tables.create( + "Table", constrained_to_indexsets=[indexset.name] + ) + table.add(data={indexset.name: ["baz", "foo", "bar"]}) + container_list = _convert_ixmp4_items_to_containerdata(items=[table]) + table_data = container_list[0] + assert table.name == table_data.name + assert table.data == table_data.records + + +def test__record_version() -> None: + container = gt.Container() + + # Test recording the package version of something that's always present + _record_versions(container=container, packages=["ixmp"]) + indexsets = container.getSets() + assert len(indexsets) == 1 + indexset = indexsets[0] + assert indexset.name == "ixmp_version" + assert indexset.domain == ["*", "*"] + + +@pytest.mark.ixmp4 +def test_write_run_to_gdx(test_mp: ixmp.Platform, tmp_path: Path) -> None: + scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") + + # TODO Remove after proper parametrization + assert isinstance(test_mp._backend, IXMP4Backend) + + run = test_mp._backend.index[scen] + + # Test minimal call targeting all kinds of items + # NOTE All other functions are unit-tested already and other tests call write() with + # actual data, so we probably don't need to replicate here + file_path = tmp_path / "test_write.gdx" + write_run_to_gdx( + run=run, + file_name=file_path, + container_data=[], + record_version_packages=["ixmp"], + include_variables_and_equations=True, + ) + container = gt.Container(load_from=tmp_path / "test_write.gdx") + assert len(container.data) == 1 + assert "ixmp_version" in container.data.keys() + + +@pytest.mark.ixmp4 +def test__set_columns_to_read_from_records(test_mp: ixmp.Platform) -> None: + scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") + + # TODO Remove after proper parametrization + assert isinstance(test_mp._backend, IXMP4Backend) + + run = test_mp._backend.index[scen] + default_columns = ["levels", "marginals"] + + # Test an Equation without dimensions + equation = run.backend.optimization.equations.create(run_id=run.id, name="Equation") + columns = _set_columns_to_read_from_records(item=equation) + assert columns == default_columns + + # Test an indexed Variable + indexset = run.optimization.indexsets.create("Indexset") + variable = run.backend.optimization.variables.create( + run_id=run.id, name="Variable", constrained_to_indexsets=[indexset.name] + ) + columns = _set_columns_to_read_from_records(item=variable) + assert columns == [indexset.name] + default_columns + + # Test an Equation with dimension names + equation_2 = run.backend.optimization.equations.create( + run_id=run.id, + name="Equation 2", + constrained_to_indexsets=[indexset.name], + column_names=["Column"], + ) + columns = _set_columns_to_read_from_records(item=equation_2) + assert columns == ["Column"] + default_columns + + +# NOTE Keeping the read_equ()/read_var() tests separate because the read_*() functions +# are separate + + +@pytest.mark.ixmp4 +def test__read_variables_to_run(test_mp: ixmp.Platform) -> None: + scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") + + # TODO Remove after proper parametrization + assert isinstance(test_mp._backend, IXMP4Backend) + + run = test_mp._backend.index[scen] + indexset = run.optimization.indexsets.create("Indexset") + indexset.add(data=["foo", "bar", "baz"]) + variable = run.backend.optimization.variables.create( + run_id=run.id, name="Variable", constrained_to_indexsets=[indexset.name] + ) + container = gt.Container() + # NOTE Reading GDX always contains 'level', 'marginal', 'lower', 'upper', 'scale' + # The first two will be changed to plural, the latter three ignored + records = { + indexset.name: ["foo", "bar", "baz"], + "level": [1.0, 2.0, 3.0], + "marginal": [0, 0, 0], + "lower": [0, 0, 0], + "upper": [0, 0, 0], + "scale": [0, 0, 0], + } + container.addVariable(name="Variable", domain=[indexset.name], records=records) + + _read_variables_to_run(container=container, run=run, variables=[variable]) + expected = { + indexset.name: records[indexset.name], + "levels": records["level"], + "marginals": records["marginal"], + } + variable = run.backend.optimization.variables.get(run_id=run.id, name=variable.name) + assert variable.data == expected + + +@pytest.mark.ixmp4 +def test__read_equations_to_run(test_mp: ixmp.Platform) -> None: + scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") + + # TODO Remove after proper parametrization + assert isinstance(test_mp._backend, IXMP4Backend) + + run = test_mp._backend.index[scen] + indexset = run.optimization.indexsets.create("Indexset") + indexset.add(data=["foo", "bar", "baz"]) + equation = run.backend.optimization.equations.create( + run_id=run.id, name="Equation", constrained_to_indexsets=[indexset.name] + ) + container = gt.Container() + # NOTE Reading GDX always contains 'level', 'marginal', 'lower', 'upper', 'scale' + # The first two will be changed to plural, the latter three ignored + records = { + indexset.name: ["foo", "bar", "baz"], + "level": [1.0, 2.0, 3.0], + "marginal": [0, 0, 0], + "lower": [0, 0, 0], + "upper": [0, 0, 0], + "scale": [0, 0, 0], + } + container.addEquation( + name="Equation", type="E", domain=[indexset.name], records=records + ) + + _read_equations_to_run(container=container, run=run, equations=[equation]) + expected = { + indexset.name: records[indexset.name], + "levels": records["level"], + "marginals": records["marginal"], + } + equation = run.backend.optimization.equations.get(run_id=run.id, name=equation.name) + assert equation.data == expected + + +@pytest.mark.ixmp4 +def test_read_gdx_to_run(test_mp: ixmp.Platform, tmp_path: Path) -> None: + scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") + + # TODO Remove after proper parametrization + assert isinstance(test_mp._backend, IXMP4Backend) + + run = test_mp._backend.index[scen] + + # NOTE Names without space to produce "valid GAMS names" + variable_1 = run.backend.optimization.variables.create( + run_id=run.id, name="Variable1" + ) + variable_2 = run.backend.optimization.variables.create( + run_id=run.id, name="Variable2" + ) + variable_3 = run.backend.optimization.variables.create( + run_id=run.id, name="Variable3" + ) + records: dict[str, Union[list[float], list[int], list[str]]] = { + "level": [1.0], + "marginal": [0], + "lower": [0], + "upper": [0], + "scale": [0], + } + file_path = tmp_path / "test_read.gdx" + write_run_to_gdx( + run=run, + file_name=file_path, + container_data=[ + ContainerData(name=var.name, kind="Variable", records=records) + for var in [variable_1, variable_2, variable_3] + ], + record_version_packages=["ixmp"], + include_variables_and_equations=True, + ) + var_list = [variable_1.name, variable_3.name] + + # Test minimal confirmable call (with otherwise unused parameters) + read_gdx_to_run( + run=run, + result_file=file_path, + equ_list=[], + var_list=var_list, + comment="Test comment", + check_solution=True, + ) + for var in run.optimization.variables.list(): + # 'variable_2' is not supposed to be read in + expected = ( + {} + if var.name == variable_2.name + else {"levels": records["level"], "marginals": records["marginal"]} + ) + + assert var.data == expected diff --git a/ixmp/tests/backend/test_ixmp4.py b/ixmp/tests/backend/test_ixmp4.py new file mode 100644 index 000000000..e7745fbb2 --- /dev/null +++ b/ixmp/tests/backend/test_ixmp4.py @@ -0,0 +1,226 @@ +from typing import Any, Literal, cast + +import pandas as pd +import pytest +from ixmp4.core.optimization.equation import EquationRepository +from ixmp4.core.optimization.indexset import IndexSetRepository +from ixmp4.core.optimization.parameter import ParameterRepository +from ixmp4.core.optimization.scalar import ScalarRepository +from ixmp4.core.optimization.table import TableRepository +from ixmp4.core.optimization.variable import VariableRepository + +from ixmp import Platform, Scenario +from ixmp.backend.ixmp4 import ( + IXMP4Backend, + _align_dtypes_for_filters, + _ensure_filters_values_are_lists, +) + +# NOTE Not writing a test for _index_and_set_attrs(). This is a helper function that's +# also used by JDBC, and is always called for Backend.init() and Backend.get(), so is +# tested already. +# @pytest.mark.ixmp4 +# def test__index_and_set_attrs(test_mp: Platform) -> None: + + +@pytest.mark.ixmp4 +def test__get_repo(test_mp: Platform) -> None: + scenario = Scenario(test_mp, "model", "scenario", version="new") + repos: dict[Literal["indexset", "scalar", "set", "par", "equ", "var"], Any] = { + "indexset": IndexSetRepository, + "scalar": ScalarRepository, + "set": TableRepository, + "par": ParameterRepository, + "equ": EquationRepository, + "var": VariableRepository, + } + + # Ensure we're handling the correct type of backend + backend = test_mp._backend + assert isinstance(backend, IXMP4Backend) + + # Test correct kind of instance is returned + for type, expected_repo in repos.items(): + repo = backend._get_repo(s=scenario, type=type) + assert isinstance(repo, expected_repo) + + +@pytest.mark.ixmp4 +def test__find_item(test_mp: Platform) -> None: + scenario = Scenario(test_mp, "model", "scenario", version="new") + + # Ensure we're handling the correct type of backend + backend = test_mp._backend + assert isinstance(backend, IXMP4Backend) + + # Test unknown item name raises + with pytest.raises(KeyError, match="No item called 'NotFound' found"): + backend._find_item(s=scenario, name="NotFound") + + # Test finding an item and its type + name = "foo" + _type: Literal["indexset"] = "indexset" + scenario.init_set(name=name) + return_type, return_item = backend._find_item( + s=scenario, name=name, types=tuple([_type]) + ) + assert (_type, name) == (return_type, return_item.name) + + +@pytest.mark.ixmp4 +def test__get_item(test_mp: Platform) -> None: + scenario = Scenario(test_mp, "model", "scenario", version="new") + + # Ensure we're handling the correct type of backend + backend = test_mp._backend + assert isinstance(backend, IXMP4Backend) + + # Test getting an item + name = "foo" + _type: Literal["indexset"] = "indexset" + scenario.init_set(name=name) + indexset = backend._get_item(s=scenario, name=name, type=_type) + assert indexset.name == name + + +@pytest.mark.ixmp4 +def test__get_indexset_or_table(test_mp: Platform) -> None: + scenario = Scenario(test_mp, "model", "scenario", version="new") + + # Ensure we're handling the correct type of backend + backend = test_mp._backend + assert isinstance(backend, IXMP4Backend) + + # Test getting an item of "unknown" type (either 'indexset' or 'table'); + # getting a Table first tests getting an IndexSet, too + indexset_name = "foo" + scenario.init_set(name=indexset_name) + table_name = "bar" + scenario.init_set(name=table_name, idx_sets=[indexset_name]) + table = backend._get_indexset_or_table(s=scenario, name=table_name) + assert table.name == table_name + + +@pytest.mark.ixmp4 +def test__add_data_to_set(test_mp: Platform) -> None: + scenario = Scenario(test_mp, "model", "scenario", version="new") + + # Ensure we're handling the correct type of backend + backend = test_mp._backend + assert isinstance(backend, IXMP4Backend) + + # Test adding to an Indexset and warning about `comment` + indexset_name = "Indexset" + scenario.init_set(name=indexset_name) + key = "foo" + backend._add_data_to_set( + s=scenario, name=indexset_name, key=key, comment="Test comment" + ) + indexset_data = scenario.set(indexset_name) + assert isinstance(indexset_data, pd.Series) + pd.testing.assert_series_equal(indexset_data, pd.Series([key])) + + # Test adding to a Table + table_name = "Table" + scenario.init_set(name=table_name, idx_sets=[indexset_name]) + backend._add_data_to_set(s=scenario, name=table_name, key=[key]) + # We can assume this data type for Tables + pd.testing.assert_frame_equal( + cast(pd.DataFrame, scenario.set(table_name)), + pd.DataFrame({indexset_name: [key]}), + ) + + +@pytest.mark.ixmp4 +def test__create_scalar(test_mp: Platform) -> None: + scenario = Scenario(test_mp, "model", "scenario", version="new") + + # Ensure we're handling the correct type of backend + backend = test_mp._backend + assert isinstance(backend, IXMP4Backend) + + # Test creating a scalar with a comment + name = "Scalar" + value = 3.141592653589793238462643383279502884197169399375105820 + comment = "Comment" + backend._create_scalar( + s=scenario, name=name, value=value, unit=None, comment=comment + ) + # NOTE We don't really have a way to retrieve Scalars from IXMP4Backend right now + scalar = backend.index[scenario].optimization.scalars.get(name=name) + assert scalar.value == value + assert scalar.docs == comment + + +@pytest.mark.ixmp4 +def test__add_data_to_parameter(test_mp: Platform) -> None: + scenario = Scenario(test_mp, "model", "scenario", version="new") + + # Ensure we're handling the correct type of backend + backend = test_mp._backend + assert isinstance(backend, IXMP4Backend) + + # Set up auxiliary items + unit_name = "Unit" + test_mp.add_unit(unit=unit_name) + indexset_name = "Indexset" + scenario.init_set(name=indexset_name) + key = "foo" + scenario.add_set(name=indexset_name, key=key) + + # Test creating a parameter with a comment and key conversion to list[str] + name = "Parameter" + value = 3.141592653589793238462643383279502884197169399375105820 + comment = "Comment" + scenario.init_par(name=name, idx_sets=[indexset_name]) + backend._add_data_to_parameter( + s=scenario, name=name, key=key, value=value, unit=unit_name, comment=comment + ) + parameter = backend.index[scenario].optimization.parameters.get(name=name) + assert parameter.data == { + indexset_name: [key], + "values": [value], + "units": [unit_name], + } + + +@pytest.mark.ixmp4 +def test__get_set_data(test_mp: Platform) -> None: + scenario = Scenario(test_mp, "model", "scenario", version="new") + + # Ensure we're handling the correct type of backend + backend = test_mp._backend + assert isinstance(backend, IXMP4Backend) + + indexset_name = "Indexset" + scenario.init_set(name=indexset_name) + table_name = "Table" + scenario.init_set(name=table_name, idx_sets=[indexset_name]) + + # Test getting empty data (other cases tested in integration tests) + indexset_data = backend._get_set_data(s=scenario, name=indexset_name) + # We can assume this return type for Indexsets + pd.testing.assert_series_equal(cast(pd.Series, indexset_data), pd.Series([])) + table_data = backend._get_set_data(s=scenario, name=table_name) + # We can assume this return type for Tables + pd.testing.assert_frame_equal( + cast(pd.DataFrame, table_data), pd.DataFrame(columns=[indexset_name]) + ) + + +def test__ensure_filters_values_are_lists() -> None: + filters = {"foo": [1, 2], "bar": 3} + expected = {"foo": [1, 2], "bar": [3]} + _ensure_filters_values_are_lists(filters=filters) + assert filters == expected + + +def test__align_dtypes_for_filters() -> None: + # This leads to dtypes 'int64' and 'object' + df = pd.DataFrame({"foo": [1, 2, 3], "bar": ["baz", "foo", "bar"]}) + # Filters' types are determined based on first item assuming one type per key + filters: dict[str, list[Any]] = {"foo": [1.0, 2.0, 3.0], "bar": [1, 2, 3]} + expected = {"foo": [1, 2, 3], "bar": ["1", "2", "3"]} + _align_dtypes_for_filters(filters=filters, data=df) + + assert filters == expected From eb4c628ade260a2989a45da2ef2779c015428920 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 15 Apr 2025 13:12:43 +0200 Subject: [PATCH 55/72] Apply new markers to test suite as needed --- ixmp/tests/backend/test_base.py | 2 + ixmp/tests/backend/test_jdbc.py | 8 + ixmp/tests/core/test_meta.py | 612 +++++++++++++++-------------- ixmp/tests/core/test_platform.py | 33 +- ixmp/tests/core/test_scenario.py | 56 ++- ixmp/tests/core/test_timeseries.py | 29 ++ ixmp/tests/report/test_operator.py | 6 + ixmp/tests/report/test_reporter.py | 2 +- ixmp/tests/test_cli.py | 17 +- ixmp/tests/test_integration.py | 4 + ixmp/tests/test_model.py | 8 + ixmp/tests/test_util.py | 7 + 12 files changed, 467 insertions(+), 317 deletions(-) diff --git a/ixmp/tests/backend/test_base.py b/ixmp/tests/backend/test_base.py index b84920bea..e12253353 100644 --- a/ixmp/tests/backend/test_base.py +++ b/ixmp/tests/backend/test_base.py @@ -135,6 +135,8 @@ def test_cache_invalidate(self, test_mp): backend.cache_invalidate(ts, "par", "baz", dict(x=["x1", "x2"], y=["y1", "y2"])) + # TODO IXMP4Backend needs to handle ._cache correctly + @pytest.mark.jdbc def test_del_ts(self, test_mp, request): """Test CachingBackend.del_ts().""" # Since CachingBackend is an abstract class, test it via JDBCBackend diff --git a/ixmp/tests/backend/test_jdbc.py b/ixmp/tests/backend/test_jdbc.py index 3642ee80c..b0e722b9e 100644 --- a/ixmp/tests/backend/test_jdbc.py +++ b/ixmp/tests/backend/test_jdbc.py @@ -61,6 +61,8 @@ def test_close_default_logging(test_mp_f, capfd): assert captured.out == "" +# NOTE IXMP4Backend's close_db() is a noop +@pytest.mark.jdbc def test_close_increased_logging(test_mp_f, capfd): """Platform.close_db() doesn't throw needless exceptions.""" # Use the session-scoped fixture to avoid affecting other tests in this file @@ -217,6 +219,8 @@ def test_gc(self, monkeypatch, be): be.gc() +# TODO IXMP4Backend needs to handle change_scalar() correctly +@pytest.mark.jdbc def test_exceptions(test_mp): """Ensure that Python exceptions are raised for some actions.""" s = ixmp.Scenario(test_mp, "model name", "scenario name", "new") @@ -366,6 +370,8 @@ def exception_verbose_true(): ixmp.backend.jdbc._EXCEPTION_VERBOSE = tmp # Restore value +# FIMXE This raises a RunNotFound on IXMP4Backend +@pytest.mark.jdbc def test_verbose_exception(test_mp, exception_verbose_true): # Exception stack trace is logged for debugging with pytest.raises(RuntimeError) as exc_info: @@ -654,6 +660,8 @@ def test_reload_cycle( memory_usage("shutdown") +# TODO Not yet implemented by IXMP4Backend +@pytest.mark.jdbc def test_docs(test_mp, request): scen = make_dantzig(test_mp, request=request) # test model docs diff --git a/ixmp/tests/core/test_meta.py b/ixmp/tests/core/test_meta.py index 7d91f7825..dda74d564 100644 --- a/ixmp/tests/core/test_meta.py +++ b/ixmp/tests/core/test_meta.py @@ -3,6 +3,7 @@ # behaviour they actually test. import copy +from typing import Any, Generator import pytest @@ -25,7 +26,7 @@ @pytest.fixture(scope="function") -def mp(test_mp_f): +def mp(test_mp_f: ixmp.Platform) -> Generator[ixmp.Platform, Any, None]: """A test Platform. The platform contains one time series with the "dantizg" model & scenario name from @@ -41,311 +42,312 @@ def mp(test_mp_f): yield test_mp_f -@pytest.mark.parametrize("meta", META_ENTRIES) -def test_set_meta_missing_argument(mp, meta): - with pytest.raises(ValueError): - mp.set_meta(meta) - with pytest.raises(ValueError): - mp.set_meta(meta, model=DANTZIG["model"], version=0) - with pytest.raises(ValueError): - mp.set_meta(meta, scenario=DANTZIG["scenario"], version=0) - - -@pytest.mark.parametrize("meta", META_ENTRIES) -def test_set_get_meta(mp, meta): - """Assert that storing+retrieving meta yields expected values.""" - mp.set_meta(meta, model=DANTZIG["model"]) - obs = mp.get_meta(model=DANTZIG["model"]) - assert obs == meta - - -@pytest.mark.parametrize("meta", META_ENTRIES) -def test_unique_meta(mp, meta): - """ - When setting a meta category on two distinct levels, a uniqueness error is - expected. - """ - scenario = ixmp.Scenario(mp, **DANTZIG, version="new") - scenario.commit("save dummy scenario") - mp.set_meta(meta, model=DANTZIG["model"]) - expected = ( - r"The meta category .* is already used at another level: " - r"model canning problem, scenario null, version null" - ) - with pytest.raises(Exception, match=expected): - mp.set_meta(meta, **DANTZIG, version=scenario.version) - scen = ixmp.Scenario(mp, **DANTZIG) - with pytest.raises(Exception, match=expected): +# TODO IXMP4Backend needs to handle meta data +@pytest.mark.jdbc +class TestMeta: + @pytest.mark.parametrize("meta", META_ENTRIES) + def test_set_meta_missing_argument(self, mp: ixmp.Platform, meta) -> None: + with pytest.raises(ValueError): + mp.set_meta(meta) + with pytest.raises(ValueError): + mp.set_meta(meta, model=DANTZIG["model"], version=0) + with pytest.raises(ValueError): + mp.set_meta(meta, scenario=DANTZIG["scenario"], version=0) + + @pytest.mark.parametrize("meta", META_ENTRIES) + def test_set_get_meta(self, mp: ixmp.Platform, meta) -> None: + """Assert that storing+retrieving meta yields expected values.""" + mp.set_meta(meta, model=DANTZIG["model"]) + obs = mp.get_meta(model=DANTZIG["model"]) + assert obs == meta + + @pytest.mark.parametrize("meta", META_ENTRIES) + def test_unique_meta(self, mp: ixmp.Platform, meta) -> None: + """ + When setting a meta category on two distinct levels, a uniqueness error is + expected. + """ + scenario = ixmp.Scenario(mp, **DANTZIG, version="new") + scenario.commit("save dummy scenario") + mp.set_meta(meta, model=DANTZIG["model"]) + expected = ( + r"The meta category .* is already used at another level: " + r"model canning problem, scenario null, version null" + ) + with pytest.raises(Exception, match=expected): + mp.set_meta(meta, **DANTZIG, version=scenario.version) + scen = ixmp.Scenario(mp, **DANTZIG) + with pytest.raises(Exception, match=expected): + scen.set_meta(meta) + # changing the category value type of an entry should also raise an error + meta = {"sample_entry": 3} + mp.set_meta(meta, **DANTZIG) + meta["sample_entry"] = "test-string" + expected = ( + r"The meta category .* is already used at another level: " + r"model canning problem, scenario standard, version null" + ) + with pytest.raises(Exception, match=expected): + mp.set_meta(meta, **DANTZIG, version=scenario.version) + + @pytest.mark.parametrize("meta", META_ENTRIES) + def test_set_get_meta_equals(self, mp: ixmp.Platform, meta) -> None: + initial_meta = mp.get_meta(scenario=DANTZIG["scenario"]) + mp.set_meta(meta, model=DANTZIG["model"]) + obs_meta = mp.get_meta(scenario=DANTZIG["scenario"]) + assert obs_meta == initial_meta + + @pytest.mark.parametrize("meta", META_ENTRIES) + def test_unique_meta_model_scenario(self, mp: ixmp.Platform, meta) -> None: + """ + When setting a meta key for a Model, it shouldn't be possible to set it + for a Model+Scenario then. + """ + mp.set_meta(meta, model=DANTZIG["model"]) + expected = r"The meta category .* is already used at another level: " + with pytest.raises(Exception, match=expected): + mp.set_meta(meta, **DANTZIG) + + # Setting this meta category on a new model should fail too + dantzig2 = { + "model": "canning problem 2", + "scenario": "standard", + } + mp.add_model_name(dantzig2["model"]) + expected = r"The meta category .* is already used at another level: " + with pytest.raises(Exception, match=expected): + mp.set_meta(meta, **dantzig2) + + @pytest.mark.parametrize("meta", META_ENTRIES) + def test_get_meta_strict(self, mp: ixmp.Platform, meta) -> None: + """ + Set meta indicators on several model/scenario/version levels and test + the 'strict' parameter of get_meta(). + """ + # set meta on various levels + model_meta = { + "model_int": 3, + "model_string": "string_value", + "model_bool": False, + } + scenario_meta = { + "scenario_int": 3, + "scenario_string": "string_value", + "scenario_bool": False, + } + meta2 = { + "sample_int2": 3, + "sample_string2": "string_value2", + "sample_bool2": False, + } + meta3 = { + "sample_int3": 3, + "sample_string3": "string_value3", + "sample_bool3": False, + "mixed3": ["string", 0.01, 2, True], + } + meta_scen = { + "sample_int4": 3, + "sample_string4": "string_value4", + "sample_bool4": False, + "mixed4": ["string", 0.01, 2, True], + } + scenario2 = "standard 2" + model2 = "canning problem 2" + mp.add_scenario_name(scenario2) + mp.add_model_name(model2) + dantzig2 = { + "model": model2, + "scenario": "standard", + } + dantzig3 = { + "model": model2, + "scenario": scenario2, + } + mp.set_meta(model_meta, model=DANTZIG["model"]) + mp.set_meta(scenario_meta, scenario=DANTZIG["scenario"]) + mp.set_meta(meta, **DANTZIG) + mp.set_meta(meta2, **dantzig2) + mp.set_meta(meta3, **dantzig3) + scen = ixmp.Scenario(mp, **DANTZIG, version="new") + scen.commit("save dummy scenario") + scen.set_meta(meta_scen) + + # Retrieve and validate meta indicators + # model + obs1 = mp.get_meta(model=DANTZIG["model"]) + assert obs1 == model_meta + # scenario + obs2 = mp.get_meta(scenario=DANTZIG["scenario"], strict=True) + assert obs2 == scenario_meta + # model+scenario + obs3 = mp.get_meta(**DANTZIG) + exp3 = copy.copy(meta) + exp3.update(model_meta) + exp3.update(scenario_meta) + assert obs3 == exp3 + # model+scenario, strict + obs3_strict = mp.get_meta(**DANTZIG, strict=True) + assert obs3_strict == meta + assert obs3 != obs3_strict + + # second model+scenario combination + obs4 = mp.get_meta(**dantzig2) + exp4 = copy.copy(meta2) + exp4.update(scenario_meta) + assert obs4 == exp4 + # second model+scenario combination, strict + obs4_strict = mp.get_meta(**dantzig2, strict=True) + assert obs4_strict == meta2 + assert obs4 != obs4_strict + + # second model+scenario combination + obs5 = mp.get_meta(**dantzig3) + exp5 = copy.copy(meta3) + assert obs5 == exp5 + + # model+scenario+version + obs6 = mp.get_meta(**DANTZIG, version=scen.version) + exp6 = copy.copy(meta_scen) + exp6.update(meta) + exp6.update(model_meta) + exp6.update(scenario_meta) + assert obs6 == exp6 + obs6_strict = mp.get_meta( + DANTZIG["model"], DANTZIG["scenario"], scen.version, strict=True + ) + assert obs6_strict == meta_scen + + @pytest.mark.parametrize("meta", META_ENTRIES) + def test_unique_meta_scenario(self, mp: ixmp.Platform, meta) -> None: + """ + When setting a meta key on a specific Scenario run, setting the same key + on an higher level (Model or Model+Scenario) should fail. + """ + scen = ixmp.Scenario(mp, **DANTZIG) scen.set_meta(meta) - # changing the category value type of an entry should also raise an error - meta = {"sample_entry": 3} - mp.set_meta(meta, **DANTZIG) - meta["sample_entry"] = "test-string" - expected = ( - r"The meta category .* is already used at another level: " - r"model canning problem, scenario standard, version null" - ) - with pytest.raises(Exception, match=expected): - mp.set_meta(meta, **DANTZIG, version=scenario.version) - - -@pytest.mark.parametrize("meta", META_ENTRIES) -def test_set_get_meta_equals(mp, meta): - initial_meta = mp.get_meta(scenario=DANTZIG["scenario"]) - mp.set_meta(meta, model=DANTZIG["model"]) - obs_meta = mp.get_meta(scenario=DANTZIG["scenario"]) - assert obs_meta == initial_meta - - -@pytest.mark.parametrize("meta", META_ENTRIES) -def test_unique_meta_model_scenario(mp, meta): - """ - When setting a meta key for a Model, it shouldn't be possible to set it - for a Model+Scenario then. - """ - mp.set_meta(meta, model=DANTZIG["model"]) - expected = r"The meta category .* is already used at another level: " - with pytest.raises(Exception, match=expected): + # add a second scenario and verify that setting+getting Meta works + scen2 = ixmp.Scenario(mp, **DANTZIG, version="new") + scen2.commit("save dummy scenario") + scen2.set_meta(meta) + assert scen2.get_meta() == scen.get_meta() + + expected = ( + r"The meta category .* is already used at another level: " + r"model canning problem, scenario standard, " + ) + with pytest.raises(Exception, match=expected): + mp.set_meta(meta, **DANTZIG) + with pytest.raises(Exception, match=expected): + mp.set_meta(meta, model=DANTZIG["model"]) + + def test_meta_partial_overwrite(self, mp: ixmp.Platform) -> None: + meta1 = { + "sample_string": 3.0, + "another_string": "string_value", + "sample_bool": False, + } + meta2 = { + "sample_string": 5.0, + "yet_another_string": "hello", + "sample_bool": True, + } + scen = ixmp.Scenario(mp, **DANTZIG) + scen.set_meta(meta1) + scen.set_meta(meta2) + expected = copy.copy(meta1) + expected.update(meta2) + obs = scen.get_meta() + assert obs == expected + + def test_remove_meta(self, mp: ixmp.Platform) -> None: + meta = {"sample_int": 3.0, "another_string": "string_value"} + remove_key = "another_string" mp.set_meta(meta, **DANTZIG) + mp.remove_meta(remove_key, **DANTZIG) + expected = copy.copy(meta) + del expected[remove_key] + obs = mp.get_meta(**DANTZIG) + assert expected == obs + + def test_remove_invalid_meta(self, mp: ixmp.Platform) -> None: + """ + Removing nonexisting meta entries or None shouldn't result in any meta + being removed. Providing None should give a ValueError. + """ + mp.set_meta(SAMPLE_META, **DANTZIG) + with pytest.raises(ValueError): + mp.remove_meta(None, **DANTZIG) + mp.remove_meta("nonexisting_category", **DANTZIG) + mp.remove_meta([], **DANTZIG) + obs = mp.get_meta(**DANTZIG) + assert obs == SAMPLE_META + + def test_set_and_remove_meta_scenario(self, mp: ixmp.Platform) -> None: + """ + Test partial overwriting and meta deletion on scenario level. + """ + meta1 = {"sample_string": 3.0, "another_string": "string_value"} + meta2 = {"sample_string": 5.0, "yet_another_string": "hello"} + remove_key = "another_string" + + scen = ixmp.Scenario(mp, **DANTZIG) + scen.set_meta(meta1) + scen.set_meta(meta2) + expected = copy.copy(meta1) + expected.update(meta2) + obs = scen.get_meta() + assert expected == obs + + scen.remove_meta(remove_key) + del expected[remove_key] + obs = scen.get_meta() + assert obs == expected + + def test_scenario_delete_meta_warning(self, mp: ixmp.Platform) -> None: + """ + Scenario.delete_meta works but raises a deprecation warning. + + This test can be removed once Scenario.delete_meta is removed. + """ + scen = ixmp.Scenario(mp, **DANTZIG) + meta = {"sample_int": 3, "sample_string": "string_value"} + remove_key = "sample_string" - # Setting this meta category on a new model should fail too - dantzig2 = { - "model": "canning problem 2", - "scenario": "standard", - } - mp.add_model_name(dantzig2["model"]) - expected = r"The meta category .* is already used at another level: " - with pytest.raises(Exception, match=expected): - mp.set_meta(meta, **dantzig2) - - -@pytest.mark.parametrize("meta", META_ENTRIES) -def test_get_meta_strict(mp, meta): - """ - Set meta indicators on several model/scenario/version levels and test - the 'strict' parameter of get_meta(). - """ - # set meta on various levels - model_meta = {"model_int": 3, "model_string": "string_value", "model_bool": False} - scenario_meta = { - "scenario_int": 3, - "scenario_string": "string_value", - "scenario_bool": False, - } - meta2 = {"sample_int2": 3, "sample_string2": "string_value2", "sample_bool2": False} - meta3 = { - "sample_int3": 3, - "sample_string3": "string_value3", - "sample_bool3": False, - "mixed3": ["string", 0.01, 2, True], - } - meta_scen = { - "sample_int4": 3, - "sample_string4": "string_value4", - "sample_bool4": False, - "mixed4": ["string", 0.01, 2, True], - } - scenario2 = "standard 2" - model2 = "canning problem 2" - mp.add_scenario_name(scenario2) - mp.add_model_name(model2) - dantzig2 = { - "model": model2, - "scenario": "standard", - } - dantzig3 = { - "model": model2, - "scenario": scenario2, - } - mp.set_meta(model_meta, model=DANTZIG["model"]) - mp.set_meta(scenario_meta, scenario=DANTZIG["scenario"]) - mp.set_meta(meta, **DANTZIG) - mp.set_meta(meta2, **dantzig2) - mp.set_meta(meta3, **dantzig3) - scen = ixmp.Scenario(mp, **DANTZIG, version="new") - scen.commit("save dummy scenario") - scen.set_meta(meta_scen) - - # Retrieve and validate meta indicators - # model - obs1 = mp.get_meta(model=DANTZIG["model"]) - assert obs1 == model_meta - # scenario - obs2 = mp.get_meta(scenario=DANTZIG["scenario"], strict=True) - assert obs2 == scenario_meta - # model+scenario - obs3 = mp.get_meta(**DANTZIG) - exp3 = copy.copy(meta) - exp3.update(model_meta) - exp3.update(scenario_meta) - assert obs3 == exp3 - # model+scenario, strict - obs3_strict = mp.get_meta(**DANTZIG, strict=True) - assert obs3_strict == meta - assert obs3 != obs3_strict - - # second model+scenario combination - obs4 = mp.get_meta(**dantzig2) - exp4 = copy.copy(meta2) - exp4.update(scenario_meta) - assert obs4 == exp4 - # second model+scenario combination, strict - obs4_strict = mp.get_meta(**dantzig2, strict=True) - assert obs4_strict == meta2 - assert obs4 != obs4_strict - - # second model+scenario combination - obs5 = mp.get_meta(**dantzig3) - exp5 = copy.copy(meta3) - assert obs5 == exp5 - - # model+scenario+version - obs6 = mp.get_meta(**DANTZIG, version=scen.version) - exp6 = copy.copy(meta_scen) - exp6.update(meta) - exp6.update(model_meta) - exp6.update(scenario_meta) - assert obs6 == exp6 - obs6_strict = mp.get_meta( - DANTZIG["model"], DANTZIG["scenario"], scen.version, strict=True - ) - assert obs6_strict == meta_scen - - -@pytest.mark.parametrize("meta", META_ENTRIES) -def test_unique_meta_scenario(mp, meta): - """ - When setting a meta key on a specific Scenario run, setting the same key - on an higher level (Model or Model+Scenario) should fail. - """ - scen = ixmp.Scenario(mp, **DANTZIG) - scen.set_meta(meta) - # add a second scenario and verify that setting+getting Meta works - scen2 = ixmp.Scenario(mp, **DANTZIG, version="new") - scen2.commit("save dummy scenario") - scen2.set_meta(meta) - assert scen2.get_meta() == scen.get_meta() - - expected = ( - r"The meta category .* is already used at another level: " - r"model canning problem, scenario standard, " - ) - with pytest.raises(Exception, match=expected): - mp.set_meta(meta, **DANTZIG) - with pytest.raises(Exception, match=expected): + scen.set_meta(meta) + with pytest.warns(DeprecationWarning): + scen.delete_meta(remove_key) + expected = copy.copy(meta) + del expected[remove_key] + obs = scen.get_meta() + assert obs == expected + + def test_meta_arguments(self, mp: ixmp.Platform) -> None: + """Set scenario meta with key-value arguments""" + meta = {"sample_int": 3} + scen = ixmp.Scenario(mp, **DANTZIG) + scen.set_meta(meta) + # add a second scenario and verify that setting Meta for it works + scen2 = ixmp.Scenario(mp, **DANTZIG, version="new") + scen2.commit("save dummy scenario") + scen2.set_meta(*meta.popitem()) + assert scen.get_meta() == scen2.get_meta() + + def test_update_meta_lists(self, mp: ixmp.Platform) -> None: + """Set metadata categories having list/array values.""" + SAMPLE_META = {"list_category": ["a", "b", "c"]} + mp.set_meta(SAMPLE_META, model=DANTZIG["model"]) + obs = mp.get_meta(model=DANTZIG["model"]) + assert obs == SAMPLE_META + # try updating meta + SAMPLE_META = {"list_category": ["a", "e", "f"]} + mp.set_meta(SAMPLE_META, model=DANTZIG["model"]) + obs = mp.get_meta(model=DANTZIG["model"]) + assert obs == SAMPLE_META + + def test_meta_mixed_list(self, mp: ixmp.Platform) -> None: + """Set metadata categories having list/array values.""" + meta = {"mixed_category": ["string", 0.01, True]} mp.set_meta(meta, model=DANTZIG["model"]) - - -def test_meta_partial_overwrite(mp): - meta1 = { - "sample_string": 3.0, - "another_string": "string_value", - "sample_bool": False, - } - meta2 = {"sample_string": 5.0, "yet_another_string": "hello", "sample_bool": True} - scen = ixmp.Scenario(mp, **DANTZIG) - scen.set_meta(meta1) - scen.set_meta(meta2) - expected = copy.copy(meta1) - expected.update(meta2) - obs = scen.get_meta() - assert obs == expected - - -def test_remove_meta(mp): - meta = {"sample_int": 3.0, "another_string": "string_value"} - remove_key = "another_string" - mp.set_meta(meta, **DANTZIG) - mp.remove_meta(remove_key, **DANTZIG) - expected = copy.copy(meta) - del expected[remove_key] - obs = mp.get_meta(**DANTZIG) - assert expected == obs - - -def test_remove_invalid_meta(mp): - """ - Removing nonexisting meta entries or None shouldn't result in any meta - being removed. Providing None should give a ValueError. - """ - mp.set_meta(SAMPLE_META, **DANTZIG) - with pytest.raises(ValueError): - mp.remove_meta(None, **DANTZIG) - mp.remove_meta("nonexisting_category", **DANTZIG) - mp.remove_meta([], **DANTZIG) - obs = mp.get_meta(**DANTZIG) - assert obs == SAMPLE_META - - -def test_set_and_remove_meta_scenario(mp): - """ - Test partial overwriting and meta deletion on scenario level. - """ - meta1 = {"sample_string": 3.0, "another_string": "string_value"} - meta2 = {"sample_string": 5.0, "yet_another_string": "hello"} - remove_key = "another_string" - - scen = ixmp.Scenario(mp, **DANTZIG) - scen.set_meta(meta1) - scen.set_meta(meta2) - expected = copy.copy(meta1) - expected.update(meta2) - obs = scen.get_meta() - assert expected == obs - - scen.remove_meta(remove_key) - del expected[remove_key] - obs = scen.get_meta() - assert obs == expected - - -def test_scenario_delete_meta_warning(mp): - """ - Scenario.delete_meta works but raises a deprecation warning. - - This test can be removed once Scenario.delete_meta is removed. - """ - scen = ixmp.Scenario(mp, **DANTZIG) - meta = {"sample_int": 3, "sample_string": "string_value"} - remove_key = "sample_string" - - scen.set_meta(meta) - with pytest.warns(DeprecationWarning): - scen.delete_meta(remove_key) - expected = copy.copy(meta) - del expected[remove_key] - obs = scen.get_meta() - assert obs == expected - - -def test_meta_arguments(mp): - """Set scenario meta with key-value arguments""" - meta = {"sample_int": 3} - scen = ixmp.Scenario(mp, **DANTZIG) - scen.set_meta(meta) - # add a second scenario and verify that setting Meta for it works - scen2 = ixmp.Scenario(mp, **DANTZIG, version="new") - scen2.commit("save dummy scenario") - scen2.set_meta(*meta.popitem()) - assert scen.get_meta() == scen2.get_meta() - - -def test_update_meta_lists(mp): - """Set metadata categories having list/array values.""" - SAMPLE_META = {"list_category": ["a", "b", "c"]} - mp.set_meta(SAMPLE_META, model=DANTZIG["model"]) - obs = mp.get_meta(model=DANTZIG["model"]) - assert obs == SAMPLE_META - # try updating meta - SAMPLE_META = {"list_category": ["a", "e", "f"]} - mp.set_meta(SAMPLE_META, model=DANTZIG["model"]) - obs = mp.get_meta(model=DANTZIG["model"]) - assert obs == SAMPLE_META - - -def test_meta_mixed_list(mp): - """Set metadata categories having list/array values.""" - meta = {"mixed_category": ["string", 0.01, True]} - mp.set_meta(meta, model=DANTZIG["model"]) - obs = mp.get_meta(model=DANTZIG["model"]) - assert obs == meta + obs = mp.get_meta(model=DANTZIG["model"]) + assert obs == meta diff --git a/ixmp/tests/core/test_platform.py b/ixmp/tests/core/test_platform.py index 23af1fce4..fea41ebd5 100644 --- a/ixmp/tests/core/test_platform.py +++ b/ixmp/tests/core/test_platform.py @@ -39,18 +39,19 @@ def test_init0(self) -> None: mp = ixmp.Platform() assert "local" == mp.name + # NOTE Can't use 'backend' due to duplicate parametrization @pytest.mark.parametrize( - "backend, backend_args", + "_backend, backend_args", ( ("jdbc", dict(driver="hsqldb", url="jdbc:hsqldb:mem:TestPlatform")), pytest.param("ixmp4", dict(), marks=min_ixmp4_version), ), ) def test_init1( - self, backend: Literal["jdbc", "ixmp4"], backend_args: dict[str, str] + self, _backend: Literal["jdbc", "ixmp4"], backend_args: dict[str, str] ) -> None: # Platform can be instantiated - ixmp.Platform(backend=backend, **backend_args) + ixmp.Platform(backend=_backend, **backend_args) def test_getattr(self, test_mp: ixmp.Platform) -> None: """Test __getattr__.""" @@ -70,6 +71,8 @@ def log_level_mp(test_mp: ixmp.Platform) -> Generator[ixmp.Platform, Any, None]: test_mp.set_log_level(tmp) +# TODO Not sure why 'NOTSET' fails on IXMP4Backend +@pytest.mark.jdbc @pytest.mark.parametrize( "level, exc", [ @@ -98,6 +101,8 @@ def test_scenario_list(mp: ixmp.Platform) -> None: assert scenario[0] == "Hitchhiker" +# TODO Not sure why this fails on IXMP4Backend +@pytest.mark.jdbc def test_export_timeseries_data(mp: ixmp.Platform, tmp_path: Path) -> None: path = tmp_path / "export.csv" mp.export_timeseries_data(path, model="Douglas Adams", unit="???", region="World") @@ -126,6 +131,8 @@ def test_export_ts_wrong_params(test_mp: ixmp.Platform, tmp_path: Path) -> None: ) +# TODO Not sure why this fails on IXMP4Backend +@pytest.mark.jdbc def test_export_ts_of_all_runs(mp: ixmp.Platform, tmp_path: Path) -> None: """Export timeseries of all runs.""" path = tmp_path / "export.csv" @@ -140,11 +147,15 @@ def test_export_ts_of_all_runs(mp: ixmp.Platform, tmp_path: Path) -> None: mp.export_timeseries_data( path, unit="???", region="World", default=True, export_all_runs=True ) + # Find the one default version of the expected scenario + exp_default_version = mp.scenario_list( + model=models["h2g2"]["model"], scen=models["h2g2"]["scenario"] + )["version"].item() obs = pd.read_csv(path, index_col=False, header=0) exp = ( DATA[0] - .assign(**models["h2g2"], version=2, subannual="Year", meta=0) + .assign(**models["h2g2"], version=exp_default_version, subannual="Year", meta=0) .rename(columns=lambda c: c.upper()) .reindex(columns=FIELDS["write_file"]) ) @@ -156,7 +167,9 @@ def test_export_ts_of_all_runs(mp: ixmp.Platform, tmp_path: Path) -> None: path, unit="???", region="World", default=False, export_all_runs=True ) obs = pd.read_csv(path, index_col=False, header=0) - assert 4 == len(obs) + + # NOTE It seems there are more lines here, maybe due to an increased number of runs? + assert 6 == len(obs) def test_export_timeseries_data_empty(mp: ixmp.Platform, tmp_path: Path) -> None: @@ -192,6 +205,8 @@ def test_regions(test_mp: ixmp.Platform) -> None: assert all([list(obs.loc[0]) == ["World", None, "World", "common"]]) +# NOTE IXMP4(Backend) doesn't store `Parent`; instead, it returns the region name itself +@pytest.mark.jdbc def test_add_region(test_mp: ixmp.Platform) -> None: # Region can be added test_mp.add_region("foo", "bar", "World") @@ -202,6 +217,8 @@ def test_add_region(test_mp: ixmp.Platform) -> None: assert all([list(obs.loc[0]) == ["foo", None, "World", "bar"]]) +# NOTE IXMP4Backend doesn't handle synonyms, it might not ever +@pytest.mark.jdbc def test_add_region_synonym(test_mp: ixmp.Platform) -> None: test_mp.add_region("foo", "bar", "World") test_mp.add_region_synonym("foo2", "foo") @@ -218,6 +235,8 @@ def test_add_region_synonym(test_mp: ixmp.Platform) -> None: assert_frame_equal(obs, exp) +# TODO Not yet implemented on IXMP4Backend +@pytest.mark.jdbc def test_timeslices(test_mp: ixmp.Platform) -> None: timeslices = test_mp.timeslices() obs = timeslices[timeslices.category == "Common"] @@ -227,6 +246,8 @@ def test_timeslices(test_mp: ixmp.Platform) -> None: assert all([list(obs.iloc[0]) == ["Year", "Common", 1.0]]) +# TODO Not yet implemented on IXMP4Backend +@pytest.mark.jdbc def test_add_timeslice(test_mp: ixmp.Platform) -> None: test_mp.add_timeslice("January, 1st", "Days", 1.0 / 366) timeslices = test_mp.timeslices() @@ -237,6 +258,8 @@ def test_add_timeslice(test_mp: ixmp.Platform) -> None: assert all([list(obs.iloc[0]) == ["January, 1st", "Days", 1.0 / 366]]) +# TODO Not yet implemented on IXMP4Backend +@pytest.mark.jdbc def test_add_timeslice_duplicate(caplog, test_mp: ixmp.Platform) -> None: test_mp.add_timeslice("foo_slice", "foo_category", 0.2) diff --git a/ixmp/tests/core/test_scenario.py b/ixmp/tests/core/test_scenario.py index f2b17b1ef..53d09f258 100644 --- a/ixmp/tests/core/test_scenario.py +++ b/ixmp/tests/core/test_scenario.py @@ -45,6 +45,8 @@ class TestScenario: """Tests of :class:`ixmp.Scenario`.""" # Initialize Scenario + # NOTE IXMP4-backed Scenarios start with version = 1 + @pytest.mark.jdbc def test_init(self, test_mp, scen_empty): # Empty scenario has version == 0 assert scen_empty.version == 0 @@ -76,16 +78,26 @@ class Scenario(ixmp.Scenario): ): Scenario(test_mp, model="foo", scenario="bar", cache=False) - def test_default_version(self, mp): + def test_default_version(self, mp: ixmp.Platform) -> None: scen = ixmp.Scenario(mp, **models["dantzig"]) - assert scen.version == 2 + scenario_df = mp.scenario_list( + model=models["dantzig"]["model"], scen=models["dantzig"]["scenario"] + ) + assert len(scenario_df["version"]) == 1 + assert scen.version == scenario_df["version"].item() + # NOTE IXMP4(Backend) doesn't raise the same error/message as expected here + @pytest.mark.jdbc def test_from_url(self, mp, caplog): url = f"ixmp://{mp.name}/Douglas Adams/Hitchhiker" # Default version is loaded scen, mp = ixmp.Scenario.from_url(url) - assert scen.version == 1 + scenario_df = mp.scenario_list( + model=models["h2g2"]["model"], scen=models["h2g2"]["scenario"] + ) + assert len(scenario_df["version"]) == 1 + assert scen.version == scenario_df["version"].item() # Giving an invalid version with errors='raise' raises an exception expected = ( @@ -118,6 +130,8 @@ def test_clone(self, mp): obs = scen2.set("h") npt.assert_array_equal(obs, ["test"]) + # FIXME IXMP4Backend needs to handle change_scalar correctly + @pytest.mark.jdbc def test_clone_edit(self, scen): scen2 = scen.clone(keep_solution=False) scen2.check_out() @@ -131,6 +145,8 @@ def test_clone_edit(self, scen): assert scen2.scalar("f") == {"unit": "USD/km", "value": 95} # Initialize items + # NOTE IXMP4Backend doesn't handle commits yet + @pytest.mark.jdbc def test_init_set(self, scen): """Test ixmp.Scenario.init_set().""" @@ -149,6 +165,9 @@ def test_init_set(self, scen): with pytest.raises(ValueError, match="'foo' already exists"): scen.init_set("foo") + # FIXME IXMP4Backend recognized 'foo' as having no idx_sets, likely due to the + # assumption of all names of all items being unique. We should drop that, it seems. + @pytest.mark.jdbc def test_init_par(self, scen) -> None: scen = scen.clone(keep_solution=False) scen.check_out() @@ -170,6 +189,8 @@ def test_init_scalar(self, scen): scen2.commit("adding a scalar 'g'") # Existence checks + # TODO IXMP4Backend doesn't handle scalars correctly yet + @pytest.mark.jdbc def test_has_par(self, scen): assert scen.has_par("f") assert not scen.has_par("m") @@ -182,6 +203,8 @@ def test_has_var(self, scen): assert scen.has_var("x") assert not scen.has_var("y") + # TODO IXMP4Backend doesn't handle scalars correctly yet + @pytest.mark.jdbc def test_scalar(self, scen): assert scen.scalar("f") == {"unit": "USD/km", "value": 90} @@ -226,6 +249,8 @@ def test_add_par(self, scen, args, kwargs): scen.check_out() scen.add_par(*args, **kwargs) + # TODO IXMP4Backend should support this, I think + @pytest.mark.jdbc def test_add_par2(self, scen): scen = scen.clone(keep_solution=False) scen.check_out() @@ -255,6 +280,8 @@ def test_idx(self, scen): assert scen.idx_sets("d") == ["i", "j"] assert scen.idx_names("d") == ["i", "j"] + # FIXME IXMP4-backed par doesn't have 0 as a key; avoid this hardcoding + @pytest.mark.jdbc def test_par(self, scen): # Parameter data can be retrieved with filters df = scen.par("d", filters={"i": ["seattle"]}) @@ -264,6 +291,8 @@ def test_par(self, scen): with pytest.warns(DeprecationWarning, match="ignored kwargs"): scen.par("d", i=["seattle"]) + # FIXME IXMP4Backend is missing an item, likely the scalar f again + @pytest.mark.jdbc def test_items0(self, scen): # Without filters iterator = scen.items() @@ -294,6 +323,9 @@ def test_items0(self, scen): assert i == 1 + # FIXME For test case 3, IXMP4Backend somehow also lists 'foo' (likely defined in + # the fixture); in test case 2 it's missing f (the scalar) + @pytest.mark.jdbc @pytest.mark.parametrize( "item_type, indexed_by, exp", ( @@ -333,6 +365,8 @@ def test_var(self, scen): # Marginals npt.assert_array_almost_equal(df["mrg"], [0, 0, 0.036]) + # TODO IXMP4Backend is not handling _cache correctly + @pytest.mark.jdbc def test_load_scenario_data(self, mp): """load_scenario_data() caches all data.""" scen = ixmp.Scenario(mp, **models["dantzig"]) @@ -366,6 +400,9 @@ def test_load_scenario_data_clear_cache(self, monkeypatch, mp): scen.load_scenario_data() # I/O + # TODO For IXMP4Backend, this somehow triggers a GAMS-related error, while this test + # should never touch GAMS + @pytest.mark.jdbc def test_excel_io(self, scen, scen_empty, tmp_path, caplog): tmp_path /= "output.xlsx" @@ -448,6 +485,9 @@ def test_excel_io(self, scen, scen_empty, tmp_path, caplog): # Succeeds with add_units=True s.read_excel(tmp_path, add_units=True, init_items=True) + # NOTE IXMP4-backed Scenarios should not call remove_solution() if they don't have + # one + @pytest.mark.jdbc def test_solve(self, tmp_path, scen): from subprocess import run @@ -483,6 +523,8 @@ def test_solve(self, tmp_path, scen): assert "'notapackage'.'(not installed)'" in result.stdout.decode() # Combined tests + # NOTE Not yet implemented on IXMP4Backend + @pytest.mark.jdbc def test_meta(self, mp, test_dict): scen = ixmp.Scenario(mp, **models["dantzig"], version=1) for k, v in test_dict.items(): @@ -513,6 +555,8 @@ def test_meta(self, mp, test_dict): with pytest.raises(ValueError, match="Cannot use value"): scen.set_meta("test_string", complex(1, 1)) + # NOTE Not yet implemented on IXMP4Backend + @pytest.mark.jdbc def test_meta_bulk(self, mp, test_dict): scen = ixmp.Scenario(mp, **models["dantzig"], version=1) scen.set_meta(test_dict) @@ -560,6 +604,9 @@ def test_gh_210(scen_empty): assert all(foo_data.columns == columns) +# NOTE IXMP4(Backend) raises an OptimizationDataValidationError if 'bar' is missing from +# index set 'i' instead of a ValueError +@pytest.mark.jdbc def test_set(scen_empty) -> None: """Test ixmp.Scenario.add_set(), .set(), and .remove_set().""" scen = scen_empty @@ -690,6 +737,9 @@ def test_filter_str(scen_empty): assert_frame_equal(exp[["s", "value"]], obs[["s", "value"]]) +# FIXME Calling scen.solve(change_distance, ...) triggers a bug in the GAMS code +# IXMP4Backend might need to avoid similar calls (which properties exactly?) +@pytest.mark.jdbc def test_solve_callback(test_mp, request): """Test the callback argument to Scenario.solve(). diff --git a/ixmp/tests/core/test_timeseries.py b/ixmp/tests/core/test_timeseries.py index f7af7263b..138aa147d 100644 --- a/ixmp/tests/core/test_timeseries.py +++ b/ixmp/tests/core/test_timeseries.py @@ -132,6 +132,8 @@ def test_default(self, mp, ts): # Original TimeSeries is no longer default assert not ts.is_default() + # NOTE IXMP4Backend doesn't handle commits yet, so run_id is 1 immediately + @pytest.mark.jdbc def test_run_id(self, ts): # New, un-committed TimeSeries has run_id of -1 assert ts.run_id() == -1 @@ -140,6 +142,8 @@ def test_run_id(self, ts): ts.commit("") assert ts.run_id() > 0 and isinstance(ts.run_id(), int) + # NOTE Not yet implemented on IXMP4Backend + @pytest.mark.jdbc def test_last_update(self, ts): # New, un-committed TimeSeries has no last update date assert ts.last_update() is None @@ -174,6 +178,9 @@ def test_discard_changes(self, ts): assert 0 == len(ts.timeseries()) + # FIXME IXMP4Backend seems to handle timeseries() incorrectly; the last call returns + # 0 rows + @pytest.mark.jdbc @pytest.mark.parametrize("format", ["long", "wide"]) def test_get(self, ts, format): data = DATA[0] if format == "long" else wide(DATA[0]) @@ -211,6 +218,9 @@ def test_get_year(self, ts, year_arg): # year filters with integer values are handled correctly (iiasa/ixmp#440) ts.timeseries(year=year_arg) + # FIXME IXMP4Backend seems to handle timeseries() incorrectly; the last call returns + # 0 rows + @pytest.mark.jdbc @pytest.mark.parametrize("format", ["long", "wide"]) def test_edit(self, mp, ts, format): """Tests that data can be overwritten.""" @@ -248,6 +258,9 @@ def test_edit(self, mp, ts, format): exp = wide(exp) assert_frame_equal(exp, ts.timeseries(**args)) + # NOTE IXMP4Backend requires version-specifier or setting a default for the + # corresponding Run since Run doesn't store `version` + @pytest.mark.jdbc @pytest.mark.parametrize("cls", [TimeSeries, Scenario]) def test_edit_with_region_synonyms(self, mp, ts, cls): info = dict(model=ts.model, scenario=ts.scenario) @@ -268,6 +281,8 @@ def test_edit_with_region_synonyms(self, mp, ts, cls): assert_frame_equal(expected(DATA[2050], ts), ts.timeseries()) + # NOTE Not yet implemented on IXMP4Backend + @pytest.mark.jdbc # TODO parametrize format as wide/long @pytest.mark.parametrize( "commit", @@ -315,6 +330,8 @@ def test_remove(self, mp, ts, commit): # Result is empty assert ts.timeseries().empty + # NOTE Not yet implemented on IXMP4Backend + @pytest.mark.jdbc def test_transact_discard(self, caplog, mp, ts): caplog.set_level(logging.INFO, "ixmp.util") @@ -345,6 +362,8 @@ def test_transact_discard(self, caplog, mp, ts): # Geodata + # NOTE Not yet implemented on IXMP4Backend + @pytest.mark.jdbc def test_add_geodata(self, ts): # Empty TimeSeries includes no geodata assert_frame_equal(DATA["geo"].loc[[False, False, False]], ts.get_geodata()) @@ -357,6 +376,8 @@ def test_add_geodata(self, ts): obs = ts.get_geodata().sort_values("year").reset_index(drop=True) assert_frame_equal(DATA["geo"], obs) + # NOTE Not yet implemented on IXMP4Backend + @pytest.mark.jdbc @pytest.mark.parametrize( "rows", [(1,), (1, 2), (0, 1, 2)], ids=["single", "multiple", "all"] ) @@ -424,6 +445,8 @@ def test_add_timeseries_with_extra_col(self, caplog, ts, format): "Dropped extra column(s) ['climate_model'] from data" ] == caplog.messages + # NOTE Not yet implemented on IXMP4Backend + @pytest.mark.jdbc def test_new_timeseries_as_iamc(self, test_mp): # TODO rescue use of subannual= here @@ -458,6 +481,8 @@ def test_new_timeseries_error(self, test_mp): # column `unit` is missing pytest.raises(ValueError, scen.add_timeseries, df) + # NOTE Not yet implemented on IXMP4Backend + @pytest.mark.jdbc def test_new_subannual_timeseries_as_iamc(self, mp): mp.add_timeslice("Summer", "Season", 1.0 / 4) scen = TimeSeries(mp, *models["h2g2"], version="new", annotation="fo") @@ -495,11 +520,15 @@ def test_new_subannual_timeseries_as_iamc(self, mp): # setting False raises an error because subannual data exists pytest.raises(ValueError, scen.timeseries, subannual=False) + # NOTE Not yet implemented on IXMP4Backend + @pytest.mark.jdbc def test_fetch_empty_geodata(self, mp): scen = TimeSeries(mp, *models["h2g2"], version="new", annotation="fo") empty = scen.get_geodata() assert_geodata(empty, DATA["geo"].loc[[False, False, False]]) + # NOTE Not yet implemented on IXMP4Backend + @pytest.mark.jdbc def test_remove_multiple_geodata(self, mp): scen = TimeSeries(mp, *models["h2g2"], version="new", annotation="fo") scen.add_geodata(DATA["geo"]) diff --git a/ixmp/tests/report/test_operator.py b/ixmp/tests/report/test_operator.py index 04ddfc95f..f48b7328b 100644 --- a/ixmp/tests/report/test_operator.py +++ b/ixmp/tests/report/test_operator.py @@ -45,6 +45,8 @@ def test_from_url(test_mp, request) -> None: assert ts.url == result.url +# FIXME Not sure why introducing the Computer fails for IXMP4Backend +@pytest.mark.jdbc def test_get_remove_ts(caplog, test_mp, request) -> None: ts = make_dantzig(test_mp, request=request) @@ -107,6 +109,8 @@ def test_map_as_qty() -> None: assert_qty_equal(exp, result) +# FIXME IXMP4Backend seems to return d with unexpected ordering +@pytest.mark.jdbc def test_update_scenario(caplog, test_mp, request) -> None: scen = make_dantzig(test_mp, request=request) scen.check_out() @@ -161,6 +165,8 @@ def test_update_scenario(caplog, test_mp, request) -> None: assert_frame_equal(scen.par("d"), data) +# FIXME Nots ure why this fails on IXMP4Backend +@pytest.mark.jdbc def test_store_ts(request, caplog, test_mp) -> None: # Computer and target scenario c = Computer() diff --git a/ixmp/tests/report/test_reporter.py b/ixmp/tests/report/test_reporter.py index 22e9cdf35..dd49dc101 100644 --- a/ixmp/tests/report/test_reporter.py +++ b/ixmp/tests/report/test_reporter.py @@ -124,7 +124,7 @@ def test_platform_units(test_mp, caplog, ureg) -> None: scen.add_par("x", x) with assert_logs( - caplog, "x: mixed units ['kg', 'USD/pkm'] discarded", at_level=logging.INFO + caplog, ["x: mixed units", "kg", "USD/pkm", "discarded"], at_level=logging.INFO ): rep.get(x_key) diff --git a/ixmp/tests/test_cli.py b/ixmp/tests/test_cli.py index 982414319..5a17c6746 100644 --- a/ixmp/tests/test_cli.py +++ b/ixmp/tests/test_cli.py @@ -27,6 +27,8 @@ def test_versiontype(): vt.convert("xx", None, None) +# FIXME This should work for IXMP4Backend, I think +@pytest.mark.jdbc def test_main(ixmp_cli, test_mp, tmp_path): # Name of a temporary file that doesn't exist tmp_path /= "temp.properties" @@ -96,7 +98,8 @@ def test_config(ixmp_cli): def test_list(ixmp_cli, test_mp): - cmd = ["list", "--match", "foo"] + # NOTE Some other test may leak scenarios onto test_mp + cmd = ["list", "--match", "no-scenario-named-foo"] # 'list' without specifying a platform/scenario is a UsageError result = ixmp_cli.invoke(cmd) @@ -113,7 +116,7 @@ def test_list(ixmp_cli, test_mp): 0 (model, scenario) combination(s) 0 total scenarios """ - ) + ), result.output def test_platform(ixmp_cli, tmp_path): @@ -121,7 +124,7 @@ def test_platform(ixmp_cli, tmp_path): def call(*args, exit_0=True): result = ixmp_cli.invoke(["platform"] + list(map(str, args))) - assert not exit_0 or result.exit_code == 0 + assert not exit_0 or result.exit_code == 0, result.output return result # The default platform is 'local' @@ -217,6 +220,8 @@ def call(*args, exit_0=True): call("copy", f"p1-{test_specific_name}", f"p3-{test_specific_name}") +# TODO Version 1 for IXMP4Backend returns an empty DF for scen.timeseries() +@pytest.mark.jdbc def test_import_ts(ixmp_cli, test_mp, test_data_path): # Ensure the 'canning problem'/'standard' TimeSeries exists populate_test_platform(test_mp) @@ -326,6 +331,10 @@ def test_excel_io(ixmp_cli, test_mp, tmp_path): assert result.exit_code == 0, result.output +# FIXME IXMP4Backend requires a version to be specified when loading a Scenario +# or a default to be set for the corresponding Run before +# since Run doesn't store `version` +@pytest.mark.jdbc def test_excel_io_filters(ixmp_cli, test_mp, tmp_path): populate_test_platform(test_mp) tmp_path /= "dantzig.xlsx" @@ -381,6 +390,8 @@ def test_show_versions(ixmp_cli): assert result.exit_code == 0, result.output +# FIXME This is again the parameter error for IXMP4Backend and the DantzigModel +@pytest.mark.jdbc def test_solve(ixmp_cli, test_mp): populate_test_platform(test_mp) cmd = [ diff --git a/ixmp/tests/test_integration.py b/ixmp/tests/test_integration.py index caf5d6b88..014a8c26b 100644 --- a/ixmp/tests/test_integration.py +++ b/ixmp/tests/test_integration.py @@ -12,6 +12,8 @@ TS_DF_CLEARED.loc[0, 2005] = np.nan +# FIXME IXMP4Backend seems to return columns with no/wrong names for scen.timeseries() +@pytest.mark.jdbc def test_run_clone(caplog, test_mp, request): caplog.set_level(logging.WARNING) @@ -66,6 +68,8 @@ def test_run_clone(caplog, test_mp, request): ) +# FIXME Fix IXMP4Backend return value for s.var()["lvl"] so that np.isnan() accepts it +@pytest.mark.jdbc def test_run_remove_solution(test_mp, request): # create a new instance of the transport problem and solve it mp = test_mp diff --git a/ixmp/tests/test_model.py b/ixmp/tests/test_model.py index 83259563d..73c3106e2 100644 --- a/ixmp/tests/test_model.py +++ b/ixmp/tests/test_model.py @@ -23,6 +23,9 @@ class M1(Model): M1() +# FIXME IXMP4Backend doesn't handle scenario.change_scalar() correctly: +# We can't just create() it, it might already exist, so then we need update() +@pytest.mark.jdbc def test_model_initialize(test_mp, caplog, request): # Model.initialize runs on an empty Scenario s = make_dantzig(test_mp, request=request) @@ -111,6 +114,9 @@ class TestGAMSModel: def dantzig(self, test_mp, request): yield make_dantzig(test_mp, request=request) + # FIXME The name_ seems to be handled correctly, but IXMP4Backend struggles with + # make_dantzig().clone().solve() somehow. + @pytest.mark.jdbc @pytest.mark.parametrize("char", r'<>"/\|?*') def test_filename_invalid_char(self, dantzig, char): """Model can be solved with invalid character names.""" @@ -122,6 +128,8 @@ def test_filename_invalid_char(self, dantzig, char): # the GAMSModel.model_name attribute, and in turn the GDX file names used. s.solve(name_=name, quiet=True) + # FIXME IXMP4Backend should support this + @pytest.mark.jdbc @pytest.mark.parametrize( "kwargs", [ diff --git a/ixmp/tests/test_util.py b/ixmp/tests/test_util.py index 05f12e37e..29c608f55 100644 --- a/ixmp/tests/test_util.py +++ b/ixmp/tests/test_util.py @@ -72,6 +72,8 @@ def test_diff_identical(test_mp, request): assert exp_name == name and len(df) == N +# FIXME I don't see why IXMP4Backend shouldn't support this, but it's failing. +@pytest.mark.jdbc def test_diff_data(test_mp, request): """diff() when Scenarios contain the same items, but different data.""" scen_a = make_dantzig(test_mp, request=request) @@ -156,6 +158,9 @@ def test_diff_items(test_mp, request): pass # No check of the contents +# TODO IXMP4Backend doesn't handle retrieval of scalars correctly yet; +# but look here for a test case! +@pytest.mark.jdbc def test_discard_on_error(caplog, test_mp, request): caplog.set_level(logging.INFO, "ixmp.util") @@ -293,6 +298,8 @@ def test_format_scenario_list(test_mp_f): ) == util.format_scenario_list(mp, as_url=True) +# IXMP4Backend doesn't have proper commits yet, so these never raise RuntimeErrors +@pytest.mark.jdbc def test_maybe_commit(caplog, test_mp): s = Scenario(test_mp, "maybe_commit", "maybe_commit", version="new") From 5c8004f55236b2b09b20b55129327271df61c68d Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 15 Apr 2025 14:11:53 +0200 Subject: [PATCH 56/72] Install gams version matching gamsapi --- .github/workflows/pytest.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 0fff3445d..ee1e3f4a6 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -30,7 +30,8 @@ jobs: - "3.12" - "3.13" # Latest supported by ixmp gams-version: - - "43.4.1" # First version including a macOS arm64 distribution + # - "43.4.1" # First version including a macOS arm64 distribution + - "48.1.0" # Version matching the current gamsapi installation exclude: # Specific version combinations that are invalid / not to be used From 95d9e1e9a12ef0a865cc380f53b616f20648a79a Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Wed, 16 Apr 2025 13:48:38 +0200 Subject: [PATCH 57/72] Avoid hardcoded test expectation --- ixmp/core/timeseries.py | 1 + ixmp/tests/core/test_platform.py | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ixmp/core/timeseries.py b/ixmp/core/timeseries.py index f80492656..5dc4a7af3 100644 --- a/ixmp/core/timeseries.py +++ b/ixmp/core/timeseries.py @@ -158,6 +158,7 @@ def from_url( ts = cls(platform, **scenario_info) except Exception as e: if errors == "warn": + # FIXME ixmp4 errors might have empty e.args log.warning( f"{e.__class__.__name__}: {e.args[0]}\n" f"when loading {cls.__name__} from url: {repr(url)}" diff --git a/ixmp/tests/core/test_platform.py b/ixmp/tests/core/test_platform.py index fea41ebd5..864cc908e 100644 --- a/ixmp/tests/core/test_platform.py +++ b/ixmp/tests/core/test_platform.py @@ -168,8 +168,11 @@ def test_export_ts_of_all_runs(mp: ixmp.Platform, tmp_path: Path) -> None: ) obs = pd.read_csv(path, index_col=False, header=0) - # NOTE It seems there are more lines here, maybe due to an increased number of runs? - assert 6 == len(obs) + # We only have 1 model, scenario, region, variable, unit, so we expect lines equal + # to number of versions (equals default version number) times number of years: + expected = exp_default_version * len(DATA[0]["year"]) # Usually 4 + + assert expected == len(obs) def test_export_timeseries_data_empty(mp: ixmp.Platform, tmp_path: Path) -> None: From 442ab730bc7ec1fec52c2ad49f01ca0d2f025567 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Wed, 16 Apr 2025 13:49:06 +0200 Subject: [PATCH 58/72] Fix IXMP4Backend.__init__() * Enable repeated calls for in-memory DBs * Remove expectation of un-pickle-able kwargs --- ixmp/backend/ixmp4.py | 56 ++++++++++++++++++++++++++++------------ ixmp/testing/__init__.py | 19 +++++--------- 2 files changed, 46 insertions(+), 29 deletions(-) diff --git a/ixmp/backend/ixmp4.py b/ixmp/backend/ixmp4.py index 768f1d04a..a322ca35c 100644 --- a/ixmp/backend/ixmp4.py +++ b/ixmp/backend/ixmp4.py @@ -4,6 +4,8 @@ from pathlib import Path from typing import Any, Literal, Optional, Union, cast, overload +import ixmp4.conf +import ixmp4.conf.manager import pandas as pd from ixmp4 import DataPoint from ixmp4 import Platform as ixmp4_platform @@ -63,28 +65,50 @@ class IXMP4Backend(CachingBackend): # subclasses of either) index: dict[TimeSeries, "Run"] = {} - def __init__(self, _backend: Optional["ixmp4_backend"] = None) -> None: + # Mapping from platform name to ixmp4 Backend; this enables repeated __init__() + # calls for in-memory DBs + backend_index: dict[str, "ixmp4_backend"] = {} + + def __init__(self, _name: Optional[str] = None) -> None: from ixmp4.conf.base import PlatformInfo from ixmp4.data.backend import SqliteTestBackend - # Create a default backend if None is provided - if not _backend: - log.warning("Falling back to default SqliteBackend 'ixmp4-local'") - sqlite = SqliteTestBackend( - PlatformInfo( - name="ixmp4-local", - dsn=( - "sqlite:///" - f"{Path.home().joinpath('.local', 'share', 'ixmp4', 'databases')}" # noqa: E501 - "/ixmp4-local.sqlite3" - ), - ) + DEFAULT_NAME = "ixmp4-local" + + if not _name: + # Warn about using the default backend + log.warning(f"Falling back to default SqliteBackend '{DEFAULT_NAME}'") + + name = _name if _name else DEFAULT_NAME + + # Create default platform information if None is registered + platform_info = ( + ixmp4.conf.settings.toml.get_platform(key=_name) + if _name + else PlatformInfo( + name=name, + dsn=( + "sqlite:///" + f"{Path.home().joinpath('.local', 'share', 'ixmp4', 'databases')}" + f"/{name}.sqlite3" + ), ) + ) + + # Get the ixmp4 Backend object if one already exists or create and register it + try: + sqlite = self.backend_index[name] + except KeyError: + sqlite = SqliteTestBackend(platform_info) + + # Ensure default DB is setup correctly + # NOTE sqlalchemy catches existing tables to avoid superfluous CREATEs sqlite.setup() - self._backend = _backend if _backend else sqlite + self.backend_index[name] = sqlite # Instantiate and store + self._backend = sqlite self._platform = ixmp4_platform(_backend=self._backend) # def __del__(self) -> None: @@ -108,9 +132,9 @@ def handle_config(cls, args: Sequence, kwargs: MutableMapping) -> dict[str, Any] info: dict[str, Any] = {} try: - info["_backend"] = kwargs["_backend"] + info["_name"] = kwargs["_name"] except KeyError: - raise ValueError(f"Missing key '_backend' for backend=ixmp4; got {kwargs}") + raise ValueError(f"Missing key '_name' for backend=ixmp4; got {kwargs}") return info diff --git a/ixmp/testing/__init__.py b/ixmp/testing/__init__.py index 2996196fe..8fd19fdee 100644 --- a/ixmp/testing/__init__.py +++ b/ixmp/testing/__init__.py @@ -456,17 +456,14 @@ def _platform_fixture( platform_name, backend, "hsqldb", url=f"jdbc:hsqldb:mem:{platform_name}" ) elif backend == "ixmp4": - from ixmp4.conf.base import PlatformInfo - from ixmp4.data.backend import SqliteTestBackend + import ixmp4.conf - # Setup ixmp4 backend and run DB migrations - sqlite = SqliteTestBackend( - PlatformInfo(name=platform_name, dsn="sqlite:///:memory:") - ) - sqlite.setup() + # Make in-memory DB/Platform for ixmp4 + dsn = "sqlite:///:memory:" + ixmp4.conf.settings.toml.add_platform(name=platform_name, dsn=dsn) # Add ixmp4 backend to ixmp platforms - ixmp_config.add_platform(platform_name, backend, _backend=sqlite) + ixmp_config.add_platform(platform_name, backend, _name=platform_name) # Launch Platform mp = Platform(name=platform_name, backend=backend) @@ -483,8 +480,4 @@ def _platform_fixture( ixmp_config.remove_platform(platform_name) if backend == "ixmp4": - assert sqlite # to satisfy type checkers - - # Close DB connection and remove platform - sqlite.close() - sqlite.teardown() + ixmp4.conf.settings.toml.remove_platform(key=platform_name) From ed53038b01fc9e51edb6159a2a6d7709213ab4e4 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Wed, 16 Apr 2025 13:51:02 +0200 Subject: [PATCH 59/72] Avoid ixmp4 imports for Python 3.9 --- ixmp/__init__.py | 7 +++++-- ixmp/backend/io.py | 29 ++++++++++++++++------------- ixmp/testing/data.py | 9 ++++++--- ixmp/util/ixmp4.py | 3 ++- 4 files changed, 29 insertions(+), 19 deletions(-) diff --git a/ixmp/__init__.py b/ixmp/__init__.py index 2d51580aa..7ebd9e653 100644 --- a/ixmp/__init__.py +++ b/ixmp/__init__.py @@ -4,7 +4,6 @@ from ixmp._config import config from ixmp.backend import BACKENDS, IAMC_IDX, ItemType -from ixmp.backend.ixmp4 import IXMP4Backend from ixmp.backend.jdbc import JDBCBackend from ixmp.core.platform import Platform from ixmp.core.scenario import Scenario, TimeSeries @@ -48,7 +47,11 @@ # Register Backends provided by ixmp BACKENDS["jdbc"] = JDBCBackend -BACKENDS["ixmp4"] = IXMP4Backend + +if sys.version_info >= (3, 10): + from ixmp.backend.ixmp4 import IXMP4Backend + + BACKENDS["ixmp4"] = IXMP4Backend # Register Models provided by ixmp MODELS.update( diff --git a/ixmp/backend/io.py b/ixmp/backend/io.py index d3ce8ee33..105ff36bd 100644 --- a/ixmp/backend/io.py +++ b/ixmp/backend/io.py @@ -1,29 +1,32 @@ import logging +import sys from collections import deque from collections.abc import Iterable from pathlib import Path from typing import Literal, Optional, TypeVar, Union, cast -import gams.transfer as gt import pandas as pd -from ixmp4.core import Run -from ixmp4.core.optimization.base import Lister -from ixmp4.core.optimization.equation import Equation -from ixmp4.core.optimization.indexset import IndexSet, IndexSetRepository -from ixmp4.core.optimization.parameter import Parameter -from ixmp4.core.optimization.scalar import Scalar -from ixmp4.core.optimization.table import Table -from ixmp4.core.optimization.variable import Variable -from ixmp4.data.abstract.optimization.equation import Equation as AbstractEquation -from ixmp4.data.abstract.optimization.variable import Variable as AbstractVariable from ixmp.util import as_str_list, maybe_check_out, maybe_commit from ixmp.util.ixmp4 import ContainerData from . import ItemType -# Type variable that can be any one of these 6 types, but not a union of 2+ of them -Item4 = TypeVar("Item4", Equation, IndexSet, Parameter, Scalar, Table, Variable) +if sys.version_info >= (3, 10): + import gams.transfer as gt + from ixmp4.core import Run + from ixmp4.core.optimization.base import Lister + from ixmp4.core.optimization.equation import Equation + from ixmp4.core.optimization.indexset import IndexSet, IndexSetRepository + from ixmp4.core.optimization.parameter import Parameter + from ixmp4.core.optimization.scalar import Scalar + from ixmp4.core.optimization.table import Table + from ixmp4.core.optimization.variable import Variable + from ixmp4.data.abstract.optimization.equation import Equation as AbstractEquation + from ixmp4.data.abstract.optimization.variable import Variable as AbstractVariable + + # Type variable that can be any one of these 6 types, but not a union of 2+ of them + Item4 = TypeVar("Item4", Equation, IndexSet, Parameter, Scalar, Table, Variable) log = logging.getLogger(__name__) diff --git a/ixmp/testing/data.py b/ixmp/testing/data.py index 888e1208d..b328272e1 100644 --- a/ixmp/testing/data.py +++ b/ixmp/testing/data.py @@ -1,4 +1,5 @@ # Methods are in alphabetical order +import sys from itertools import product from math import ceil from typing import TYPE_CHECKING, Any, Optional @@ -11,7 +12,6 @@ from ixmp import Platform, Scenario, TimeSeries from ixmp.backend import IAMC_IDX -from ixmp.backend.ixmp4 import IXMP4Backend if TYPE_CHECKING: from typing import TypedDict @@ -215,8 +215,11 @@ def make_dantzig( -------- .DantzigModel """ - if isinstance(mp._backend, IXMP4Backend): - _add_required_units(mp=mp) + if sys.version_info >= (3, 10): + from ixmp.backend.ixmp4 import IXMP4Backend + + if isinstance(mp._backend, IXMP4Backend): + _add_required_units(mp=mp) # Add custom units and region for time series data try: diff --git a/ixmp/util/ixmp4.py b/ixmp/util/ixmp4.py index 927126480..80d2d7399 100644 --- a/ixmp/util/ixmp4.py +++ b/ixmp/util/ixmp4.py @@ -1,8 +1,9 @@ -# TODO Import this from typing when dropping Python 3.11 from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Literal, Optional, Union import pandas as pd + +# TODO Import this from typing when dropping Python 3.11 from typing_extensions import TypedDict if TYPE_CHECKING: From 6342d9d596d58bc3bdaa729b36966cfa7e58e8ff Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Wed, 16 Apr 2025 14:11:55 +0200 Subject: [PATCH 60/72] Allow test platforms to already exist --- ixmp/testing/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ixmp/testing/__init__.py b/ixmp/testing/__init__.py index 8fd19fdee..52f4a6e67 100644 --- a/ixmp/testing/__init__.py +++ b/ixmp/testing/__init__.py @@ -457,10 +457,14 @@ def _platform_fixture( ) elif backend == "ixmp4": import ixmp4.conf + from ixmp4.core.exceptions import PlatformNotUnique - # Make in-memory DB/Platform for ixmp4 + # Add to or get from ixmp4 an in-memory DB/Platform dsn = "sqlite:///:memory:" - ixmp4.conf.settings.toml.add_platform(name=platform_name, dsn=dsn) + try: + ixmp4.conf.settings.toml.add_platform(name=platform_name, dsn=dsn) + except PlatformNotUnique: + pass # Add ixmp4 backend to ixmp platforms ixmp_config.add_platform(platform_name, backend, _name=platform_name) From 0055bc82ad0a91b10e38ffd8bf613e475d8e2373 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Wed, 16 Apr 2025 14:12:20 +0200 Subject: [PATCH 61/72] Fix outdated test expectation --- ixmp/tests/core/test_timeseries.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ixmp/tests/core/test_timeseries.py b/ixmp/tests/core/test_timeseries.py index 138aa147d..3dc12fc97 100644 --- a/ixmp/tests/core/test_timeseries.py +++ b/ixmp/tests/core/test_timeseries.py @@ -441,9 +441,7 @@ def test_add_timeseries_with_extra_col(self, caplog, ts, format): ts.add_timeseries(data) # TODO: add check that warning message is displayed ts.commit("") - assert [ - "Dropped extra column(s) ['climate_model'] from data" - ] == caplog.messages + assert "Dropped extra column(s) ['climate_model'] from data" in caplog.messages # NOTE Not yet implemented on IXMP4Backend @pytest.mark.jdbc From 05b950aae08621aea9b0c0a774c69adb1a585d55 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Wed, 16 Apr 2025 14:52:20 +0200 Subject: [PATCH 62/72] Handle conditional import for ixmp4 IO functions --- ixmp/backend/io.py | 426 --------------------------- ixmp/backend/ixmp4.py | 3 +- ixmp/backend/ixmp4_io.py | 428 ++++++++++++++++++++++++++++ ixmp/tests/backend/test_io.py | 422 --------------------------- ixmp/tests/backend/test_ixmp4_io.py | 424 +++++++++++++++++++++++++++ 5 files changed, 853 insertions(+), 850 deletions(-) create mode 100644 ixmp/backend/ixmp4_io.py create mode 100644 ixmp/tests/backend/test_ixmp4_io.py diff --git a/ixmp/backend/io.py b/ixmp/backend/io.py index 105ff36bd..92b6e2c1a 100644 --- a/ixmp/backend/io.py +++ b/ixmp/backend/io.py @@ -1,34 +1,12 @@ import logging -import sys from collections import deque -from collections.abc import Iterable -from pathlib import Path -from typing import Literal, Optional, TypeVar, Union, cast import pandas as pd from ixmp.util import as_str_list, maybe_check_out, maybe_commit -from ixmp.util.ixmp4 import ContainerData from . import ItemType -if sys.version_info >= (3, 10): - import gams.transfer as gt - from ixmp4.core import Run - from ixmp4.core.optimization.base import Lister - from ixmp4.core.optimization.equation import Equation - from ixmp4.core.optimization.indexset import IndexSet, IndexSetRepository - from ixmp4.core.optimization.parameter import Parameter - from ixmp4.core.optimization.scalar import Scalar - from ixmp4.core.optimization.table import Table - from ixmp4.core.optimization.variable import Variable - from ixmp4.data.abstract.optimization.equation import Equation as AbstractEquation - from ixmp4.data.abstract.optimization.variable import Variable as AbstractVariable - - # Type variable that can be any one of these 6 types, but not a union of 2+ of them - Item4 = TypeVar("Item4", Equation, IndexSet, Parameter, Scalar, Table, Variable) - - log = logging.getLogger(__name__) #: Maximum number of rows supported by the Excel file format. See @@ -335,407 +313,3 @@ def parse_item_sheets(name): maybe_commit(s, commit_steps, f"Loaded {ix_type} {repr(name)} from {path}") maybe_commit(s, not commit_steps, f"Import from {path}") - - -def _domain(item: Item4) -> Optional[list[str]]: - """Return domain for `item`. - - For IndexSets and Scalars, this is :obj:`None`. - - For all others, this is `item.indexsets`. - """ - if isinstance(item, (IndexSet, Scalar)): - return None - else: - return item.indexsets - - -def _records( - item: Item4, -) -> Union[ - float, - Union[list[float], list[int], list[str]], - dict[str, Union[list[float], list[int], list[str]]], - None, -]: - """Return records for `item`. - - For Scalars, this is `item.value`. - - For empty other items, this is :obj:`None`. - - For IndexSets and Tables, this is `item.data`. - - For Parameters, this is `item.data` without the column 'units'. - - For Equations and Variables, this is `item.data` - without the columns 'levels' and 'marginals'. - """ - if isinstance(item, Scalar): - return item.value - elif not len(item.data): - return None - - if isinstance(item, (IndexSet, Table)): - return item.data - - result = item.data - - # Pop items not to be stored - for name in { - Equation: ["levels", "marginals"], - Parameter: ["units"], - Variable: ["levels", "marginals"], - }[type(item)]: - result.pop(name) - - return result - - -def _ensure_correct_item_order(items: list[Item4], repo: Lister) -> list[Item4]: - """Reorder items to ensure the GDX file is written correctly. - - gamsapi stores all unique elements of all items in a single list internally. If item - A adds elements that item B shares, item B is not adding them again. If B's elements - are then requested, they are returned starting with those in A in their order, - disrupting expectations. - """ - if isinstance(repo, IndexSetRepository): - # Move all indexsets called 'type_*' to the end of the list - for type_indexset in list( - filter(lambda item: item.name.startswith("type_"), items) - ): - items.remove(type_indexset) - items.append(type_indexset) - - return items - - -def _align_records_and_domain( - item: Item4, records: dict[str, Union[list[float], list[int], list[str]]] -) -> dict[str, Union[list[float], list[int], list[str]]]: - """Align the order of `records.keys()` with domain of `item`.""" - # This function will only be called for these types - assert isinstance(item, (Table, Parameter, Variable, Equation)) - - # `records` may contain keys that are column_names, but not indexsets - domain_order = item.column_names or item.indexsets - - if not domain_order: - # This could happen for Variables or Equations - return records - else: - # Parameters contain values that we want to keep - if isinstance(item, Parameter): - domain_order.append("values") - - return {key: records[key] for key in domain_order} - - -def _update_item_in_container(container: gt.Container, item: ContainerData) -> None: - """Update `item` in `container`. - - Note that this overwrites existing records of `item`. - """ - # NOTE This is not updating the comment - - kwargs = dict(name=item.name, domain=item.domain, records=item.records) - - if item.kind in ("IndexSet", "Table"): - container.addSet(**kwargs) - elif item.kind in ("Scalar", "Parameter"): - container.addParameter(**kwargs) - elif item.kind == "Equation": - # NOTE This was hardcoded in ixmp_source the same way; not sure how the - # 'type' in the container affects anything - kwargs["type"] = "E" - container.addEquation(**kwargs) - else: # Variable - container.addVariable(**kwargs) - - -def _add_items_to_container( - container: gt.Container, items: list[ContainerData] -) -> None: - """Add or update `items` to/in `container`.""" - if not items: - return # Nothing to be done - - for item in items: - # Identify the corresponding gams.transfer class - klass = { - "IndexSet": gt.Set, - "Parameter": gt.Parameter, - "Scalar": gt.Parameter, - "Table": gt.Set, - "Variable": gt.Variable, - "Equation": gt.Equation, - }[item.kind] - - # NOTE A container may already contain some data from the MESSAGE-specific setup - # This seems to imply a UniqueConstraint over all item names, though. - if not container.hasSymbols(item.name): - # NOTE This was hardcoded in ixmp_source the same way; not sure how the - # 'type' in the container affects anything - kwargs = {"type": "E"} if item.kind == "Equation" else {} - - # Create an item instance linked to the container - klass( - container=container, - name=item.name, - domain=item.domain, - records=item.records, - # Optional, but must be str - description=item.docs or "", - **kwargs, - ) - else: - _update_item_in_container(container=container, item=item) - - -def _convert_ixmp4_items_to_containerdata(items: list[Item4]) -> list[ContainerData]: - """Convert list of ixmp4 `items` to ContainerData.""" - if not items: - return [] # Nothing to be done - - # Identify the ixmp4 class name - kind = cast( - Literal["Scalar", "IndexSet", "Table", "Parameter", "Equation", "Variable"], - type(items[0]).__name__, - ) - - container_items: list[ContainerData] = [] - - for item in items: - records = _records(item=item) - # The order of keys in 'domain' is used by gamsapi to overwrite the order of - # keys in 'records', so make sure they are aligned: - # NOTE ixmp4 ensures that all _records.keys() are in _domain and vice-versa - if isinstance(records, dict): - records = _align_records_and_domain(item=item, records=records) - - container_items.append( - ContainerData( - name=item.name, - kind=kind, - records=records, - domain=_domain(item), - docs=item.docs, - ) - ) - - return container_items - - -def _record_versions(container: gt.Container, packages: list[str]) -> None: - """Store Python package versions as set elements to be written to GDX. - - The values are stored in a 2-dimensional set named ``ixmp_version``, where the - first element is the package name, and the second is its version according to - :func:`importlib.metadata.version`). If the package is not installed, the - string "(not installed)" is stored. - """ - from importlib.metadata import PackageNotFoundError, version - - name = "ixmp_version" - - # Each tuple consists of (package_name, package_version) - versions: list[tuple[str, str]] = [] - - # Handle each identified package - for package in packages: - try: - # Retrieve the version; replace characters not supported by GAMS - package_version = version(package).replace(".", "-") - except PackageNotFoundError: - package_version = "(not installed)" # Not installed - - versions.append((package, package_version)) - - # Add Set to the container - container.addSet(name=name, domain=["*", "*"], records=versions) - - -def write_run_to_gdx( - run: Run, - file_name: Path, - container_data: list[ContainerData], - record_version_packages: list[str], - include_variables_and_equations: bool = False, -) -> None: - """Write data from the `run` to a GDX file at `file_name` via a GAMS Container. - - Parameters - ---------- - run : :class:`ixmp4.core.Run` - The Run in which the Scenario data is stored. - file_name: :obj:`Path` - The location at which the GDX file should be written. - container_data : list of :class:`ixmp.util.ixmp4.ContainerData`. - Data that should also be written to the GDX file - in the form of ContainerData. - record_version_packages : list of str - A list of package names for which versions will be stored in the GDX file. - include_variables_and_equations: bool, optional - Flag to indicate whether Variabels and Equations should be written to the GDX - file. - Default: :obj:`False`. - """ - - # TODO How to deal with [*] Set? That seems to be handled by GAMS automatically. - - # Define the container - container = gt.Container() - - repository: list[Lister] = [ - run.optimization.indexsets, - run.optimization.scalars, - run.optimization.tables, - run.optimization.parameters, - run.optimization.variables, - run.optimization.equations, - ] - idx = slice(None) if include_variables_and_equations else slice(-2) - for r in repository[idx]: - # Reorder items if necessary for GAMS to successfully read the GDX - ixmp4_items = _ensure_correct_item_order(items=r.list(), repo=r) - - # Convert ixmp4 items to ContainerData to streamline adding to container - container_items = _convert_ixmp4_items_to_containerdata(items=ixmp4_items) - - _add_items_to_container(container, container_items) - - # Add additional data *after* the required items to avoid confusing GAMS' internal - # Unique Element List - _add_items_to_container(container, container_data) - - _record_versions(container=container, packages=record_version_packages) - - container.write(write_to=file_name) - - -# NOTE since we currently only read Variables and Equations, this function only covers -# these cases -def _set_columns_to_read_from_records( - item: Union[AbstractVariable, AbstractEquation], -) -> list[str]: - """Gather all columns for `item` to read from GDX records.""" - # Prepare columns to select from container.data - # DF also includes lower, upper, scale - columns = ["levels", "marginals"] - item_columns = item.column_names or item.indexsets - if item_columns: - item_columns.extend(columns) - - return item_columns if item_columns else columns - - -# NOTE not sure we only need equations and variables; if we need others, abstracting one -# function for reading would not be as easy, since we might need different details, so -# I'm keeping them separate for now - - -def _read_variables_to_run( - container: gt.Container, run: Run, variables: Iterable[AbstractVariable] -) -> None: - """Read `variables` from `container` and store them in `run`.""" - for variable in variables: - columns_of_interest = _set_columns_to_read_from_records(item=variable) - - records = pd.DataFrame(container.data[variable.name].records) - - # Avoid touching the DB for empty data - if not records.empty: - # NOTE Overwriting the columns changes GAMS' "level" and "marginal" to the - # ixmp4-expected "levels" and "marginals" - # NOTE This assumes that the order the columns are defined in the gams code - # is equal to the order in item.column_names and item.indexsets - records.columns = pd.Index( - columns_of_interest + ["lower", "upper", "scale"] - ) - run.backend.optimization.variables.add_data( - variable_id=variable.id, data=records[columns_of_interest] - ) - - -def _read_equations_to_run( - container: gt.Container, run: Run, equations: Iterable[AbstractEquation] -) -> None: - """Read `equations` from `container` and store them in `run`.""" - for equation in equations: - columns_of_interest = _set_columns_to_read_from_records(item=equation) - - records = pd.DataFrame(container.data[equation.name].records) - - # Avoid touching the DB for empty data - if not records.empty: - # NOTE Overwriting the columns changes GAMS' "level" and "marginal" to the - # ixmp4-expected "levels" and "marginals" - # NOTE This assumes that the order the columns are defined in the gams code - # is equal to the order in item.column_names and item.indexsets - records.columns = pd.Index( - columns_of_interest + ["lower", "upper", "scale"] - ) - run.backend.optimization.equations.add_data( - equation_id=equation.id, data=records[columns_of_interest] - ) - - -def read_gdx_to_run( - run: Run, - result_file: Path, - equ_list: list[str], - var_list: list[str], - comment: str, - check_solution: bool, -) -> None: - """Read data from `result_file` to `run` via a GAMS Container. - - Parameters - ---------- - run : :class:`ixmp4.core.Run` - The Run in which the Scenario data is to be stored. - result_file: :obj:`Path` - The location from which the GDX file should be read. - equ_list : list of str - Names of Equations to read from `result_file`. - var_list : list of str - Names of Variables to read from `result_file`. - comment : str - Unused by ixmp4. - A comment to store when adding the data to `run`. - check_solution : bool - Unused by ixmp4. - A flag to indicate whether the solution should be checked - (for consistency, maybe?). - """ - # Warn about unused parameters - if comment: - log.warning(f"Ignoring comment {comment} for now; unused by ixmp4!") - if check_solution: - log.warning( - f"Ignoring check_solution={check_solution} for now; unused by ixmp4!" - ) - - # Create a GAMS Container from the `result_file` - container = gt.Container(load_from=result_file) - - # Load requested Variables and read them to `run` - # NOTE This handles empty `var_list`, too, - # which is not necessary as long as any Variables are required in message_ix - variables = ( - run.backend.optimization.variables.list(run_id=run.id, name__in=var_list) - if len(var_list) - else run.backend.optimization.variables.list(run_id=run.id) - ) - _read_variables_to_run(container=container, run=run, variables=variables) - - # Load requested Equations and read them to `run` - # NOTE This handles empty `equ_list`, too, - # which is not necessary as long as any Equations are required in message_ix - equations = ( - run.backend.optimization.equations.list(run_id=run.id, name__in=equ_list) - if len(equ_list) - else run.backend.optimization.equations.list(run_id=run.id) - ) - _read_equations_to_run(container=container, run=run, equations=equations) diff --git a/ixmp/backend/ixmp4.py b/ixmp/backend/ixmp4.py index a322ca35c..b477a8d25 100644 --- a/ixmp/backend/ixmp4.py +++ b/ixmp/backend/ixmp4.py @@ -5,7 +5,6 @@ from typing import Any, Literal, Optional, Union, cast, overload import ixmp4.conf -import ixmp4.conf.manager import pandas as pd from ixmp4 import DataPoint from ixmp4 import Platform as ixmp4_platform @@ -23,7 +22,7 @@ from ixmp.backend import ItemType from ixmp.backend.base import CachingBackend, ReadKwargs, WriteKwargs -from ixmp.backend.io import read_gdx_to_run, write_run_to_gdx +from ixmp.backend.ixmp4_io import read_gdx_to_run, write_run_to_gdx from ixmp.core.platform import Platform from ixmp.core.scenario import Scenario from ixmp.core.timeseries import TimeSeries diff --git a/ixmp/backend/ixmp4_io.py b/ixmp/backend/ixmp4_io.py new file mode 100644 index 000000000..5cbd47f4b --- /dev/null +++ b/ixmp/backend/ixmp4_io.py @@ -0,0 +1,428 @@ +import logging +from collections.abc import Iterable +from pathlib import Path +from typing import Literal, Optional, TypeVar, Union, cast + +import gams.transfer as gt +import pandas as pd +from ixmp4.core import Run +from ixmp4.core.optimization.base import Lister +from ixmp4.core.optimization.equation import Equation +from ixmp4.core.optimization.indexset import IndexSet, IndexSetRepository +from ixmp4.core.optimization.parameter import Parameter +from ixmp4.core.optimization.scalar import Scalar +from ixmp4.core.optimization.table import Table +from ixmp4.core.optimization.variable import Variable +from ixmp4.data.abstract.optimization.equation import Equation as AbstractEquation +from ixmp4.data.abstract.optimization.variable import Variable as AbstractVariable + +from ixmp.util.ixmp4 import ContainerData + +log = logging.getLogger(__name__) + +# Type variable that can be any one of these 6 types, but not a union of 2+ of them +Item4 = TypeVar("Item4", Equation, IndexSet, Parameter, Scalar, Table, Variable) + + +def _domain(item: Item4) -> Optional[list[str]]: + """Return domain for `item`. + + For IndexSets and Scalars, this is :obj:`None`. + + For all others, this is `item.indexsets`. + """ + if isinstance(item, (IndexSet, Scalar)): + return None + else: + return item.indexsets + + +def _records( + item: Item4, +) -> Union[ + float, + Union[list[float], list[int], list[str]], + dict[str, Union[list[float], list[int], list[str]]], + None, +]: + """Return records for `item`. + + For Scalars, this is `item.value`. + + For empty other items, this is :obj:`None`. + + For IndexSets and Tables, this is `item.data`. + + For Parameters, this is `item.data` without the column 'units'. + + For Equations and Variables, this is `item.data` + without the columns 'levels' and 'marginals'. + """ + if isinstance(item, Scalar): + return item.value + elif not len(item.data): + return None + + if isinstance(item, (IndexSet, Table)): + return item.data + + result = item.data + + # Pop items not to be stored + for name in { + Equation: ["levels", "marginals"], + Parameter: ["units"], + Variable: ["levels", "marginals"], + }[type(item)]: + result.pop(name) + + return result + + +def _ensure_correct_item_order(items: list[Item4], repo: Lister) -> list[Item4]: + """Reorder items to ensure the GDX file is written correctly. + + gamsapi stores all unique elements of all items in a single list internally. If item + A adds elements that item B shares, item B is not adding them again. If B's elements + are then requested, they are returned starting with those in A in their order, + disrupting expectations. + """ + if isinstance(repo, IndexSetRepository): + # Move all indexsets called 'type_*' to the end of the list + for type_indexset in list( + filter(lambda item: item.name.startswith("type_"), items) + ): + items.remove(type_indexset) + items.append(type_indexset) + + return items + + +def _align_records_and_domain( + item: Item4, records: dict[str, Union[list[float], list[int], list[str]]] +) -> dict[str, Union[list[float], list[int], list[str]]]: + """Align the order of `records.keys()` with domain of `item`.""" + # This function will only be called for these types + assert isinstance(item, (Table, Parameter, Variable, Equation)) + + # `records` may contain keys that are column_names, but not indexsets + domain_order = item.column_names or item.indexsets + + if not domain_order: + # This could happen for Variables or Equations + return records + else: + # Parameters contain values that we want to keep + if isinstance(item, Parameter): + domain_order.append("values") + + return {key: records[key] for key in domain_order} + + +def _update_item_in_container(container: gt.Container, item: ContainerData) -> None: + """Update `item` in `container`. + + Note that this overwrites existing records of `item`. + """ + # NOTE This is not updating the comment + + kwargs = dict(name=item.name, domain=item.domain, records=item.records) + + if item.kind in ("IndexSet", "Table"): + container.addSet(**kwargs) + elif item.kind in ("Scalar", "Parameter"): + container.addParameter(**kwargs) + elif item.kind == "Equation": + # NOTE This was hardcoded in ixmp_source the same way; not sure how the + # 'type' in the container affects anything + kwargs["type"] = "E" + container.addEquation(**kwargs) + else: # Variable + container.addVariable(**kwargs) + + +def _add_items_to_container( + container: gt.Container, items: list[ContainerData] +) -> None: + """Add or update `items` to/in `container`.""" + if not items: + return # Nothing to be done + + for item in items: + # Identify the corresponding gams.transfer class + klass = { + "IndexSet": gt.Set, + "Parameter": gt.Parameter, + "Scalar": gt.Parameter, + "Table": gt.Set, + "Variable": gt.Variable, + "Equation": gt.Equation, + }[item.kind] + + # NOTE A container may already contain some data from the MESSAGE-specific setup + # This seems to imply a UniqueConstraint over all item names, though. + if not container.hasSymbols(item.name): + # NOTE This was hardcoded in ixmp_source the same way; not sure how the + # 'type' in the container affects anything + kwargs = {"type": "E"} if item.kind == "Equation" else {} + + # Create an item instance linked to the container + klass( + container=container, + name=item.name, + domain=item.domain, + records=item.records, + # Optional, but must be str + description=item.docs or "", + **kwargs, + ) + else: + _update_item_in_container(container=container, item=item) + + +def _convert_ixmp4_items_to_containerdata(items: list[Item4]) -> list[ContainerData]: + """Convert list of ixmp4 `items` to ContainerData.""" + if not items: + return [] # Nothing to be done + + # Identify the ixmp4 class name + kind = cast( + Literal["Scalar", "IndexSet", "Table", "Parameter", "Equation", "Variable"], + type(items[0]).__name__, + ) + + container_items: list[ContainerData] = [] + + for item in items: + records = _records(item=item) + # The order of keys in 'domain' is used by gamsapi to overwrite the order of + # keys in 'records', so make sure they are aligned: + # NOTE ixmp4 ensures that all _records.keys() are in _domain and vice-versa + if isinstance(records, dict): + records = _align_records_and_domain(item=item, records=records) + + container_items.append( + ContainerData( + name=item.name, + kind=kind, + records=records, + domain=_domain(item), + docs=item.docs, + ) + ) + + return container_items + + +def _record_versions(container: gt.Container, packages: list[str]) -> None: + """Store Python package versions as set elements to be written to GDX. + + The values are stored in a 2-dimensional set named ``ixmp_version``, where the + first element is the package name, and the second is its version according to + :func:`importlib.metadata.version`). If the package is not installed, the + string "(not installed)" is stored. + """ + from importlib.metadata import PackageNotFoundError, version + + name = "ixmp_version" + + # Each tuple consists of (package_name, package_version) + versions: list[tuple[str, str]] = [] + + # Handle each identified package + for package in packages: + try: + # Retrieve the version; replace characters not supported by GAMS + package_version = version(package).replace(".", "-") + except PackageNotFoundError: + package_version = "(not installed)" # Not installed + + versions.append((package, package_version)) + + # Add Set to the container + container.addSet(name=name, domain=["*", "*"], records=versions) + + +def write_run_to_gdx( + run: Run, + file_name: Path, + container_data: list[ContainerData], + record_version_packages: list[str], + include_variables_and_equations: bool = False, +) -> None: + """Write data from the `run` to a GDX file at `file_name` via a GAMS Container. + + Parameters + ---------- + run : :class:`ixmp4.core.Run` + The Run in which the Scenario data is stored. + file_name: :obj:`Path` + The location at which the GDX file should be written. + container_data : list of :class:`ixmp.util.ixmp4.ContainerData`. + Data that should also be written to the GDX file + in the form of ContainerData. + record_version_packages : list of str + A list of package names for which versions will be stored in the GDX file. + include_variables_and_equations: bool, optional + Flag to indicate whether Variabels and Equations should be written to the GDX + file. + Default: :obj:`False`. + """ + + # TODO How to deal with [*] Set? That seems to be handled by GAMS automatically. + + # Define the container + container = gt.Container() + + repository: list[Lister] = [ + run.optimization.indexsets, + run.optimization.scalars, + run.optimization.tables, + run.optimization.parameters, + run.optimization.variables, + run.optimization.equations, + ] + idx = slice(None) if include_variables_and_equations else slice(-2) + for r in repository[idx]: + # Reorder items if necessary for GAMS to successfully read the GDX + ixmp4_items = _ensure_correct_item_order(items=r.list(), repo=r) + + # Convert ixmp4 items to ContainerData to streamline adding to container + container_items = _convert_ixmp4_items_to_containerdata(items=ixmp4_items) + + _add_items_to_container(container, container_items) + + # Add additional data *after* the required items to avoid confusing GAMS' internal + # Unique Element List + _add_items_to_container(container, container_data) + + _record_versions(container=container, packages=record_version_packages) + + container.write(write_to=file_name) + + +# NOTE since we currently only read Variables and Equations, this function only covers +# these cases +def _set_columns_to_read_from_records( + item: Union[AbstractVariable, AbstractEquation], +) -> list[str]: + """Gather all columns for `item` to read from GDX records.""" + # Prepare columns to select from container.data + # DF also includes lower, upper, scale + columns = ["levels", "marginals"] + item_columns = item.column_names or item.indexsets + if item_columns: + item_columns.extend(columns) + + return item_columns if item_columns else columns + + +# NOTE not sure we only need equations and variables; if we need others, abstracting one +# function for reading would not be as easy, since we might need different details, so +# I'm keeping them separate for now + + +def _read_variables_to_run( + container: gt.Container, run: Run, variables: Iterable[AbstractVariable] +) -> None: + """Read `variables` from `container` and store them in `run`.""" + for variable in variables: + columns_of_interest = _set_columns_to_read_from_records(item=variable) + + records = pd.DataFrame(container.data[variable.name].records) + + # Avoid touching the DB for empty data + if not records.empty: + # NOTE Overwriting the columns changes GAMS' "level" and "marginal" to the + # ixmp4-expected "levels" and "marginals" + # NOTE This assumes that the order the columns are defined in the gams code + # is equal to the order in item.column_names and item.indexsets + records.columns = pd.Index( + columns_of_interest + ["lower", "upper", "scale"] + ) + run.backend.optimization.variables.add_data( + variable_id=variable.id, data=records[columns_of_interest] + ) + + +def _read_equations_to_run( + container: gt.Container, run: Run, equations: Iterable[AbstractEquation] +) -> None: + """Read `equations` from `container` and store them in `run`.""" + for equation in equations: + columns_of_interest = _set_columns_to_read_from_records(item=equation) + + records = pd.DataFrame(container.data[equation.name].records) + + # Avoid touching the DB for empty data + if not records.empty: + # NOTE Overwriting the columns changes GAMS' "level" and "marginal" to the + # ixmp4-expected "levels" and "marginals" + # NOTE This assumes that the order the columns are defined in the gams code + # is equal to the order in item.column_names and item.indexsets + records.columns = pd.Index( + columns_of_interest + ["lower", "upper", "scale"] + ) + run.backend.optimization.equations.add_data( + equation_id=equation.id, data=records[columns_of_interest] + ) + + +def read_gdx_to_run( + run: Run, + result_file: Path, + equ_list: list[str], + var_list: list[str], + comment: str, + check_solution: bool, +) -> None: + """Read data from `result_file` to `run` via a GAMS Container. + + Parameters + ---------- + run : :class:`ixmp4.core.Run` + The Run in which the Scenario data is to be stored. + result_file: :obj:`Path` + The location from which the GDX file should be read. + equ_list : list of str + Names of Equations to read from `result_file`. + var_list : list of str + Names of Variables to read from `result_file`. + comment : str + Unused by ixmp4. + A comment to store when adding the data to `run`. + check_solution : bool + Unused by ixmp4. + A flag to indicate whether the solution should be checked + (for consistency, maybe?). + """ + # Warn about unused parameters + if comment: + log.warning(f"Ignoring comment {comment} for now; unused by ixmp4!") + if check_solution: + log.warning( + f"Ignoring check_solution={check_solution} for now; unused by ixmp4!" + ) + + # Create a GAMS Container from the `result_file` + container = gt.Container(load_from=result_file) + + # Load requested Variables and read them to `run` + # NOTE This handles empty `var_list`, too, + # which is not necessary as long as any Variables are required in message_ix + variables = ( + run.backend.optimization.variables.list(run_id=run.id, name__in=var_list) + if len(var_list) + else run.backend.optimization.variables.list(run_id=run.id) + ) + _read_variables_to_run(container=container, run=run, variables=variables) + + # Load requested Equations and read them to `run` + # NOTE This handles empty `equ_list`, too, + # which is not necessary as long as any Equations are required in message_ix + equations = ( + run.backend.optimization.equations.list(run_id=run.id, name__in=equ_list) + if len(equ_list) + else run.backend.optimization.equations.list(run_id=run.id) + ) + _read_equations_to_run(container=container, run=run, equations=equations) diff --git a/ixmp/tests/backend/test_io.py b/ixmp/tests/backend/test_io.py index 394e2185f..f3b113150 100644 --- a/ixmp/tests/backend/test_io.py +++ b/ixmp/tests/backend/test_io.py @@ -1,28 +1,7 @@ from pathlib import Path -from typing import Union - -import gams.transfer as gt -import pytest import ixmp -from ixmp.backend.io import ( - _add_items_to_container, - _align_records_and_domain, - _convert_ixmp4_items_to_containerdata, - _domain, - _ensure_correct_item_order, - _read_equations_to_run, - _read_variables_to_run, - _record_versions, - _records, - _set_columns_to_read_from_records, - _update_item_in_container, - read_gdx_to_run, - write_run_to_gdx, -) -from ixmp.backend.ixmp4 import IXMP4Backend from ixmp.testing import add_random_model_data, models -from ixmp.util.ixmp4 import ContainerData def test_read_excel_big(test_mp: ixmp.Platform, tmp_path: Path) -> None: @@ -48,404 +27,3 @@ def test_read_excel_big(test_mp: ixmp.Platform, tmp_path: Path) -> None: scen_empty.read_excel(tmp_path) assert len(scen_empty.par("random_par")) == 25 - - -# TODO Should all tests handling scenarios get `request` and use unique scenario names? -# TODO Should we make the run-retrieval its own fixture? - - -@pytest.mark.ixmp4 -def test__domain(test_mp: ixmp.Platform) -> None: - scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") - - # TODO Remove after proper parametrization - assert isinstance(test_mp._backend, IXMP4Backend) - - run = test_mp._backend.index[scen] - indexset = run.optimization.indexsets.create("Indexset") - table = run.optimization.tables.create( - "Table", constrained_to_indexsets=[indexset.name] - ) - - assert _domain(indexset) is None - assert _domain(table) == [indexset.name] - - -@pytest.mark.ixmp4 -def test__records(test_mp: ixmp.Platform) -> None: - scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") - - test_mp.add_unit("unit") - - # TODO Remove after proper parametrization - assert isinstance(test_mp._backend, IXMP4Backend) - - run = test_mp._backend.index[scen] - - # Test records of a Scalar - scalar = run.optimization.scalars.create("Scalar", 3.14, "unit") - assert _records(scalar) == scalar.value - - # Test records of an empty Table - indexset = run.optimization.indexsets.create("Indexset") - table = run.optimization.tables.create( - "Table", constrained_to_indexsets=[indexset.name] - ) - assert _records(table) is None - - # Test records of an IndexSet - indexset.add(["foo", "bar"]) - assert _records(indexset) == indexset.data - - # Test records of a Parameter - parameter = run.optimization.parameters.create( - "Parameter", constrained_to_indexsets=[indexset.name] - ) - parameter.add( - {indexset.name: ["foo", "bar"], "values": [1, 2], "units": ["unit"] * 2} - ) - expected = parameter.data.copy() - expected.pop("units") - assert _records(parameter) == expected - - -@pytest.mark.ixmp4 -def test__ensure_correct_item_order(test_mp: ixmp.Platform) -> None: - scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") - - test_mp.add_unit("unit") - - # TODO Remove after proper parametrization - assert isinstance(test_mp._backend, IXMP4Backend) - - run = test_mp._backend.index[scen] - - years = run.optimization.indexsets.create("years") - nodes = run.optimization.indexsets.create("nodes") - type_years = run.optimization.indexsets.create("type_years") - type_nodes = run.optimization.indexsets.create("type_nodes") - - assert _ensure_correct_item_order( - items=[type_years, years, type_nodes, nodes], repo=run.optimization.indexsets - ) == [years, nodes, type_years, type_nodes] - - -@pytest.mark.ixmp4 -def test__align_records_and_domain(test_mp: ixmp.Platform) -> None: - scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") - - # TODO Remove after proper parametrization - assert isinstance(test_mp._backend, IXMP4Backend) - - run = test_mp._backend.index[scen] - - # Test item without domain order - equation = run.optimization.equations.create("Equation") - expected: dict[str, Union[list[float], list[int], list[str]]] = {"foo": [1, 2, 3]} - assert _align_records_and_domain(item=equation, records=expected) == expected - - # Test sorting of records - indexset_1 = run.optimization.indexsets.create("Indexset 1") - indexset_2 = run.optimization.indexsets.create("Indexset 2") - parameter = run.optimization.parameters.create( - "Parameter", constrained_to_indexsets=[indexset_1.name, indexset_2.name] - ) - records: dict[str, Union[list[float], list[int], list[str]]] = { - indexset_2.name: [1, 2, 3], - "values": [1.0, 2.0, 3.0], - indexset_1.name: ["1", "2", "3"], - } - expected = { - indexset_1.name: ["1", "2", "3"], - indexset_2.name: [1, 2, 3], - "values": [1.0, 2.0, 3.0], - } - - assert _align_records_and_domain(item=parameter, records=records) == expected - - -def test__update_item_in_container() -> None: - item_list = [ - ContainerData( - name="table", kind="Table", records=["foo", "bar"], domain=["indexset"] - ), - ContainerData(name="scalar", kind="Scalar", records=3.14), - ContainerData(name="equation", kind="Equation", records=None), - ContainerData(name="variable", kind="Variable", records=None), - ] - - container = gt.Container() - - for item in item_list: - _update_item_in_container(container=container, item=item) - - assert all(container.hasSymbols(symbols=[item.name for item in item_list])) - - -def test__add_items_to_container() -> None: - container = gt.Container() - - # Test handling of empty list - empty_list: list[ContainerData] = [] - _add_items_to_container(container=container, items=empty_list) - assert container.data == {} - - # Test adding new item - records = ["foo", "bar"] - item_list = [ContainerData(name="Indexset", kind="IndexSet", records=records)] - _add_items_to_container(container=container, items=item_list) - assert container.hasSymbols("Indexset") - indexsets = container.getSets() - assert len(indexsets) == 1 - indexset = indexsets[0] - assert indexset.records["uni"].to_list() == records - - # Test updating existing item - records = ["baz"] - item_list = [ContainerData(name="Indexset", kind="IndexSet", records=records)] - _add_items_to_container(container=container, items=item_list) - indexsets = container.getSets() - assert len(indexsets) == 1 - indexset = indexsets[0] - assert indexset.records["uni"].to_list() == records - - -@pytest.mark.ixmp4 -def test__convert_ixmp4_items_to_containerdata(test_mp: ixmp.Platform) -> None: - scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") - - # TODO Remove after proper parametrization - assert isinstance(test_mp._backend, IXMP4Backend) - - run = test_mp._backend.index[scen] - - # Test handling of empty list - empty_list = _convert_ixmp4_items_to_containerdata(items=[]) - assert len(empty_list) == 0 - - # Test adding an Indexset (covers no new cases, but we need to index a Table) - indexset = run.optimization.indexsets.create("Indexset") - indexset.add(data=["foo", "bar", "baz"]) - container_list = _convert_ixmp4_items_to_containerdata(items=[indexset]) - assert len(container_list) == 1 - indexset_data = container_list[0] - assert indexset.name == indexset_data.name - assert indexset.data == indexset_data.records - - # Test adding an item where records are a dict - table = run.optimization.tables.create( - "Table", constrained_to_indexsets=[indexset.name] - ) - table.add(data={indexset.name: ["baz", "foo", "bar"]}) - container_list = _convert_ixmp4_items_to_containerdata(items=[table]) - table_data = container_list[0] - assert table.name == table_data.name - assert table.data == table_data.records - - -def test__record_version() -> None: - container = gt.Container() - - # Test recording the package version of something that's always present - _record_versions(container=container, packages=["ixmp"]) - indexsets = container.getSets() - assert len(indexsets) == 1 - indexset = indexsets[0] - assert indexset.name == "ixmp_version" - assert indexset.domain == ["*", "*"] - - -@pytest.mark.ixmp4 -def test_write_run_to_gdx(test_mp: ixmp.Platform, tmp_path: Path) -> None: - scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") - - # TODO Remove after proper parametrization - assert isinstance(test_mp._backend, IXMP4Backend) - - run = test_mp._backend.index[scen] - - # Test minimal call targeting all kinds of items - # NOTE All other functions are unit-tested already and other tests call write() with - # actual data, so we probably don't need to replicate here - file_path = tmp_path / "test_write.gdx" - write_run_to_gdx( - run=run, - file_name=file_path, - container_data=[], - record_version_packages=["ixmp"], - include_variables_and_equations=True, - ) - container = gt.Container(load_from=tmp_path / "test_write.gdx") - assert len(container.data) == 1 - assert "ixmp_version" in container.data.keys() - - -@pytest.mark.ixmp4 -def test__set_columns_to_read_from_records(test_mp: ixmp.Platform) -> None: - scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") - - # TODO Remove after proper parametrization - assert isinstance(test_mp._backend, IXMP4Backend) - - run = test_mp._backend.index[scen] - default_columns = ["levels", "marginals"] - - # Test an Equation without dimensions - equation = run.backend.optimization.equations.create(run_id=run.id, name="Equation") - columns = _set_columns_to_read_from_records(item=equation) - assert columns == default_columns - - # Test an indexed Variable - indexset = run.optimization.indexsets.create("Indexset") - variable = run.backend.optimization.variables.create( - run_id=run.id, name="Variable", constrained_to_indexsets=[indexset.name] - ) - columns = _set_columns_to_read_from_records(item=variable) - assert columns == [indexset.name] + default_columns - - # Test an Equation with dimension names - equation_2 = run.backend.optimization.equations.create( - run_id=run.id, - name="Equation 2", - constrained_to_indexsets=[indexset.name], - column_names=["Column"], - ) - columns = _set_columns_to_read_from_records(item=equation_2) - assert columns == ["Column"] + default_columns - - -# NOTE Keeping the read_equ()/read_var() tests separate because the read_*() functions -# are separate - - -@pytest.mark.ixmp4 -def test__read_variables_to_run(test_mp: ixmp.Platform) -> None: - scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") - - # TODO Remove after proper parametrization - assert isinstance(test_mp._backend, IXMP4Backend) - - run = test_mp._backend.index[scen] - indexset = run.optimization.indexsets.create("Indexset") - indexset.add(data=["foo", "bar", "baz"]) - variable = run.backend.optimization.variables.create( - run_id=run.id, name="Variable", constrained_to_indexsets=[indexset.name] - ) - container = gt.Container() - # NOTE Reading GDX always contains 'level', 'marginal', 'lower', 'upper', 'scale' - # The first two will be changed to plural, the latter three ignored - records = { - indexset.name: ["foo", "bar", "baz"], - "level": [1.0, 2.0, 3.0], - "marginal": [0, 0, 0], - "lower": [0, 0, 0], - "upper": [0, 0, 0], - "scale": [0, 0, 0], - } - container.addVariable(name="Variable", domain=[indexset.name], records=records) - - _read_variables_to_run(container=container, run=run, variables=[variable]) - expected = { - indexset.name: records[indexset.name], - "levels": records["level"], - "marginals": records["marginal"], - } - variable = run.backend.optimization.variables.get(run_id=run.id, name=variable.name) - assert variable.data == expected - - -@pytest.mark.ixmp4 -def test__read_equations_to_run(test_mp: ixmp.Platform) -> None: - scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") - - # TODO Remove after proper parametrization - assert isinstance(test_mp._backend, IXMP4Backend) - - run = test_mp._backend.index[scen] - indexset = run.optimization.indexsets.create("Indexset") - indexset.add(data=["foo", "bar", "baz"]) - equation = run.backend.optimization.equations.create( - run_id=run.id, name="Equation", constrained_to_indexsets=[indexset.name] - ) - container = gt.Container() - # NOTE Reading GDX always contains 'level', 'marginal', 'lower', 'upper', 'scale' - # The first two will be changed to plural, the latter three ignored - records = { - indexset.name: ["foo", "bar", "baz"], - "level": [1.0, 2.0, 3.0], - "marginal": [0, 0, 0], - "lower": [0, 0, 0], - "upper": [0, 0, 0], - "scale": [0, 0, 0], - } - container.addEquation( - name="Equation", type="E", domain=[indexset.name], records=records - ) - - _read_equations_to_run(container=container, run=run, equations=[equation]) - expected = { - indexset.name: records[indexset.name], - "levels": records["level"], - "marginals": records["marginal"], - } - equation = run.backend.optimization.equations.get(run_id=run.id, name=equation.name) - assert equation.data == expected - - -@pytest.mark.ixmp4 -def test_read_gdx_to_run(test_mp: ixmp.Platform, tmp_path: Path) -> None: - scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") - - # TODO Remove after proper parametrization - assert isinstance(test_mp._backend, IXMP4Backend) - - run = test_mp._backend.index[scen] - - # NOTE Names without space to produce "valid GAMS names" - variable_1 = run.backend.optimization.variables.create( - run_id=run.id, name="Variable1" - ) - variable_2 = run.backend.optimization.variables.create( - run_id=run.id, name="Variable2" - ) - variable_3 = run.backend.optimization.variables.create( - run_id=run.id, name="Variable3" - ) - records: dict[str, Union[list[float], list[int], list[str]]] = { - "level": [1.0], - "marginal": [0], - "lower": [0], - "upper": [0], - "scale": [0], - } - file_path = tmp_path / "test_read.gdx" - write_run_to_gdx( - run=run, - file_name=file_path, - container_data=[ - ContainerData(name=var.name, kind="Variable", records=records) - for var in [variable_1, variable_2, variable_3] - ], - record_version_packages=["ixmp"], - include_variables_and_equations=True, - ) - var_list = [variable_1.name, variable_3.name] - - # Test minimal confirmable call (with otherwise unused parameters) - read_gdx_to_run( - run=run, - result_file=file_path, - equ_list=[], - var_list=var_list, - comment="Test comment", - check_solution=True, - ) - for var in run.optimization.variables.list(): - # 'variable_2' is not supposed to be read in - expected = ( - {} - if var.name == variable_2.name - else {"levels": records["level"], "marginals": records["marginal"]} - ) - - assert var.data == expected diff --git a/ixmp/tests/backend/test_ixmp4_io.py b/ixmp/tests/backend/test_ixmp4_io.py new file mode 100644 index 000000000..655954f41 --- /dev/null +++ b/ixmp/tests/backend/test_ixmp4_io.py @@ -0,0 +1,424 @@ +from pathlib import Path +from typing import Union + +import gams.transfer as gt +import pytest + +import ixmp +from ixmp.backend.ixmp4 import IXMP4Backend +from ixmp.backend.ixmp4_io import ( + _add_items_to_container, + _align_records_and_domain, + _convert_ixmp4_items_to_containerdata, + _domain, + _ensure_correct_item_order, + _read_equations_to_run, + _read_variables_to_run, + _record_versions, + _records, + _set_columns_to_read_from_records, + _update_item_in_container, + read_gdx_to_run, + write_run_to_gdx, +) +from ixmp.util.ixmp4 import ContainerData + +# TODO Should all tests handling scenarios get `request` and use unique scenario names? +# TODO Should we make the run-retrieval its own fixture? + + +@pytest.mark.ixmp4 +def test__domain(test_mp: ixmp.Platform) -> None: + scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") + + # TODO Remove after proper parametrization + assert isinstance(test_mp._backend, IXMP4Backend) + + run = test_mp._backend.index[scen] + indexset = run.optimization.indexsets.create("Indexset") + table = run.optimization.tables.create( + "Table", constrained_to_indexsets=[indexset.name] + ) + + assert _domain(indexset) is None + assert _domain(table) == [indexset.name] + + +@pytest.mark.ixmp4 +def test__records(test_mp: ixmp.Platform) -> None: + scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") + + test_mp.add_unit("unit") + + # TODO Remove after proper parametrization + assert isinstance(test_mp._backend, IXMP4Backend) + + run = test_mp._backend.index[scen] + + # Test records of a Scalar + scalar = run.optimization.scalars.create("Scalar", 3.14, "unit") + assert _records(scalar) == scalar.value + + # Test records of an empty Table + indexset = run.optimization.indexsets.create("Indexset") + table = run.optimization.tables.create( + "Table", constrained_to_indexsets=[indexset.name] + ) + assert _records(table) is None + + # Test records of an IndexSet + indexset.add(["foo", "bar"]) + assert _records(indexset) == indexset.data + + # Test records of a Parameter + parameter = run.optimization.parameters.create( + "Parameter", constrained_to_indexsets=[indexset.name] + ) + parameter.add( + {indexset.name: ["foo", "bar"], "values": [1, 2], "units": ["unit"] * 2} + ) + expected = parameter.data.copy() + expected.pop("units") + assert _records(parameter) == expected + + +@pytest.mark.ixmp4 +def test__ensure_correct_item_order(test_mp: ixmp.Platform) -> None: + scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") + + test_mp.add_unit("unit") + + # TODO Remove after proper parametrization + assert isinstance(test_mp._backend, IXMP4Backend) + + run = test_mp._backend.index[scen] + + years = run.optimization.indexsets.create("years") + nodes = run.optimization.indexsets.create("nodes") + type_years = run.optimization.indexsets.create("type_years") + type_nodes = run.optimization.indexsets.create("type_nodes") + + assert _ensure_correct_item_order( + items=[type_years, years, type_nodes, nodes], repo=run.optimization.indexsets + ) == [years, nodes, type_years, type_nodes] + + +@pytest.mark.ixmp4 +def test__align_records_and_domain(test_mp: ixmp.Platform) -> None: + scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") + + # TODO Remove after proper parametrization + assert isinstance(test_mp._backend, IXMP4Backend) + + run = test_mp._backend.index[scen] + + # Test item without domain order + equation = run.optimization.equations.create("Equation") + expected: dict[str, Union[list[float], list[int], list[str]]] = {"foo": [1, 2, 3]} + assert _align_records_and_domain(item=equation, records=expected) == expected + + # Test sorting of records + indexset_1 = run.optimization.indexsets.create("Indexset 1") + indexset_2 = run.optimization.indexsets.create("Indexset 2") + parameter = run.optimization.parameters.create( + "Parameter", constrained_to_indexsets=[indexset_1.name, indexset_2.name] + ) + records: dict[str, Union[list[float], list[int], list[str]]] = { + indexset_2.name: [1, 2, 3], + "values": [1.0, 2.0, 3.0], + indexset_1.name: ["1", "2", "3"], + } + expected = { + indexset_1.name: ["1", "2", "3"], + indexset_2.name: [1, 2, 3], + "values": [1.0, 2.0, 3.0], + } + + assert _align_records_and_domain(item=parameter, records=records) == expected + + +def test__update_item_in_container() -> None: + item_list = [ + ContainerData( + name="table", kind="Table", records=["foo", "bar"], domain=["indexset"] + ), + ContainerData(name="scalar", kind="Scalar", records=3.14), + ContainerData(name="equation", kind="Equation", records=None), + ContainerData(name="variable", kind="Variable", records=None), + ] + + container = gt.Container() + + for item in item_list: + _update_item_in_container(container=container, item=item) + + assert all(container.hasSymbols(symbols=[item.name for item in item_list])) + + +def test__add_items_to_container() -> None: + container = gt.Container() + + # Test handling of empty list + empty_list: list[ContainerData] = [] + _add_items_to_container(container=container, items=empty_list) + assert container.data == {} + + # Test adding new item + records = ["foo", "bar"] + item_list = [ContainerData(name="Indexset", kind="IndexSet", records=records)] + _add_items_to_container(container=container, items=item_list) + assert container.hasSymbols("Indexset") + indexsets = container.getSets() + assert len(indexsets) == 1 + indexset = indexsets[0] + assert indexset.records["uni"].to_list() == records + + # Test updating existing item + records = ["baz"] + item_list = [ContainerData(name="Indexset", kind="IndexSet", records=records)] + _add_items_to_container(container=container, items=item_list) + indexsets = container.getSets() + assert len(indexsets) == 1 + indexset = indexsets[0] + assert indexset.records["uni"].to_list() == records + + +@pytest.mark.ixmp4 +def test__convert_ixmp4_items_to_containerdata(test_mp: ixmp.Platform) -> None: + scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") + + # TODO Remove after proper parametrization + assert isinstance(test_mp._backend, IXMP4Backend) + + run = test_mp._backend.index[scen] + + # Test handling of empty list + empty_list = _convert_ixmp4_items_to_containerdata(items=[]) + assert len(empty_list) == 0 + + # Test adding an Indexset (covers no new cases, but we need to index a Table) + indexset = run.optimization.indexsets.create("Indexset") + indexset.add(data=["foo", "bar", "baz"]) + container_list = _convert_ixmp4_items_to_containerdata(items=[indexset]) + assert len(container_list) == 1 + indexset_data = container_list[0] + assert indexset.name == indexset_data.name + assert indexset.data == indexset_data.records + + # Test adding an item where records are a dict + table = run.optimization.tables.create( + "Table", constrained_to_indexsets=[indexset.name] + ) + table.add(data={indexset.name: ["baz", "foo", "bar"]}) + container_list = _convert_ixmp4_items_to_containerdata(items=[table]) + table_data = container_list[0] + assert table.name == table_data.name + assert table.data == table_data.records + + +def test__record_version() -> None: + container = gt.Container() + + # Test recording the package version of something that's always present + _record_versions(container=container, packages=["ixmp"]) + indexsets = container.getSets() + assert len(indexsets) == 1 + indexset = indexsets[0] + assert indexset.name == "ixmp_version" + assert indexset.domain == ["*", "*"] + + +@pytest.mark.ixmp4 +def test_write_run_to_gdx(test_mp: ixmp.Platform, tmp_path: Path) -> None: + scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") + + # TODO Remove after proper parametrization + assert isinstance(test_mp._backend, IXMP4Backend) + + run = test_mp._backend.index[scen] + + # Test minimal call targeting all kinds of items + # NOTE All other functions are unit-tested already and other tests call write() with + # actual data, so we probably don't need to replicate here + file_path = tmp_path / "test_write.gdx" + write_run_to_gdx( + run=run, + file_name=file_path, + container_data=[], + record_version_packages=["ixmp"], + include_variables_and_equations=True, + ) + container = gt.Container(load_from=tmp_path / "test_write.gdx") + assert len(container.data) == 1 + assert "ixmp_version" in container.data.keys() + + +@pytest.mark.ixmp4 +def test__set_columns_to_read_from_records(test_mp: ixmp.Platform) -> None: + scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") + + # TODO Remove after proper parametrization + assert isinstance(test_mp._backend, IXMP4Backend) + + run = test_mp._backend.index[scen] + default_columns = ["levels", "marginals"] + + # Test an Equation without dimensions + equation = run.backend.optimization.equations.create(run_id=run.id, name="Equation") + columns = _set_columns_to_read_from_records(item=equation) + assert columns == default_columns + + # Test an indexed Variable + indexset = run.optimization.indexsets.create("Indexset") + variable = run.backend.optimization.variables.create( + run_id=run.id, name="Variable", constrained_to_indexsets=[indexset.name] + ) + columns = _set_columns_to_read_from_records(item=variable) + assert columns == [indexset.name] + default_columns + + # Test an Equation with dimension names + equation_2 = run.backend.optimization.equations.create( + run_id=run.id, + name="Equation 2", + constrained_to_indexsets=[indexset.name], + column_names=["Column"], + ) + columns = _set_columns_to_read_from_records(item=equation_2) + assert columns == ["Column"] + default_columns + + +# NOTE Keeping the read_equ()/read_var() tests separate because the read_*() functions +# are separate + + +@pytest.mark.ixmp4 +def test__read_variables_to_run(test_mp: ixmp.Platform) -> None: + scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") + + # TODO Remove after proper parametrization + assert isinstance(test_mp._backend, IXMP4Backend) + + run = test_mp._backend.index[scen] + indexset = run.optimization.indexsets.create("Indexset") + indexset.add(data=["foo", "bar", "baz"]) + variable = run.backend.optimization.variables.create( + run_id=run.id, name="Variable", constrained_to_indexsets=[indexset.name] + ) + container = gt.Container() + # NOTE Reading GDX always contains 'level', 'marginal', 'lower', 'upper', 'scale' + # The first two will be changed to plural, the latter three ignored + records = { + indexset.name: ["foo", "bar", "baz"], + "level": [1.0, 2.0, 3.0], + "marginal": [0, 0, 0], + "lower": [0, 0, 0], + "upper": [0, 0, 0], + "scale": [0, 0, 0], + } + container.addVariable(name="Variable", domain=[indexset.name], records=records) + + _read_variables_to_run(container=container, run=run, variables=[variable]) + expected = { + indexset.name: records[indexset.name], + "levels": records["level"], + "marginals": records["marginal"], + } + variable = run.backend.optimization.variables.get(run_id=run.id, name=variable.name) + assert variable.data == expected + + +@pytest.mark.ixmp4 +def test__read_equations_to_run(test_mp: ixmp.Platform) -> None: + scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") + + # TODO Remove after proper parametrization + assert isinstance(test_mp._backend, IXMP4Backend) + + run = test_mp._backend.index[scen] + indexset = run.optimization.indexsets.create("Indexset") + indexset.add(data=["foo", "bar", "baz"]) + equation = run.backend.optimization.equations.create( + run_id=run.id, name="Equation", constrained_to_indexsets=[indexset.name] + ) + container = gt.Container() + # NOTE Reading GDX always contains 'level', 'marginal', 'lower', 'upper', 'scale' + # The first two will be changed to plural, the latter three ignored + records = { + indexset.name: ["foo", "bar", "baz"], + "level": [1.0, 2.0, 3.0], + "marginal": [0, 0, 0], + "lower": [0, 0, 0], + "upper": [0, 0, 0], + "scale": [0, 0, 0], + } + container.addEquation( + name="Equation", type="E", domain=[indexset.name], records=records + ) + + _read_equations_to_run(container=container, run=run, equations=[equation]) + expected = { + indexset.name: records[indexset.name], + "levels": records["level"], + "marginals": records["marginal"], + } + equation = run.backend.optimization.equations.get(run_id=run.id, name=equation.name) + assert equation.data == expected + + +@pytest.mark.ixmp4 +def test_read_gdx_to_run(test_mp: ixmp.Platform, tmp_path: Path) -> None: + scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") + + # TODO Remove after proper parametrization + assert isinstance(test_mp._backend, IXMP4Backend) + + run = test_mp._backend.index[scen] + + # NOTE Names without space to produce "valid GAMS names" + variable_1 = run.backend.optimization.variables.create( + run_id=run.id, name="Variable1" + ) + variable_2 = run.backend.optimization.variables.create( + run_id=run.id, name="Variable2" + ) + variable_3 = run.backend.optimization.variables.create( + run_id=run.id, name="Variable3" + ) + records: dict[str, Union[list[float], list[int], list[str]]] = { + "level": [1.0], + "marginal": [0], + "lower": [0], + "upper": [0], + "scale": [0], + } + file_path = tmp_path / "test_read.gdx" + write_run_to_gdx( + run=run, + file_name=file_path, + container_data=[ + ContainerData(name=var.name, kind="Variable", records=records) + for var in [variable_1, variable_2, variable_3] + ], + record_version_packages=["ixmp"], + include_variables_and_equations=True, + ) + var_list = [variable_1.name, variable_3.name] + + # Test minimal confirmable call (with otherwise unused parameters) + read_gdx_to_run( + run=run, + result_file=file_path, + equ_list=[], + var_list=var_list, + comment="Test comment", + check_solution=True, + ) + for var in run.optimization.variables.list(): + # 'variable_2' is not supposed to be read in + expected = ( + {} + if var.name == variable_2.name + else {"levels": records["level"], "marginals": records["marginal"]} + ) + + assert var.data == expected From 06eca42373a9e470db4dee27a2937b6ceb533fbb Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Thu, 17 Apr 2025 08:35:09 +0200 Subject: [PATCH 63/72] Limit all ixmp4 tests to Python > 3.9 --- ixmp/testing/__init__.py | 8 +++++--- ixmp/tests/backend/test_ixmp4.py | 2 ++ ixmp/tests/backend/test_ixmp4_io.py | 2 ++ ixmp/tests/core/test_platform.py | 7 +------ 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/ixmp/testing/__init__.py b/ixmp/testing/__init__.py index 52f4a6e67..d28bacf35 100644 --- a/ixmp/testing/__init__.py +++ b/ixmp/testing/__init__.py @@ -90,9 +90,11 @@ # Parametrize platforms for jdbc and ixmp4 backends = list(BACKENDS.keys()) -# ixmp4 is not published for Python 3.9 -if sys.version_info < (3, 10): - backends.remove("ixmp4") + +# Provide a skip marker since ixmp4 is not published for Python 3.9 +min_ixmp4_version = pytest.mark.skipif( + sys.version_info < (3, 10), reason="ixmp4 requires Python 3.10 or higher" +) # Pytest hooks diff --git a/ixmp/tests/backend/test_ixmp4.py b/ixmp/tests/backend/test_ixmp4.py index e7745fbb2..2147a4723 100644 --- a/ixmp/tests/backend/test_ixmp4.py +++ b/ixmp/tests/backend/test_ixmp4.py @@ -22,6 +22,8 @@ # @pytest.mark.ixmp4 # def test__index_and_set_attrs(test_mp: Platform) -> None: +pytestmark = pytest.mark.min_ixmp4_version + @pytest.mark.ixmp4 def test__get_repo(test_mp: Platform) -> None: diff --git a/ixmp/tests/backend/test_ixmp4_io.py b/ixmp/tests/backend/test_ixmp4_io.py index 655954f41..9c3901d8d 100644 --- a/ixmp/tests/backend/test_ixmp4_io.py +++ b/ixmp/tests/backend/test_ixmp4_io.py @@ -26,6 +26,8 @@ # TODO Should all tests handling scenarios get `request` and use unique scenario names? # TODO Should we make the run-retrieval its own fixture? +pytestmark = pytest.mark.min_ixmp4_version + @pytest.mark.ixmp4 def test__domain(test_mp: ixmp.Platform) -> None: diff --git a/ixmp/tests/core/test_platform.py b/ixmp/tests/core/test_platform.py index 864cc908e..4bcbf2caa 100644 --- a/ixmp/tests/core/test_platform.py +++ b/ixmp/tests/core/test_platform.py @@ -2,7 +2,6 @@ import logging import re -import sys from collections.abc import Generator from pathlib import Path from sys import getrefcount @@ -16,15 +15,11 @@ import ixmp from ixmp.backend import FIELDS -from ixmp.testing import DATA, assert_logs, models +from ixmp.testing import DATA, assert_logs, min_ixmp4_version, models if TYPE_CHECKING: pass -min_ixmp4_version = pytest.mark.skipif( - sys.version_info < (3, 10), reason="ixmp4 requires Python 3.10 or higher" -) - class TestPlatform: def test_init0(self) -> None: From 46f34c6475502a1aaa0cd14e415beddcdcf3c8de Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Thu, 17 Apr 2025 11:57:41 +0200 Subject: [PATCH 64/72] Improve new tests * Extract fixtures for common setup * Avoid all top-level imports offending Python 3.9 --- ixmp/testing/__init__.py | 27 +- ixmp/tests/backend/test_ixmp4.py | 357 +++++++--------- ixmp/tests/backend/test_ixmp4_io.py | 635 +++++++++++++--------------- 3 files changed, 468 insertions(+), 551 deletions(-) diff --git a/ixmp/testing/__init__.py b/ixmp/testing/__init__.py index d28bacf35..e92c75de9 100644 --- a/ixmp/testing/__init__.py +++ b/ixmp/testing/__init__.py @@ -47,7 +47,7 @@ import pytest from click.testing import CliRunner -from ixmp import BACKENDS, Platform, cli +from ixmp import BACKENDS, Platform, Scenario, cli from ixmp import config as ixmp_config from .data import ( @@ -300,6 +300,31 @@ def test_mp_f( yield from _platform_fixture(request, tmp_env, test_data_path, backend=backend) +# NOTE No type hint for Python 3.9 compliance +@pytest.fixture +def ixmp4_backend(test_mp: Platform): + from ixmp.backend.ixmp4 import IXMP4Backend + + assert isinstance(test_mp._backend, IXMP4Backend) + return test_mp._backend + + +@pytest.fixture +def scenario(test_mp: Platform, request: pytest.FixtureRequest) -> Scenario: + return Scenario( + mp=test_mp, + model=request.node.nodeid + "model", + scenario="scenario", + version="new", + ) + + +# NOTE No type hint for Python 3.9 compliance +@pytest.fixture +def run(ixmp4_backend, scenario: Scenario): + return ixmp4_backend.index[scenario] + + # Assertions diff --git a/ixmp/tests/backend/test_ixmp4.py b/ixmp/tests/backend/test_ixmp4.py index 2147a4723..d8abac1ce 100644 --- a/ixmp/tests/backend/test_ixmp4.py +++ b/ixmp/tests/backend/test_ixmp4.py @@ -2,215 +2,16 @@ import pandas as pd import pytest -from ixmp4.core.optimization.equation import EquationRepository -from ixmp4.core.optimization.indexset import IndexSetRepository -from ixmp4.core.optimization.parameter import ParameterRepository -from ixmp4.core.optimization.scalar import ScalarRepository -from ixmp4.core.optimization.table import TableRepository -from ixmp4.core.optimization.variable import VariableRepository -from ixmp import Platform, Scenario -from ixmp.backend.ixmp4 import ( - IXMP4Backend, - _align_dtypes_for_filters, - _ensure_filters_values_are_lists, -) +from ixmp import Scenario +from ixmp.testing import min_ixmp4_version -# NOTE Not writing a test for _index_and_set_attrs(). This is a helper function that's -# also used by JDBC, and is always called for Backend.init() and Backend.get(), so is -# tested already. -# @pytest.mark.ixmp4 -# def test__index_and_set_attrs(test_mp: Platform) -> None: - -pytestmark = pytest.mark.min_ixmp4_version - - -@pytest.mark.ixmp4 -def test__get_repo(test_mp: Platform) -> None: - scenario = Scenario(test_mp, "model", "scenario", version="new") - repos: dict[Literal["indexset", "scalar", "set", "par", "equ", "var"], Any] = { - "indexset": IndexSetRepository, - "scalar": ScalarRepository, - "set": TableRepository, - "par": ParameterRepository, - "equ": EquationRepository, - "var": VariableRepository, - } - - # Ensure we're handling the correct type of backend - backend = test_mp._backend - assert isinstance(backend, IXMP4Backend) - - # Test correct kind of instance is returned - for type, expected_repo in repos.items(): - repo = backend._get_repo(s=scenario, type=type) - assert isinstance(repo, expected_repo) - - -@pytest.mark.ixmp4 -def test__find_item(test_mp: Platform) -> None: - scenario = Scenario(test_mp, "model", "scenario", version="new") - - # Ensure we're handling the correct type of backend - backend = test_mp._backend - assert isinstance(backend, IXMP4Backend) - - # Test unknown item name raises - with pytest.raises(KeyError, match="No item called 'NotFound' found"): - backend._find_item(s=scenario, name="NotFound") - - # Test finding an item and its type - name = "foo" - _type: Literal["indexset"] = "indexset" - scenario.init_set(name=name) - return_type, return_item = backend._find_item( - s=scenario, name=name, types=tuple([_type]) - ) - assert (_type, name) == (return_type, return_item.name) - - -@pytest.mark.ixmp4 -def test__get_item(test_mp: Platform) -> None: - scenario = Scenario(test_mp, "model", "scenario", version="new") - - # Ensure we're handling the correct type of backend - backend = test_mp._backend - assert isinstance(backend, IXMP4Backend) - - # Test getting an item - name = "foo" - _type: Literal["indexset"] = "indexset" - scenario.init_set(name=name) - indexset = backend._get_item(s=scenario, name=name, type=_type) - assert indexset.name == name - - -@pytest.mark.ixmp4 -def test__get_indexset_or_table(test_mp: Platform) -> None: - scenario = Scenario(test_mp, "model", "scenario", version="new") - - # Ensure we're handling the correct type of backend - backend = test_mp._backend - assert isinstance(backend, IXMP4Backend) - - # Test getting an item of "unknown" type (either 'indexset' or 'table'); - # getting a Table first tests getting an IndexSet, too - indexset_name = "foo" - scenario.init_set(name=indexset_name) - table_name = "bar" - scenario.init_set(name=table_name, idx_sets=[indexset_name]) - table = backend._get_indexset_or_table(s=scenario, name=table_name) - assert table.name == table_name - - -@pytest.mark.ixmp4 -def test__add_data_to_set(test_mp: Platform) -> None: - scenario = Scenario(test_mp, "model", "scenario", version="new") - - # Ensure we're handling the correct type of backend - backend = test_mp._backend - assert isinstance(backend, IXMP4Backend) - - # Test adding to an Indexset and warning about `comment` - indexset_name = "Indexset" - scenario.init_set(name=indexset_name) - key = "foo" - backend._add_data_to_set( - s=scenario, name=indexset_name, key=key, comment="Test comment" - ) - indexset_data = scenario.set(indexset_name) - assert isinstance(indexset_data, pd.Series) - pd.testing.assert_series_equal(indexset_data, pd.Series([key])) - - # Test adding to a Table - table_name = "Table" - scenario.init_set(name=table_name, idx_sets=[indexset_name]) - backend._add_data_to_set(s=scenario, name=table_name, key=[key]) - # We can assume this data type for Tables - pd.testing.assert_frame_equal( - cast(pd.DataFrame, scenario.set(table_name)), - pd.DataFrame({indexset_name: [key]}), - ) - - -@pytest.mark.ixmp4 -def test__create_scalar(test_mp: Platform) -> None: - scenario = Scenario(test_mp, "model", "scenario", version="new") - - # Ensure we're handling the correct type of backend - backend = test_mp._backend - assert isinstance(backend, IXMP4Backend) - - # Test creating a scalar with a comment - name = "Scalar" - value = 3.141592653589793238462643383279502884197169399375105820 - comment = "Comment" - backend._create_scalar( - s=scenario, name=name, value=value, unit=None, comment=comment - ) - # NOTE We don't really have a way to retrieve Scalars from IXMP4Backend right now - scalar = backend.index[scenario].optimization.scalars.get(name=name) - assert scalar.value == value - assert scalar.docs == comment - - -@pytest.mark.ixmp4 -def test__add_data_to_parameter(test_mp: Platform) -> None: - scenario = Scenario(test_mp, "model", "scenario", version="new") - - # Ensure we're handling the correct type of backend - backend = test_mp._backend - assert isinstance(backend, IXMP4Backend) - - # Set up auxiliary items - unit_name = "Unit" - test_mp.add_unit(unit=unit_name) - indexset_name = "Indexset" - scenario.init_set(name=indexset_name) - key = "foo" - scenario.add_set(name=indexset_name, key=key) - - # Test creating a parameter with a comment and key conversion to list[str] - name = "Parameter" - value = 3.141592653589793238462643383279502884197169399375105820 - comment = "Comment" - scenario.init_par(name=name, idx_sets=[indexset_name]) - backend._add_data_to_parameter( - s=scenario, name=name, key=key, value=value, unit=unit_name, comment=comment - ) - parameter = backend.index[scenario].optimization.parameters.get(name=name) - assert parameter.data == { - indexset_name: [key], - "values": [value], - "units": [unit_name], - } - - -@pytest.mark.ixmp4 -def test__get_set_data(test_mp: Platform) -> None: - scenario = Scenario(test_mp, "model", "scenario", version="new") - - # Ensure we're handling the correct type of backend - backend = test_mp._backend - assert isinstance(backend, IXMP4Backend) - - indexset_name = "Indexset" - scenario.init_set(name=indexset_name) - table_name = "Table" - scenario.init_set(name=table_name, idx_sets=[indexset_name]) - - # Test getting empty data (other cases tested in integration tests) - indexset_data = backend._get_set_data(s=scenario, name=indexset_name) - # We can assume this return type for Indexsets - pd.testing.assert_series_equal(cast(pd.Series, indexset_data), pd.Series([])) - table_data = backend._get_set_data(s=scenario, name=table_name) - # We can assume this return type for Tables - pd.testing.assert_frame_equal( - cast(pd.DataFrame, table_data), pd.DataFrame(columns=[indexset_name]) - ) +pytestmark = min_ixmp4_version def test__ensure_filters_values_are_lists() -> None: + from ixmp.backend.ixmp4 import _ensure_filters_values_are_lists + filters = {"foo": [1, 2], "bar": 3} expected = {"foo": [1, 2], "bar": [3]} _ensure_filters_values_are_lists(filters=filters) @@ -218,6 +19,8 @@ def test__ensure_filters_values_are_lists() -> None: def test__align_dtypes_for_filters() -> None: + from ixmp.backend.ixmp4 import _align_dtypes_for_filters + # This leads to dtypes 'int64' and 'object' df = pd.DataFrame({"foo": [1, 2, 3], "bar": ["baz", "foo", "bar"]}) # Filters' types are determined based on first item assuming one type per key @@ -226,3 +29,149 @@ def test__align_dtypes_for_filters() -> None: _align_dtypes_for_filters(filters=filters, data=df) assert filters == expected + + +# Overriding fixture that usually parametrizes test_mp +@pytest.fixture(scope="module") +def backend() -> Literal["ixmp4"]: + return "ixmp4" + + +# NOTE This marker usually parametrizes test creation, but with overriding `backend`, +# this might not be necessary. Keeping it for clarity. +@pytest.mark.ixmp4 +class TestIxmp4Functions: + """Test group for all functions touching ixmp4 directly.""" + + # NOTE Not writing a test for _index_and_set_attrs(). This is a helper function + # that's also used by JDBC, and is always called for Backend.init() and + # Backend.get(), so is tested already. + + def test__get_repo(self, ixmp4_backend, scenario: Scenario) -> None: + from ixmp4.core.optimization.equation import EquationRepository + from ixmp4.core.optimization.indexset import IndexSetRepository + from ixmp4.core.optimization.parameter import ParameterRepository + from ixmp4.core.optimization.scalar import ScalarRepository + from ixmp4.core.optimization.table import TableRepository + from ixmp4.core.optimization.variable import VariableRepository + + repos: dict[Literal["indexset", "scalar", "set", "par", "equ", "var"], Any] = { + "indexset": IndexSetRepository, + "scalar": ScalarRepository, + "set": TableRepository, + "par": ParameterRepository, + "equ": EquationRepository, + "var": VariableRepository, + } + + # Test correct kind of instance is returned + for type, expected_repo in repos.items(): + repo = ixmp4_backend._get_repo(s=scenario, type=type) + assert isinstance(repo, expected_repo) + + def test__find_item(self, ixmp4_backend, scenario: Scenario) -> None: + # Test unknown item name raises + with pytest.raises(KeyError, match="No item called 'NotFound' found"): + ixmp4_backend._find_item(s=scenario, name="NotFound") + + # Test finding an item and its type + name = "foo" + _type: Literal["indexset"] = "indexset" + scenario.init_set(name=name) + return_type, return_item = ixmp4_backend._find_item( + s=scenario, name=name, types=tuple([_type]) + ) + assert (_type, name) == (return_type, return_item.name) + + def test__get_item(self, ixmp4_backend, scenario: Scenario) -> None: + # Test getting an item + name = "foo" + _type: Literal["indexset"] = "indexset" + scenario.init_set(name=name) + indexset = ixmp4_backend._get_item(s=scenario, name=name, type=_type) + assert indexset.name == name + + def test__get_indexset_or_table(self, ixmp4_backend, scenario: Scenario) -> None: + # Test getting an item of "unknown" type (either 'indexset' or 'table'); + # getting a Table first tests getting an IndexSet, too + indexset_name = "foo" + scenario.init_set(name=indexset_name) + table_name = "bar" + scenario.init_set(name=table_name, idx_sets=[indexset_name]) + table = ixmp4_backend._get_indexset_or_table(s=scenario, name=table_name) + assert table.name == table_name + + def test__add_data_to_set(self, ixmp4_backend, scenario: Scenario) -> None: + # Test adding to an Indexset and warning about `comment` + indexset_name = "Indexset" + scenario.init_set(name=indexset_name) + key = "foo" + ixmp4_backend._add_data_to_set( + s=scenario, name=indexset_name, key=key, comment="Test comment" + ) + indexset_data = scenario.set(indexset_name) + assert isinstance(indexset_data, pd.Series) + pd.testing.assert_series_equal(indexset_data, pd.Series([key])) + + # Test adding to a Table + table_name = "Table" + scenario.init_set(name=table_name, idx_sets=[indexset_name]) + ixmp4_backend._add_data_to_set(s=scenario, name=table_name, key=[key]) + # We can assume this data type for Tables + pd.testing.assert_frame_equal( + cast(pd.DataFrame, scenario.set(table_name)), + pd.DataFrame({indexset_name: [key]}), + ) + + def test__create_scalar(self, ixmp4_backend, scenario: Scenario) -> None: + # Test creating a scalar with a comment + name = "Scalar" + value = 3.141592653589793238462643383279502884197169399375105820 + comment = "Comment" + ixmp4_backend._create_scalar( + s=scenario, name=name, value=value, unit=None, comment=comment + ) + # NOTE We don't really have a way to retrieve Scalars from IXMP4Backend + scalar = ixmp4_backend.index[scenario].optimization.scalars.get(name=name) + assert scalar.value == value + assert scalar.docs == comment + + def test__add_data_to_parameter(self, ixmp4_backend, scenario: Scenario) -> None: + # Set up auxiliary items + unit_name = "Unit" + ixmp4_backend.set_unit(name=unit_name, comment="Create test unit") + indexset_name = "Indexset" + scenario.init_set(name=indexset_name) + key = "foo" + scenario.add_set(name=indexset_name, key=key) + + # Test creating a parameter with a comment and key conversion to list[str] + name = "Parameter" + value = 3.141592653589793238462643383279502884197169399375105820 + comment = "Comment" + scenario.init_par(name=name, idx_sets=[indexset_name]) + ixmp4_backend._add_data_to_parameter( + s=scenario, name=name, key=key, value=value, unit=unit_name, comment=comment + ) + parameter = ixmp4_backend.index[scenario].optimization.parameters.get(name=name) + assert parameter.data == { + indexset_name: [key], + "values": [value], + "units": [unit_name], + } + + def test__get_set_data(self, ixmp4_backend, scenario: Scenario) -> None: + indexset_name = "Indexset" + scenario.init_set(name=indexset_name) + table_name = "Table" + scenario.init_set(name=table_name, idx_sets=[indexset_name]) + + # Test getting empty data (other cases tested in integration tests) + indexset_data = ixmp4_backend._get_set_data(s=scenario, name=indexset_name) + # We can assume this return type for Indexsets + pd.testing.assert_series_equal(cast(pd.Series, indexset_data), pd.Series([])) + table_data = ixmp4_backend._get_set_data(s=scenario, name=table_name) + # We can assume this return type for Tables + pd.testing.assert_frame_equal( + cast(pd.DataFrame, table_data), pd.DataFrame(columns=[indexset_name]) + ) diff --git a/ixmp/tests/backend/test_ixmp4_io.py b/ixmp/tests/backend/test_ixmp4_io.py index 9c3901d8d..bd35b69cd 100644 --- a/ixmp/tests/backend/test_ixmp4_io.py +++ b/ixmp/tests/backend/test_ixmp4_io.py @@ -1,145 +1,37 @@ from pathlib import Path -from typing import Union +from typing import Literal, Union -import gams.transfer as gt import pytest -import ixmp -from ixmp.backend.ixmp4 import IXMP4Backend -from ixmp.backend.ixmp4_io import ( - _add_items_to_container, - _align_records_and_domain, - _convert_ixmp4_items_to_containerdata, - _domain, - _ensure_correct_item_order, - _read_equations_to_run, - _read_variables_to_run, - _record_versions, - _records, - _set_columns_to_read_from_records, - _update_item_in_container, - read_gdx_to_run, - write_run_to_gdx, -) +from ixmp.testing import min_ixmp4_version from ixmp.util.ixmp4 import ContainerData -# TODO Should all tests handling scenarios get `request` and use unique scenario names? -# TODO Should we make the run-retrieval its own fixture? +pytestmark = min_ixmp4_version -pytestmark = pytest.mark.min_ixmp4_version +# NOTE No return type for Python 3.9 compliance +@pytest.fixture +def container(): + from gams.transfer import Container -@pytest.mark.ixmp4 -def test__domain(test_mp: ixmp.Platform) -> None: - scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") - - # TODO Remove after proper parametrization - assert isinstance(test_mp._backend, IXMP4Backend) - - run = test_mp._backend.index[scen] - indexset = run.optimization.indexsets.create("Indexset") - table = run.optimization.tables.create( - "Table", constrained_to_indexsets=[indexset.name] - ) - - assert _domain(indexset) is None - assert _domain(table) == [indexset.name] - - -@pytest.mark.ixmp4 -def test__records(test_mp: ixmp.Platform) -> None: - scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") - - test_mp.add_unit("unit") + return Container() - # TODO Remove after proper parametrization - assert isinstance(test_mp._backend, IXMP4Backend) - - run = test_mp._backend.index[scen] - - # Test records of a Scalar - scalar = run.optimization.scalars.create("Scalar", 3.14, "unit") - assert _records(scalar) == scalar.value - - # Test records of an empty Table - indexset = run.optimization.indexsets.create("Indexset") - table = run.optimization.tables.create( - "Table", constrained_to_indexsets=[indexset.name] - ) - assert _records(table) is None - - # Test records of an IndexSet - indexset.add(["foo", "bar"]) - assert _records(indexset) == indexset.data - - # Test records of a Parameter - parameter = run.optimization.parameters.create( - "Parameter", constrained_to_indexsets=[indexset.name] - ) - parameter.add( - {indexset.name: ["foo", "bar"], "values": [1, 2], "units": ["unit"] * 2} - ) - expected = parameter.data.copy() - expected.pop("units") - assert _records(parameter) == expected - - -@pytest.mark.ixmp4 -def test__ensure_correct_item_order(test_mp: ixmp.Platform) -> None: - scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") - test_mp.add_unit("unit") +def test__record_versions(container) -> None: + from ixmp.backend.ixmp4_io import _record_versions - # TODO Remove after proper parametrization - assert isinstance(test_mp._backend, IXMP4Backend) - - run = test_mp._backend.index[scen] - - years = run.optimization.indexsets.create("years") - nodes = run.optimization.indexsets.create("nodes") - type_years = run.optimization.indexsets.create("type_years") - type_nodes = run.optimization.indexsets.create("type_nodes") + # Test recording the package version of something that's always present + _record_versions(container=container, packages=["ixmp"]) + indexsets = container.getSets() + assert len(indexsets) == 1 + indexset = indexsets[0] + assert indexset.name == "ixmp_version" + assert indexset.domain == ["*", "*"] - assert _ensure_correct_item_order( - items=[type_years, years, type_nodes, nodes], repo=run.optimization.indexsets - ) == [years, nodes, type_years, type_nodes] +def test__update_item_in_container(container) -> None: + from ixmp.backend.ixmp4_io import _update_item_in_container -@pytest.mark.ixmp4 -def test__align_records_and_domain(test_mp: ixmp.Platform) -> None: - scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") - - # TODO Remove after proper parametrization - assert isinstance(test_mp._backend, IXMP4Backend) - - run = test_mp._backend.index[scen] - - # Test item without domain order - equation = run.optimization.equations.create("Equation") - expected: dict[str, Union[list[float], list[int], list[str]]] = {"foo": [1, 2, 3]} - assert _align_records_and_domain(item=equation, records=expected) == expected - - # Test sorting of records - indexset_1 = run.optimization.indexsets.create("Indexset 1") - indexset_2 = run.optimization.indexsets.create("Indexset 2") - parameter = run.optimization.parameters.create( - "Parameter", constrained_to_indexsets=[indexset_1.name, indexset_2.name] - ) - records: dict[str, Union[list[float], list[int], list[str]]] = { - indexset_2.name: [1, 2, 3], - "values": [1.0, 2.0, 3.0], - indexset_1.name: ["1", "2", "3"], - } - expected = { - indexset_1.name: ["1", "2", "3"], - indexset_2.name: [1, 2, 3], - "values": [1.0, 2.0, 3.0], - } - - assert _align_records_and_domain(item=parameter, records=records) == expected - - -def test__update_item_in_container() -> None: item_list = [ ContainerData( name="table", kind="Table", records=["foo", "bar"], domain=["indexset"] @@ -149,16 +41,14 @@ def test__update_item_in_container() -> None: ContainerData(name="variable", kind="Variable", records=None), ] - container = gt.Container() - for item in item_list: _update_item_in_container(container=container, item=item) assert all(container.hasSymbols(symbols=[item.name for item in item_list])) -def test__add_items_to_container() -> None: - container = gt.Container() +def test__add_items_to_container(container) -> None: + from ixmp.backend.ixmp4_io import _add_items_to_container # Test handling of empty list empty_list: list[ContainerData] = [] @@ -185,242 +75,295 @@ def test__add_items_to_container() -> None: assert indexset.records["uni"].to_list() == records +# Overriding fixture that usually parametrizes test_mp +@pytest.fixture(scope="module") +def backend() -> Literal["ixmp4"]: + return "ixmp4" + + +# NOTE This marker usually parametrizes test creation, but with overriding `backend`, +# this might not be necessary. Keeping it for clarity. @pytest.mark.ixmp4 -def test__convert_ixmp4_items_to_containerdata(test_mp: ixmp.Platform) -> None: - scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") +class TestIxmp4IOFunctions: + """Test group for all IO functions touching ixmp4 directly.""" - # TODO Remove after proper parametrization - assert isinstance(test_mp._backend, IXMP4Backend) + def test__domain(self, run) -> None: + from ixmp.backend.ixmp4_io import _domain - run = test_mp._backend.index[scen] + indexset = run.optimization.indexsets.create("Indexset") + table = run.optimization.tables.create( + "Table", constrained_to_indexsets=[indexset.name] + ) - # Test handling of empty list - empty_list = _convert_ixmp4_items_to_containerdata(items=[]) - assert len(empty_list) == 0 - - # Test adding an Indexset (covers no new cases, but we need to index a Table) - indexset = run.optimization.indexsets.create("Indexset") - indexset.add(data=["foo", "bar", "baz"]) - container_list = _convert_ixmp4_items_to_containerdata(items=[indexset]) - assert len(container_list) == 1 - indexset_data = container_list[0] - assert indexset.name == indexset_data.name - assert indexset.data == indexset_data.records - - # Test adding an item where records are a dict - table = run.optimization.tables.create( - "Table", constrained_to_indexsets=[indexset.name] - ) - table.add(data={indexset.name: ["baz", "foo", "bar"]}) - container_list = _convert_ixmp4_items_to_containerdata(items=[table]) - table_data = container_list[0] - assert table.name == table_data.name - assert table.data == table_data.records - - -def test__record_version() -> None: - container = gt.Container() + assert _domain(indexset) is None + assert _domain(table) == [indexset.name] - # Test recording the package version of something that's always present - _record_versions(container=container, packages=["ixmp"]) - indexsets = container.getSets() - assert len(indexsets) == 1 - indexset = indexsets[0] - assert indexset.name == "ixmp_version" - assert indexset.domain == ["*", "*"] + def test__records(self, run) -> None: + from ixmp.backend.ixmp4_io import _records + run.backend.units.create("unit") -@pytest.mark.ixmp4 -def test_write_run_to_gdx(test_mp: ixmp.Platform, tmp_path: Path) -> None: - scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") - - # TODO Remove after proper parametrization - assert isinstance(test_mp._backend, IXMP4Backend) - - run = test_mp._backend.index[scen] - - # Test minimal call targeting all kinds of items - # NOTE All other functions are unit-tested already and other tests call write() with - # actual data, so we probably don't need to replicate here - file_path = tmp_path / "test_write.gdx" - write_run_to_gdx( - run=run, - file_name=file_path, - container_data=[], - record_version_packages=["ixmp"], - include_variables_and_equations=True, - ) - container = gt.Container(load_from=tmp_path / "test_write.gdx") - assert len(container.data) == 1 - assert "ixmp_version" in container.data.keys() + # Test records of a Scalar + scalar = run.optimization.scalars.create("Scalar", 3.14, "unit") + assert _records(scalar) == scalar.value + # Test records of an empty Table + indexset = run.optimization.indexsets.create("Indexset") + table = run.optimization.tables.create( + "Table", constrained_to_indexsets=[indexset.name] + ) + assert _records(table) is None -@pytest.mark.ixmp4 -def test__set_columns_to_read_from_records(test_mp: ixmp.Platform) -> None: - scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") + # Test records of an IndexSet + indexset.add(["foo", "bar"]) + assert _records(indexset) == indexset.data - # TODO Remove after proper parametrization - assert isinstance(test_mp._backend, IXMP4Backend) + # Test records of a Parameter + parameter = run.optimization.parameters.create( + "Parameter", constrained_to_indexsets=[indexset.name] + ) + parameter.add( + {indexset.name: ["foo", "bar"], "values": [1, 2], "units": ["unit"] * 2} + ) + expected = parameter.data.copy() + expected.pop("units") + assert _records(parameter) == expected + + def test__ensure_correct_item_order(self, run) -> None: + from ixmp.backend.ixmp4_io import _ensure_correct_item_order + + years = run.optimization.indexsets.create("years") + nodes = run.optimization.indexsets.create("nodes") + type_years = run.optimization.indexsets.create("type_years") + type_nodes = run.optimization.indexsets.create("type_nodes") + + assert _ensure_correct_item_order( + items=[type_years, years, type_nodes, nodes], + repo=run.optimization.indexsets, + ) == [years, nodes, type_years, type_nodes] + + def test__align_records_and_domain(self, run) -> None: + from ixmp.backend.ixmp4_io import _align_records_and_domain + + # Test item without domain order + equation = run.optimization.equations.create("Equation") + expected: dict[str, Union[list[float], list[int], list[str]]] = { + "foo": [1, 2, 3] + } + assert _align_records_and_domain(item=equation, records=expected) == expected + + # Test sorting of records + indexset_1 = run.optimization.indexsets.create("Indexset 1") + indexset_2 = run.optimization.indexsets.create("Indexset 2") + parameter = run.optimization.parameters.create( + "Parameter", constrained_to_indexsets=[indexset_1.name, indexset_2.name] + ) + records: dict[str, Union[list[float], list[int], list[str]]] = { + indexset_2.name: [1, 2, 3], + "values": [1.0, 2.0, 3.0], + indexset_1.name: ["1", "2", "3"], + } + expected = { + indexset_1.name: ["1", "2", "3"], + indexset_2.name: [1, 2, 3], + "values": [1.0, 2.0, 3.0], + } + + assert _align_records_and_domain(item=parameter, records=records) == expected + + def test__convert_ixmp4_items_to_containerdata(self, run) -> None: + from ixmp.backend.ixmp4_io import _convert_ixmp4_items_to_containerdata + + # Test handling of empty list + empty_list = _convert_ixmp4_items_to_containerdata(items=[]) + assert len(empty_list) == 0 + + # Test adding an Indexset (covers no new cases, but we need to index a Table) + indexset = run.optimization.indexsets.create("Indexset") + indexset.add(data=["foo", "bar", "baz"]) + container_list = _convert_ixmp4_items_to_containerdata(items=[indexset]) + assert len(container_list) == 1 + indexset_data = container_list[0] + assert indexset.name == indexset_data.name + assert indexset.data == indexset_data.records + + # Test adding an item where records are a dict + table = run.optimization.tables.create( + "Table", constrained_to_indexsets=[indexset.name] + ) + table.add(data={indexset.name: ["baz", "foo", "bar"]}) + container_list = _convert_ixmp4_items_to_containerdata(items=[table]) + table_data = container_list[0] + assert table.name == table_data.name + assert table.data == table_data.records + + def test_write_run_to_gdx(self, run, tmp_path: Path) -> None: + from gams.transfer import Container + + from ixmp.backend.ixmp4_io import write_run_to_gdx + + # Test minimal call targeting all kinds of items + # NOTE All other functions are unit-tested already and other tests call write() + # with actual data, so we probably don't need to replicate here + file_path = tmp_path / "test_write.gdx" + write_run_to_gdx( + run=run, + file_name=file_path, + container_data=[], + record_version_packages=["ixmp"], + include_variables_and_equations=True, + ) + container = Container(load_from=tmp_path / "test_write.gdx") + assert len(container.data) == 1 + assert "ixmp_version" in container.data.keys() - run = test_mp._backend.index[scen] - default_columns = ["levels", "marginals"] + def test__set_columns_to_read_from_records(self, run) -> None: + from ixmp.backend.ixmp4_io import _set_columns_to_read_from_records - # Test an Equation without dimensions - equation = run.backend.optimization.equations.create(run_id=run.id, name="Equation") - columns = _set_columns_to_read_from_records(item=equation) - assert columns == default_columns + default_columns = ["levels", "marginals"] - # Test an indexed Variable - indexset = run.optimization.indexsets.create("Indexset") - variable = run.backend.optimization.variables.create( - run_id=run.id, name="Variable", constrained_to_indexsets=[indexset.name] - ) - columns = _set_columns_to_read_from_records(item=variable) - assert columns == [indexset.name] + default_columns + # Test an Equation without dimensions + equation = run.backend.optimization.equations.create( + run_id=run.id, name="Equation" + ) + columns = _set_columns_to_read_from_records(item=equation) + assert columns == default_columns - # Test an Equation with dimension names - equation_2 = run.backend.optimization.equations.create( - run_id=run.id, - name="Equation 2", - constrained_to_indexsets=[indexset.name], - column_names=["Column"], - ) - columns = _set_columns_to_read_from_records(item=equation_2) - assert columns == ["Column"] + default_columns + # Test an indexed Variable + indexset = run.optimization.indexsets.create("Indexset") + variable = run.backend.optimization.variables.create( + run_id=run.id, name="Variable", constrained_to_indexsets=[indexset.name] + ) + columns = _set_columns_to_read_from_records(item=variable) + assert columns == [indexset.name] + default_columns + + # Test an Equation with dimension names + equation_2 = run.backend.optimization.equations.create( + run_id=run.id, + name="Equation 2", + constrained_to_indexsets=[indexset.name], + column_names=["Column"], + ) + columns = _set_columns_to_read_from_records(item=equation_2) + assert columns == ["Column"] + default_columns + # NOTE Keeping the read_equ()/read_var() tests separate because the read_*() + # functions are separate -# NOTE Keeping the read_equ()/read_var() tests separate because the read_*() functions -# are separate + def test__read_variables_to_run(self, run, container) -> None: + from ixmp.backend.ixmp4_io import _read_variables_to_run + indexset = run.optimization.indexsets.create("Indexset") + indexset.add(data=["foo", "bar", "baz"]) + variable = run.backend.optimization.variables.create( + run_id=run.id, name="Variable", constrained_to_indexsets=[indexset.name] + ) -@pytest.mark.ixmp4 -def test__read_variables_to_run(test_mp: ixmp.Platform) -> None: - scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") - - # TODO Remove after proper parametrization - assert isinstance(test_mp._backend, IXMP4Backend) - - run = test_mp._backend.index[scen] - indexset = run.optimization.indexsets.create("Indexset") - indexset.add(data=["foo", "bar", "baz"]) - variable = run.backend.optimization.variables.create( - run_id=run.id, name="Variable", constrained_to_indexsets=[indexset.name] - ) - container = gt.Container() - # NOTE Reading GDX always contains 'level', 'marginal', 'lower', 'upper', 'scale' - # The first two will be changed to plural, the latter three ignored - records = { - indexset.name: ["foo", "bar", "baz"], - "level": [1.0, 2.0, 3.0], - "marginal": [0, 0, 0], - "lower": [0, 0, 0], - "upper": [0, 0, 0], - "scale": [0, 0, 0], - } - container.addVariable(name="Variable", domain=[indexset.name], records=records) - - _read_variables_to_run(container=container, run=run, variables=[variable]) - expected = { - indexset.name: records[indexset.name], - "levels": records["level"], - "marginals": records["marginal"], - } - variable = run.backend.optimization.variables.get(run_id=run.id, name=variable.name) - assert variable.data == expected + # NOTE A read GDX always contains 'level', 'marginal', 'lower', 'upper', 'scale' + # The first two will be changed to plural, the latter three ignored + records = { + indexset.name: ["foo", "bar", "baz"], + "level": [1.0, 2.0, 3.0], + "marginal": [0, 0, 0], + "lower": [0, 0, 0], + "upper": [0, 0, 0], + "scale": [0, 0, 0], + } + container.addVariable(name="Variable", domain=[indexset.name], records=records) + + _read_variables_to_run(container=container, run=run, variables=[variable]) + expected = { + indexset.name: records[indexset.name], + "levels": records["level"], + "marginals": records["marginal"], + } + variable = run.backend.optimization.variables.get( + run_id=run.id, name=variable.name + ) + assert variable.data == expected + def test__read_equations_to_run(self, run, container) -> None: + from ixmp.backend.ixmp4_io import _read_equations_to_run -@pytest.mark.ixmp4 -def test__read_equations_to_run(test_mp: ixmp.Platform) -> None: - scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") - - # TODO Remove after proper parametrization - assert isinstance(test_mp._backend, IXMP4Backend) - - run = test_mp._backend.index[scen] - indexset = run.optimization.indexsets.create("Indexset") - indexset.add(data=["foo", "bar", "baz"]) - equation = run.backend.optimization.equations.create( - run_id=run.id, name="Equation", constrained_to_indexsets=[indexset.name] - ) - container = gt.Container() - # NOTE Reading GDX always contains 'level', 'marginal', 'lower', 'upper', 'scale' - # The first two will be changed to plural, the latter three ignored - records = { - indexset.name: ["foo", "bar", "baz"], - "level": [1.0, 2.0, 3.0], - "marginal": [0, 0, 0], - "lower": [0, 0, 0], - "upper": [0, 0, 0], - "scale": [0, 0, 0], - } - container.addEquation( - name="Equation", type="E", domain=[indexset.name], records=records - ) - - _read_equations_to_run(container=container, run=run, equations=[equation]) - expected = { - indexset.name: records[indexset.name], - "levels": records["level"], - "marginals": records["marginal"], - } - equation = run.backend.optimization.equations.get(run_id=run.id, name=equation.name) - assert equation.data == expected + indexset = run.optimization.indexsets.create("Indexset") + indexset.add(data=["foo", "bar", "baz"]) + equation = run.backend.optimization.equations.create( + run_id=run.id, name="Equation", constrained_to_indexsets=[indexset.name] + ) + # NOTE A read GDX always contains 'level', 'marginal', 'lower', 'upper', 'scale' + # The first two will be changed to plural, the latter three ignored + records = { + indexset.name: ["foo", "bar", "baz"], + "level": [1.0, 2.0, 3.0], + "marginal": [0, 0, 0], + "lower": [0, 0, 0], + "upper": [0, 0, 0], + "scale": [0, 0, 0], + } + container.addEquation( + name="Equation", type="E", domain=[indexset.name], records=records + ) -@pytest.mark.ixmp4 -def test_read_gdx_to_run(test_mp: ixmp.Platform, tmp_path: Path) -> None: - scen = ixmp.Scenario(test_mp, "model", "scenario", version="new") - - # TODO Remove after proper parametrization - assert isinstance(test_mp._backend, IXMP4Backend) - - run = test_mp._backend.index[scen] - - # NOTE Names without space to produce "valid GAMS names" - variable_1 = run.backend.optimization.variables.create( - run_id=run.id, name="Variable1" - ) - variable_2 = run.backend.optimization.variables.create( - run_id=run.id, name="Variable2" - ) - variable_3 = run.backend.optimization.variables.create( - run_id=run.id, name="Variable3" - ) - records: dict[str, Union[list[float], list[int], list[str]]] = { - "level": [1.0], - "marginal": [0], - "lower": [0], - "upper": [0], - "scale": [0], - } - file_path = tmp_path / "test_read.gdx" - write_run_to_gdx( - run=run, - file_name=file_path, - container_data=[ - ContainerData(name=var.name, kind="Variable", records=records) - for var in [variable_1, variable_2, variable_3] - ], - record_version_packages=["ixmp"], - include_variables_and_equations=True, - ) - var_list = [variable_1.name, variable_3.name] - - # Test minimal confirmable call (with otherwise unused parameters) - read_gdx_to_run( - run=run, - result_file=file_path, - equ_list=[], - var_list=var_list, - comment="Test comment", - check_solution=True, - ) - for var in run.optimization.variables.list(): - # 'variable_2' is not supposed to be read in - expected = ( - {} - if var.name == variable_2.name - else {"levels": records["level"], "marginals": records["marginal"]} + _read_equations_to_run(container=container, run=run, equations=[equation]) + expected = { + indexset.name: records[indexset.name], + "levels": records["level"], + "marginals": records["marginal"], + } + equation = run.backend.optimization.equations.get( + run_id=run.id, name=equation.name ) + assert equation.data == expected + + def test_read_gdx_to_run(self, run, tmp_path: Path) -> None: + from ixmp.backend.ixmp4_io import read_gdx_to_run, write_run_to_gdx - assert var.data == expected + # NOTE Names without space to produce "valid GAMS names" + variable_1 = run.backend.optimization.variables.create( + run_id=run.id, name="Variable1" + ) + variable_2 = run.backend.optimization.variables.create( + run_id=run.id, name="Variable2" + ) + variable_3 = run.backend.optimization.variables.create( + run_id=run.id, name="Variable3" + ) + records: dict[str, Union[list[float], list[int], list[str]]] = { + "level": [1.0], + "marginal": [0], + "lower": [0], + "upper": [0], + "scale": [0], + } + file_path = tmp_path / "test_read.gdx" + write_run_to_gdx( + run=run, + file_name=file_path, + container_data=[ + ContainerData(name=var.name, kind="Variable", records=records) + for var in [variable_1, variable_2, variable_3] + ], + record_version_packages=["ixmp"], + include_variables_and_equations=True, + ) + var_list = [variable_1.name, variable_3.name] + + # Test minimal confirmable call (with otherwise unused parameters) + read_gdx_to_run( + run=run, + result_file=file_path, + equ_list=[], + var_list=var_list, + comment="Test comment", + check_solution=True, + ) + for var in run.optimization.variables.list(): + # 'variable_2' is not supposed to be read in + expected = ( + {} + if var.name == variable_2.name + else {"levels": records["level"], "marginals": records["marginal"]} + ) + + assert var.data == expected From 812b3ef1877da480cf6cac812a79eb8385827832 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Thu, 17 Apr 2025 14:20:09 +0200 Subject: [PATCH 65/72] Document stat IXMP4Backend requires Python > 3.9 --- doc/api-backend.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/api-backend.rst b/doc/api-backend.rst index 7872fe064..48fb4524d 100644 --- a/doc/api-backend.rst +++ b/doc/api-backend.rst @@ -61,6 +61,15 @@ Provided backends .. autofunction:: ixmp.backend.jdbc.start_jvm +.. currentmodule:: ixmp.backend.ixmp4 + +.. warning:: + `ixmp4 `_ only supports Python 3.10 and above. + Please ensure you are using a sufficiently recent Python version if you want to use IXMP4Backend. + If you are restricted to Python 3.9 and below, please use JDBCBackend instead. + +.. autoclass:: ixmp.backend.ixmp4.IXMP4Backend + .. currentmodule:: ixmp.backend From 300b3dd60081505124e25a8d12c765ce140bb216 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Thu, 17 Apr 2025 14:20:24 +0200 Subject: [PATCH 66/72] Add PR to release notes --- RELEASE_NOTES.rst | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index cf7aa4d47..6abf23f17 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -1,8 +1,15 @@ -.. Next release -.. ============ +Next release +============ + +All changes +----------- + +- Add :class:`.IXMP4Backend` as an alternative to :class:`.JDBCBackend` (:pull:`552`). + Please note: + + - This requires ixmp4, which is only compatible with Python 3.10 and above. + - :class:`.IXMP4Backend` is still missing several features and documentation, which will be added in subsequent PRs. -.. All changes -.. ----------- .. _v3.10.0: From 12d48106c11eac97a7aaf332240f17da481a82ce Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Wed, 23 Apr 2025 14:33:08 +0200 Subject: [PATCH 67/72] Pass gams system_dir when constructing Container --- ixmp/backend/ixmp4_io.py | 7 +++++-- ixmp/tests/backend/test_ixmp4_io.py | 8 ++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/ixmp/backend/ixmp4_io.py b/ixmp/backend/ixmp4_io.py index 5cbd47f4b..358cad7c5 100644 --- a/ixmp/backend/ixmp4_io.py +++ b/ixmp/backend/ixmp4_io.py @@ -16,6 +16,7 @@ from ixmp4.data.abstract.optimization.equation import Equation as AbstractEquation from ixmp4.data.abstract.optimization.variable import Variable as AbstractVariable +from ixmp.model.gams import gams_info from ixmp.util.ixmp4 import ContainerData log = logging.getLogger(__name__) @@ -272,7 +273,7 @@ def write_run_to_gdx( # TODO How to deal with [*] Set? That seems to be handled by GAMS automatically. # Define the container - container = gt.Container() + container = gt.Container(system_directory=str(gams_info().system_dir)) repository: list[Lister] = [ run.optimization.indexsets, @@ -405,7 +406,9 @@ def read_gdx_to_run( ) # Create a GAMS Container from the `result_file` - container = gt.Container(load_from=result_file) + container = gt.Container( + load_from=result_file, system_directory=str(gams_info().system_dir) + ) # Load requested Variables and read them to `run` # NOTE This handles empty `var_list`, too, diff --git a/ixmp/tests/backend/test_ixmp4_io.py b/ixmp/tests/backend/test_ixmp4_io.py index bd35b69cd..b6c58dd16 100644 --- a/ixmp/tests/backend/test_ixmp4_io.py +++ b/ixmp/tests/backend/test_ixmp4_io.py @@ -3,6 +3,7 @@ import pytest +from ixmp.model.gams import gams_info from ixmp.testing import min_ixmp4_version from ixmp.util.ixmp4 import ContainerData @@ -14,7 +15,7 @@ def container(): from gams.transfer import Container - return Container() + return Container(system_directory=str(gams_info().system_dir)) def test__record_versions(container) -> None: @@ -213,7 +214,10 @@ def test_write_run_to_gdx(self, run, tmp_path: Path) -> None: record_version_packages=["ixmp"], include_variables_and_equations=True, ) - container = Container(load_from=tmp_path / "test_write.gdx") + container = Container( + load_from=tmp_path / "test_write.gdx", + system_directory=str(gams_info().system_dir), + ) assert len(container.data) == 1 assert "ixmp_version" in container.data.keys() From be81fade1c9f20229324be44ec39c49a679d49e9 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Wed, 23 Apr 2025 14:38:19 +0200 Subject: [PATCH 68/72] Omit `.venv` from coverage reporting --- pyproject.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5a385f2a3..f538a48ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,7 +85,10 @@ exclude_also = [ # Imports only used by type checkers "if TYPE_CHECKING:", ] -omit = ["ixmp/util/sphinx_linkcode_github.py"] +omit = [ + "ixmp/util/sphinx_linkcode_github.py", + ".venv/*", +] [tool.mypy] files = [ From 48e3b44069d5825c43e3f40abcc88d955ffb1a1a Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Thu, 24 Apr 2025 14:00:34 +0200 Subject: [PATCH 69/72] Guard against wrong order of ReadKwargs --- ixmp/backend/ixmp4.py | 6 ++---- ixmp/backend/jdbc.py | 1 + 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/ixmp/backend/ixmp4.py b/ixmp/backend/ixmp4.py index b477a8d25..b2d1414c2 100644 --- a/ixmp/backend/ixmp4.py +++ b/ixmp/backend/ixmp4.py @@ -827,6 +827,7 @@ def delete_item( type: Literal["set", "par", "equ", "indexset"], name: str, ) -> None: + # NOTE We need to allow 'set' and 'indexset' for type for backward compatibility if type == "set": # type will always be either "indexset" or "set" type, _ = self._find_item(s=s, name=name, types=("indexset", "set")) # type: ignore[assignment] @@ -1163,7 +1164,7 @@ def read_file( if _path.suffix == ".gdx" and item_type is ItemType.MODEL: kw = {"check_solution", "comment", "equ_list", "var_list"} - if not isinstance(ts, Scenario): # pragma: no cover + if not isinstance(ts, Scenario): raise ValueError("read from GDX requires a Scenario object") elif set(_kwargs.keys()) != kw: raise ValueError( @@ -1178,9 +1179,6 @@ def read_file( equ_list = cast(list[str], _kwargs.pop("equ_list")) var_list = cast(list[str], _kwargs.pop("var_list")) - if len(_kwargs): - raise ValueError(f"extra keyword arguments {_kwargs}") - read_gdx_to_run( run=self.index[ts], result_file=_path, diff --git a/ixmp/backend/jdbc.py b/ixmp/backend/jdbc.py index 659d7205f..a9281daad 100644 --- a/ixmp/backend/jdbc.py +++ b/ixmp/backend/jdbc.py @@ -561,6 +561,7 @@ def read_file(self, path, item_type: ItemType, **kwargs): kwargs.pop("check_solution"), ) + # NOTE This test seems unnecessary with the 'elif' clause above if len(kwargs): raise ValueError(f"extra keyword arguments {kwargs}") From 691bb247cd125fac6dbd9d57704370ad3ba049ac Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Thu, 24 Apr 2025 14:01:14 +0200 Subject: [PATCH 70/72] Add tests for some ixmp4 edge cases --- ixmp/tests/backend/test_ixmp4.py | 155 ++++++++++++++++++++++++++++ ixmp/tests/backend/test_ixmp4_io.py | 15 +++ 2 files changed, 170 insertions(+) diff --git a/ixmp/tests/backend/test_ixmp4.py b/ixmp/tests/backend/test_ixmp4.py index d8abac1ce..f55577eef 100644 --- a/ixmp/tests/backend/test_ixmp4.py +++ b/ixmp/tests/backend/test_ixmp4.py @@ -1,9 +1,11 @@ +from pathlib import Path from typing import Any, Literal, cast import pandas as pd import pytest from ixmp import Scenario +from ixmp.backend import ItemType from ixmp.testing import min_ixmp4_version pytestmark = min_ixmp4_version @@ -47,6 +49,10 @@ class TestIxmp4Functions: # that's also used by JDBC, and is always called for Backend.init() and # Backend.get(), so is tested already. + def test__ni(self, ixmp4_backend) -> None: + with pytest.raises(NotImplementedError): + ixmp4_backend._ni() + def test__get_repo(self, ixmp4_backend, scenario: Scenario) -> None: from ixmp4.core.optimization.equation import EquationRepository from ixmp4.core.optimization.indexset import IndexSetRepository @@ -175,3 +181,152 @@ def test__get_set_data(self, ixmp4_backend, scenario: Scenario) -> None: pd.testing.assert_frame_equal( cast(pd.DataFrame, table_data), pd.DataFrame(columns=[indexset_name]) ) + + # Test some edge cases for standard functions + def test_handle_config(self, ixmp4_backend) -> None: + # Test raising for unhandled positional args + with pytest.raises(ValueError, match="Unhandled positional args"): + ixmp4_backend.handle_config(["test arg"], {"foo": "bar"}) + + # Test raising for missing required key + with pytest.raises(ValueError, match="Missing key '_name'"): + ixmp4_backend.handle_config([], {"foo": "bar"}) + + def test_set_node(self, ixmp4_backend, caplog: pytest.LogCaptureFixture) -> None: + from ixmp.backend.ixmp4 import log + + # Test logging warnings for unused/required parameters + parent = "Parent" + synonym = "Synonym" + with caplog.at_level("WARNING", logger=log.name): + ixmp4_backend.set_node( + name="Region", parent=parent, hierarchy=None, synonym=synonym + ) + + expected = [ + f"Discarding parent parameter {parent}; unused in ixmp4.", + f"Discarding synonym parameter {synonym}; unused in ixmp4.", + "IXMP4Backend.set_node() requires to specify 'hierarchy'! " + "Using 'None' as the meaningsless default.", + ] + assert caplog.messages == expected + + def test_clone( + self, ixmp4_backend, caplog: pytest.LogCaptureFixture, scenario: Scenario + ) -> None: + from ixmp.backend.ixmp4 import log + + # Test logging a warning for first_model_year + with caplog.at_level("WARNING", logger=log.name): + ixmp4_backend.clone( + s=scenario, + platform_dest=scenario.platform, + model=scenario.model, + scenario=scenario.scenario + "_clone", + annotation="not used", + keep_solution=False, + first_model_year=1, + ) + + expected = ( + "ixmp4-backed Scenarios don't support cloning from `first_model_year` only!" + ) + assert expected in caplog.messages + + def test_clear_solution( + self, ixmp4_backend, caplog: pytest.LogCaptureFixture, scenario: Scenario + ) -> None: + from ixmp.backend.ixmp4 import log + + # Test logging a warning for from_year + with caplog.at_level("WARNING", logger=log.name): + ixmp4_backend.clear_solution(s=scenario, from_year=1) + + expected = ( + "ixmp4 does not support removing the solution only after a certain year" + ) + assert expected in caplog.messages + + def test_run_id(self, ixmp4_backend, scenario: Scenario) -> None: + # NOTE Depending on what run_id() should actually fetch, this needs adapting + # scenario sets up a new Run, which has version 1 + assert ixmp4_backend.run_id(ts=scenario) == 1 + + def test_item_delete_elements(self, ixmp4_backend, scenario: Scenario) -> None: + # Prepare some data + run = ixmp4_backend.index[scenario] + indexset_data = "foo" + indexset = run.optimization.indexsets.create("Indexset") + indexset.add(data=indexset_data) + table_data = {indexset.name: [indexset_data]} + table = run.optimization.tables.create( + "Table", constrained_to_indexsets=[indexset.name] + ) + table.add(data=table_data) + + # Assert data is stored in scenario + set_data = scenario.set(name=table.name) + assert isinstance(set_data, pd.DataFrame) + assert not set_data.empty + + # Test data deletion for Tables + ixmp4_backend.item_delete_elements( + s=scenario, type="set", name=table.name, keys=[[indexset_data]] + ) + new_data = scenario.set(name=table.name) + assert isinstance(new_data, pd.DataFrame) + assert new_data.empty + + def test_delete_item(self, ixmp4_backend, scenario: Scenario) -> None: + # Create a 'set' to delete + run = ixmp4_backend.index[scenario] + indexset = run.optimization.indexsets.create("Indexset") + ixmp4_backend.delete_item(s=scenario, type="set", name=indexset.name) + + # Test there are no 'sets' on scenario anymore + assert ixmp4_backend.list_items(s=scenario, type="set") == [] + + def test_write_file(self, ixmp4_backend) -> None: + # Test raising an error for unknown file extension + with pytest.raises(NotImplementedError): + ixmp4_backend.write_file( + path=Path("none.txt"), item_type=ItemType.SET, filters={} + ) + + # Test raising with incorrect ItemType + with pytest.raises(NotImplementedError): + ixmp4_backend.write_file( + path=Path("none.gdx"), item_type=ItemType.EQU, filters={} + ) + + def test_read_file(self, ixmp4_backend, scenario: Scenario) -> None: + # Test raising an error for unknown file extension + with pytest.raises(NotImplementedError): + ixmp4_backend.read_file( + path=Path("none.txt"), item_type=ItemType.SET, filters={} + ) + + # Test raising with incorrect ItemType + with pytest.raises(NotImplementedError): + ixmp4_backend.read_file( + path=Path("none.gdx"), item_type=ItemType.EQU, filters={} + ) + + # Test raising when filters doesn't include a proper Scenario + with pytest.raises( + ValueError, match="read from GDX requires a Scenario object" + ): + ixmp4_backend.read_file( + path=Path("none.gdx"), item_type=ItemType.MODEL, filters={} + ) + + # TODO Should the ixmp4_backend get its own TypedDict for kwargs that only + # allows our expected values? If so, we probably need to adjust. + # Test raising with extra kwargs + with pytest.raises(ValueError, match="keyword arguments"): + ixmp4_backend.read_file( + path=Path("none.gdx"), + item_type=ItemType.MODEL, + init_items=True, + filters={"scenario": scenario}, + ) diff --git a/ixmp/tests/backend/test_ixmp4_io.py b/ixmp/tests/backend/test_ixmp4_io.py index b6c58dd16..2fd71176e 100644 --- a/ixmp/tests/backend/test_ixmp4_io.py +++ b/ixmp/tests/backend/test_ixmp4_io.py @@ -1,6 +1,7 @@ from pathlib import Path from typing import Literal, Union +import pandas as pd import pytest from ixmp.model.gams import gams_info @@ -29,6 +30,20 @@ def test__record_versions(container) -> None: assert indexset.name == "ixmp_version" assert indexset.domain == ["*", "*"] + # Test recording the package version of something that's never present + _record_versions(container=container, packages=["not installed"]) + indexset = container.getSets()[0] + print(indexset.records) + pd.testing.assert_frame_equal( + indexset.records[["uni_0", "uni_1"]], + pd.DataFrame( + { + "uni_0": pd.Categorical(["not installed"], ordered=True), + "uni_1": pd.Categorical(["(not installed)"], ordered=True), + } + ), + ) + def test__update_item_in_container(container) -> None: from ixmp.backend.ixmp4_io import _update_item_in_container From 8db95ab7f038d1e4f51a4ba97a90936feaff51a1 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Thu, 24 Apr 2025 14:23:16 +0200 Subject: [PATCH 71/72] Add docstring to new pytest hook --- ixmp/testing/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ixmp/testing/__init__.py b/ixmp/testing/__init__.py index e92c75de9..8ada44571 100644 --- a/ixmp/testing/__init__.py +++ b/ixmp/testing/__init__.py @@ -137,6 +137,7 @@ def pytest_report_header(config, start_path) -> str: # sound like what we need, but I couldn't quite get it to work. Instead, this is more # following https://pytest-with-eric.com/introduction/pytest-generate-tests/ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + """Parametrize tests for the two backend options.""" if "backend" in metafunc.fixturenames: markers = [m.name for m in metafunc.definition.iter_markers()] From 8e8b8708b3e2283294efef54e1782516bc7d5483 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Fri, 25 Apr 2025 11:48:59 +0200 Subject: [PATCH 72/72] Use the oldest gams version possible for gamsapi in CI --- .github/workflows/pytest.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index ee1e3f4a6..d85e21eb2 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -30,8 +30,7 @@ jobs: - "3.12" - "3.13" # Latest supported by ixmp gams-version: - # - "43.4.1" # First version including a macOS arm64 distribution - - "48.1.0" # Version matching the current gamsapi installation + - "45.7.0" # Oldest version of gamsapi allowed by pyproject.toml exclude: # Specific version combinations that are invalid / not to be used