diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 0fff3445d..d85e21eb2 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -30,7 +30,7 @@ jobs: - "3.12" - "3.13" # Latest supported by ixmp gams-version: - - "43.4.1" # First version including a macOS arm64 distribution + - "45.7.0" # Oldest version of gamsapi allowed by pyproject.toml exclude: # Specific version combinations that are invalid / not to be used diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8cb52c583..8d3a762c1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,6 +7,7 @@ repos: additional_dependencies: - genno - GitPython + - "ixmp4 @ git+https://github.com/iiasa/ixmp4@enh/remove-linked-items-when-removing-indexset-items" - nbclient - pandas-stubs - pytest diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index cf7aa4d47..6abf23f17 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -1,8 +1,15 @@ -.. Next release -.. ============ +Next release +============ + +All changes +----------- + +- Add :class:`.IXMP4Backend` as an alternative to :class:`.JDBCBackend` (:pull:`552`). + Please note: + + - This requires ixmp4, which is only compatible with Python 3.10 and above. + - :class:`.IXMP4Backend` is still missing several features and documentation, which will be added in subsequent PRs. -.. All changes -.. ----------- .. _v3.10.0: diff --git a/doc/api-backend.rst b/doc/api-backend.rst index 7872fe064..48fb4524d 100644 --- a/doc/api-backend.rst +++ b/doc/api-backend.rst @@ -61,6 +61,15 @@ Provided backends .. autofunction:: ixmp.backend.jdbc.start_jvm +.. currentmodule:: ixmp.backend.ixmp4 + +.. warning:: + `ixmp4 `_ only supports Python 3.10 and above. + Please ensure you are using a sufficiently recent Python version if you want to use IXMP4Backend. + If you are restricted to Python 3.9 and below, please use JDBCBackend instead. + +.. autoclass:: ixmp.backend.ixmp4.IXMP4Backend + .. currentmodule:: ixmp.backend diff --git a/ixmp/__init__.py b/ixmp/__init__.py index a3c881884..7ebd9e653 100644 --- a/ixmp/__init__.py +++ b/ixmp/__init__.py @@ -48,6 +48,11 @@ # Register Backends provided by ixmp BACKENDS["jdbc"] = JDBCBackend +if sys.version_info >= (3, 10): + from ixmp.backend.ixmp4 import IXMP4Backend + + BACKENDS["ixmp4"] = IXMP4Backend + # Register Models provided by ixmp MODELS.update( { diff --git a/ixmp/_config.py b/ixmp/_config.py index d5f9f139d..761e09580 100644 --- a/ixmp/_config.py +++ b/ixmp/_config.py @@ -68,6 +68,14 @@ def _platform_default(): "driver": "hsqldb", "path": next(_iter_config_paths())[1].joinpath("localdb", "default"), }, + "ixmp4-local": { + "class": "ixmp4", + "dsn": ( + "sqlite:///" + f"{Path.home().joinpath('.local', 'share', 'ixmp4', 'databases')}" + "/ixmp4-local.sqlite3" + ), + }, } diff --git a/ixmp/backend/base.py b/ixmp/backend/base.py index fd8ab0e54..2903618af 100644 --- a/ixmp/backend/base.py +++ b/ixmp/backend/base.py @@ -9,11 +9,15 @@ import pandas as pd +# TODO Import this from typing when dropping Python 3.11 +from typing_extensions import Unpack + from ixmp.backend import ItemType from ixmp.backend.io import s_read_excel, s_write_excel, ts_read_file from ixmp.core.platform import Platform from ixmp.core.scenario import Scenario from ixmp.core.timeseries import TimeSeries +from ixmp.util.ixmp4 import ReadKwargs, WriteFiltersKwargs, WriteKwargs class Backend(ABC): @@ -204,12 +208,13 @@ def set_node( """ @abstractmethod - def get_nodes(self) -> Iterable[tuple[str, Optional[str], str, str]]: + def get_nodes(self) -> Iterable[tuple[str, Optional[str], Optional[str], str]]: """Iterate over all nodes stored on the Platform. Yields ------- tuple + The members of each tuple are: ========= =========== === @@ -378,7 +383,9 @@ def get_units(self) -> list[str]: set_unit """ - def read_file(self, path: PathLike, item_type: ItemType, **kwargs) -> None: + def read_file( + self, path: PathLike, item_type: ItemType, **kwargs: Unpack[ReadKwargs] + ) -> None: """OPTIONAL: Read Platform, TimeSeries, or Scenario data from file. A backend **may** implement read_file for one or more combinations of the `path` @@ -418,16 +425,21 @@ def read_file(self, path: PathLike, item_type: ItemType, **kwargs) -> None: -------- write_file """ - s, filters = self._handle_rw_filters(kwargs.pop("filters", {})) + filters = kwargs["filters"] if "filters" in kwargs else WriteFiltersKwargs() + s, _ = self._handle_rw_filters(filters=filters) + _kwargs = {k: v for (k, v) in kwargs.items() if k != "filters"} + path = Path(path) if path.suffix in (".csv", ".xlsx") and item_type is ItemType.TS and s: - ts_read_file(s, path, **kwargs) + ts_read_file(s, path, **_kwargs) elif path.suffix == ".xlsx" and item_type is ItemType.MODEL and s: - s_read_excel(self, s, path, **kwargs) + s_read_excel(self, s, path, **_kwargs) else: raise NotImplementedError - def write_file(self, path: PathLike, item_type: ItemType, **kwargs) -> None: + def write_file( + self, path: PathLike, item_type: ItemType, **kwargs: Unpack[WriteKwargs] + ) -> None: """OPTIONAL: Write Platform, TimeSeries, or Scenario data to file. A backend **may** implement write_file for one or more combinations of the @@ -460,12 +472,19 @@ def write_file(self, path: PathLike, item_type: ItemType, **kwargs) -> None: """ # Use the "scenario" filter to retrieve the Scenario `s` to be written; reappend # any other filters - s, kwargs["filters"] = self._handle_rw_filters(kwargs.pop("filters", {})) + filters = kwargs["filters"] if "filters" in kwargs else WriteFiltersKwargs() + s, kwargs["filters"] = self._handle_rw_filters(filters=filters) xlsx_types = (ItemType.SET | ItemType.PAR, ItemType.MODEL) path = Path(path) if path.suffix == ".xlsx" and item_type in xlsx_types and s: - s_write_excel(self, s, path, item_type, **kwargs) + s_write_excel( + be=self, + s=s, + path=path, + item_type=item_type, + filters=kwargs["filters"], + ) else: # All other combinations of arguments raise NotImplementedError @@ -581,18 +600,20 @@ def preload(self, ts: TimeSeries) -> None: """OPTIONAL: Load `ts` data into memory.""" @staticmethod - def _handle_rw_filters(filters: dict) -> tuple[Optional[TimeSeries], dict]: + def _handle_rw_filters( + filters: WriteFiltersKwargs, + ) -> tuple[Optional[TimeSeries], WriteFiltersKwargs]: """Helper for :meth:`read_file` and :meth:`write_file`. The `filters` argument is unpacked if the 'scenarios' key is a single :class:`.TimeSeries` object. A 2-tuple is returned of the object (or :obj:`None`) and the remaining filters. """ - ts = None + ts: Optional[TimeSeries] = None filters = copy(filters) try: if isinstance(filters["scenario"], TimeSeries): - ts = filters.pop("scenario") + ts = filters["scenario"] except KeyError: pass # Don't modify filters at all @@ -820,7 +841,7 @@ def has_solution(self, s: Scenario) -> bool: """ @abstractmethod - def list_items(self, s: Scenario, type: str) -> list[str]: + def list_items(self, s: Scenario, type: Literal["set", "par", "equ"]) -> list[str]: """Return a list of names of items of `type`. Parameters @@ -832,7 +853,7 @@ def list_items(self, s: Scenario, type: str) -> list[str]: def init_item( self, s: Scenario, - type: str, + type: Literal["set", "par", "equ", "var"], name: str, idx_sets: Sequence[str], idx_names: Optional[Sequence[str]], @@ -860,7 +881,9 @@ def init_item( """ @abstractmethod - def delete_item(self, s: Scenario, type: str, name: str) -> None: + def delete_item( + self, s: Scenario, type: Literal["set", "par", "equ"], name: str + ) -> None: """Remove an item `name` of `type`. Parameters @@ -871,7 +894,9 @@ def delete_item(self, s: Scenario, type: str, name: str) -> None: """ @abstractmethod - def item_index(self, s: Scenario, name: str, sets_or_names: str) -> list[str]: + def item_index( + self, s: Scenario, name: str, sets_or_names: Literal["sets", "names"] + ) -> list[str]: """Return the index sets or names of item `name`. Parameters @@ -932,7 +957,7 @@ def item_get_elements( def item_set_elements( self, s: Scenario, - type: str, + type: Literal["par", "set"], name: str, elements: Iterable[tuple[Any, Optional[float], Optional[str], Optional[str]]], ) -> None: @@ -974,7 +999,9 @@ def item_set_elements( """ @abstractmethod - def item_delete_elements(self, s: Scenario, type: str, name: str, keys) -> None: + def item_delete_elements( + self, s: Scenario, type: Literal["par", "set"], name: str, keys + ) -> None: """Remove elements of item `name`. Parameters diff --git a/ixmp/backend/ixmp4.py b/ixmp/backend/ixmp4.py new file mode 100644 index 000000000..b2d1414c2 --- /dev/null +++ b/ixmp/backend/ixmp4.py @@ -0,0 +1,1210 @@ +import logging +from collections.abc import Generator, Iterable, MutableMapping, Sequence +from os import PathLike +from pathlib import Path +from typing import Any, Literal, Optional, Union, cast, overload + +import ixmp4.conf +import pandas as pd +from ixmp4 import DataPoint +from ixmp4 import Platform as ixmp4_platform +from ixmp4.core import Run +from ixmp4.core.optimization.equation import Equation, EquationRepository +from ixmp4.core.optimization.indexset import IndexSet, IndexSetRepository +from ixmp4.core.optimization.parameter import Parameter, ParameterRepository +from ixmp4.core.optimization.scalar import Scalar, ScalarRepository +from ixmp4.core.optimization.table import Table, TableRepository +from ixmp4.core.optimization.variable import Variable, VariableRepository +from ixmp4.data.backend.base import Backend as ixmp4_backend + +# TODO Import this from typing when dropping Python 3.11 +from typing_extensions import TypeAlias, Unpack + +from ixmp.backend import ItemType +from ixmp.backend.base import CachingBackend, ReadKwargs, WriteKwargs +from ixmp.backend.ixmp4_io import read_gdx_to_run, write_run_to_gdx +from ixmp.core.platform import Platform +from ixmp.core.scenario import Scenario +from ixmp.core.timeseries import TimeSeries + +log = logging.getLogger(__name__) + + +def _ensure_filters_values_are_lists(filters: dict[str, Union[Any, list[Any]]]) -> None: + """Convert singular values in `filters` to lists.""" + for k, v in filters.items(): + if not isinstance(v, list): + filters[k] = [v] + + +def _align_dtypes_for_filters( + filters: dict[str, list[Any]], data: pd.DataFrame +) -> None: + """Convert `filters` values to types enabling `data.isin()`.""" + # TODO Is this really the proper way to handle these types? + TYPE_MAP = {"object": str, "int64": int} + + for column_name in filters.keys(): + if not isinstance( + filters[column_name][0], TYPE_MAP[str(data.dtypes[column_name])] + ): + filters[column_name] = [ + TYPE_MAP[str(data.dtypes[column_name])](value) + for value in filters[column_name] + ] + + +class IXMP4Backend(CachingBackend): + """Backend using :mod:`ixmp4`.""" + + _platform: "ixmp4_platform" + _backend: "ixmp4_backend" + + # Mapping from ixmp.TimeSeries object to the underlying ixmp4.Run object (or + # subclasses of either) + index: dict[TimeSeries, "Run"] = {} + + # Mapping from platform name to ixmp4 Backend; this enables repeated __init__() + # calls for in-memory DBs + backend_index: dict[str, "ixmp4_backend"] = {} + + def __init__(self, _name: Optional[str] = None) -> None: + from ixmp4.conf.base import PlatformInfo + from ixmp4.data.backend import SqliteTestBackend + + DEFAULT_NAME = "ixmp4-local" + + if not _name: + # Warn about using the default backend + log.warning(f"Falling back to default SqliteBackend '{DEFAULT_NAME}'") + + name = _name if _name else DEFAULT_NAME + + # Create default platform information if None is registered + platform_info = ( + ixmp4.conf.settings.toml.get_platform(key=_name) + if _name + else PlatformInfo( + name=name, + dsn=( + "sqlite:///" + f"{Path.home().joinpath('.local', 'share', 'ixmp4', 'databases')}" + f"/{name}.sqlite3" + ), + ) + ) + + # Get the ixmp4 Backend object if one already exists or create and register it + try: + sqlite = self.backend_index[name] + except KeyError: + sqlite = SqliteTestBackend(platform_info) + + # Ensure default DB is setup correctly + # NOTE sqlalchemy catches existing tables to avoid superfluous CREATEs + sqlite.setup() + + self.backend_index[name] = sqlite + + # Instantiate and store + self._backend = sqlite + self._platform = ixmp4_platform(_backend=self._backend) + + # def __del__(self) -> None: + # self.close_db() + + def set_log_level(self, level: int) -> None: + # Set the level of the 'ixmp.backend.ixmp4' logger. Messages are handled by the + # 'ixmp' logger; see ixmp/__init__.py. + log.setLevel(level) + + # def get_log_level(self): + # return super().get_log_level() + + # Platform methods + + @classmethod + def handle_config(cls, args: Sequence, kwargs: MutableMapping) -> dict[str, Any]: + msg = "Unhandled {} args to Backend.handle_config(): {!r}" + if len(args): + raise ValueError(msg.format("positional", args)) + + info: dict[str, Any] = {} + try: + info["_name"] = kwargs["_name"] + except KeyError: + raise ValueError(f"Missing key '_name' for backend=ixmp4; got {kwargs}") + + return info + + # def close_db(self) -> None: + # self._backend.close() + + # Modifying the Platform + + def add_scenario_name(self, name: str) -> None: + self._platform.scenarios.create(name=name) + + # TODO clarify: ixmp4.Run doesn't have a name, but is the new ixmp.Scenario + # should it have a name or are these scenario names okay? + def get_scenario_names(self) -> Generator[str, None, None]: + for scenario in self._platform.scenarios.list(): + yield scenario.name + + def add_model_name(self, name: str) -> None: + self._platform.models.create(name=name) + + def get_model_names(self) -> Generator[str, None, None]: + for model in self._platform.models.list(): + yield model.name + + def get_scenarios( + self, default: bool, model: Optional[str], scenario: Optional[str] + ): + runs = self._platform.runs.list( + default_only=default, + model={"name": model} if model else None, + scenario={"name": scenario} if scenario else None, + ) + + for run in runs: + yield [ + run.model.name, + run.scenario.name, + # TODO What are we going to use for scheme in ixmp4? + "IXMP4Run", + run.is_default(), + # TODO Change this from being hardcoded + False, + # TODO Expose the creation, update and lock info in ixmp4 + # (if we get lock info) + "Some user", + "Some date", + "Some user", + "Some date", + "Some user", + "Some date", + # TODO Should Runs get .docs? + "Some docs", + run.version, + ] + + def set_unit(self, name: str, comment: str) -> None: + self._platform.units.create(name=name).docs = comment + + def get_units(self) -> list[str]: + return [unit.name for unit in self._platform.units.list()] + + def set_node( + self, + name: str, + parent: Optional[str] = None, + hierarchy: Optional[str] = None, + synonym: Optional[str] = None, + ) -> None: + if parent: + log.warning(f"Discarding parent parameter {parent}; unused in ixmp4.") + if synonym: + log.warning(f"Discarding synonym parameter {synonym}; unused in ixmp4.") + if hierarchy is None: + log.warning( + "IXMP4Backend.set_node() requires to specify 'hierarchy'! " + "Using 'None' as the meaningsless default." + ) + hierarchy = "None" + self._platform.regions.create(name=name, hierarchy=hierarchy) + + def get_nodes(self) -> list[tuple[str, None, str, str]]: + return [ + (region.name, None, region.name, region.hierarchy) + for region in self._platform.regions.list() + ] + + # Modifying the Run object + + def _index_and_set_attrs(self, run: "Run", ts: TimeSeries) -> None: + """Add `run` to index and update attributes of `ts`. + + Helper for `init()` and `get()`. + """ + # Add to index + self.index[ts] = run + + # Retrieve the version of the ixmp4.Run object + v = run.version + if ts.version is None: + # The default version was requested; update the attribute + ts.version = v + elif v != ts.version: # pragma: no cover + # Something went wrong on the ixmp4 side + raise RuntimeError(f"got version {v} instead of {ts.version}") + + # NOTE ixmp4 doesn't store 'scheme', which could in principle be anything. + # Since this is a shim between message_ix/ixmp4, I'm hardcoding this as default. + if isinstance(ts, Scenario): + if not ts.scheme: + ts.scheme = "IXMP4-MESSAGE" + + def init(self, ts: TimeSeries, annotation: str) -> None: + run = self._platform.runs.create(model=ts.model, scenario=ts.scenario) + # TODO either do log.warning that annotation is unused or + # run.docs = annotation + self._index_and_set_attrs(run, ts) + + def clone( + self, + s: Scenario, + platform_dest: Platform, + model: str, + scenario: str, + annotation: str, + keep_solution: bool, + first_model_year: Optional[int] = None, + ) -> Scenario: + # TODO either do log.warning that annotation is unused or + # run.docs = annotation + # TODO Should this be supported? + if first_model_year: + log.warning( + "ixmp4-backed Scenarios don't support cloning from " + "`first_model_year` only!" + ) + # TODO Is this enough? ixmp4 doesn't support cloning to a different platform at + # the moment, but maybe we can imitate this here? (Access + # platform_dest.backend._platform to create a new Run?) + cloned_run = self.index[s].clone( + model=model, scenario=scenario, keep_solution=keep_solution + ) + # Instantiate same class as the original object + cloned_s = s.__class__( + platform_dest, + model, + scenario, + version=cloned_run.version, + ) + self._index_and_set_attrs(cloned_run, cloned_s) + return cloned_s + + def get(self, ts: TimeSeries) -> None: + v = int(ts.version) if ts.version else None + run = self._platform.runs.get(model=ts.model, scenario=ts.scenario, version=v) + self._index_and_set_attrs(run, ts) + + def check_out(self, ts: TimeSeries, timeseries_only: bool) -> None: + log.warning("ixmp4 backed Scenarios/Runs don't need to be checked out!") + + def discard_changes(self, ts: TimeSeries) -> None: + log.warning( + "ixmp4 backed Scenarios/Runs are changed immediately, can't " + "discard changes. To avoid the need, be sure to start work on " + "fresh clones." + ) + + def commit(self, ts: TimeSeries, comment: str) -> None: + log.warning( + "ixmp4 backed Scenarios/Runs are changed immediately, no need for a commit!" + ) + + def clear_solution(self, s: Scenario, from_year: Optional[int] = None) -> None: + if from_year: + log.warning( + "ixmp4 does not support removing the solution only after a certain year" + ) + self.index[s].optimization.remove_solution() + + def set_as_default(self, ts: TimeSeries) -> None: + self.index[ts].set_as_default() + + # Information about the Run + + def run_id(self, ts: TimeSeries) -> int: + # TODO is the Run.version really what this function is after? + return self.index[ts].version + + def is_default(self, ts: TimeSeries) -> bool: + return self.index[ts].is_default() + + def has_solution(self, s: Scenario) -> bool: + return self.index[s].optimization.has_solution() + + # Handle optimization items + + Ixmp4ItemType: TypeAlias = Literal["scalar", "indexset", "set", "par", "equ", "var"] + + @overload + def _get_repo(self, s: Scenario, type: Literal["scalar"]) -> ScalarRepository: ... + + @overload + def _get_repo( + self, s: Scenario, type: Literal["indexset"] + ) -> IndexSetRepository: ... + + @overload + def _get_repo(self, s: Scenario, type: Literal["set"]) -> TableRepository: ... + + @overload + def _get_repo(self, s: Scenario, type: Literal["par"]) -> ParameterRepository: ... + + @overload + def _get_repo(self, s: Scenario, type: Literal["equ"]) -> EquationRepository: ... + + @overload + def _get_repo(self, s: Scenario, type: Literal["var"]) -> VariableRepository: ... + + # NOTE Type hints here ensure the function body is checked + def _get_repo( + self, s: Scenario, type: Ixmp4ItemType + ) -> Union[ + IndexSetRepository, + TableRepository, + ScalarRepository, + ParameterRepository, + EquationRepository, + VariableRepository, + ]: + """Return the repository of items of `type` belonging to `s`.""" + if type == "scalar": + return self.index[s].optimization.scalars + if type == "indexset": + return self.index[s].optimization.indexsets + if type == "set": + return self.index[s].optimization.tables + elif type == "par": + return self.index[s].optimization.parameters + elif type == "equ": + return self.index[s].optimization.equations + else: # "var" + return self.index[s].optimization.variables + + def init_item( + self, + s: Scenario, + type: Literal["set", "par", "equ", "var"], + name: str, + idx_sets: Sequence[str], + idx_names: Optional[Sequence[str]], + ) -> None: + # TODO how are scalars created? Scalars require a value in ixmp4 + # In base::item_get_elements, it sounds like "equ" and "var" can also target + # scalars, whereas below, inspired from jdbc, I'm only linking "par" to scalars + if type == "set" and len(idx_sets) == 0: + indexset_repo = self._get_repo(s=s, type="indexset") + indexset_repo.create(name=name) + elif type == "par" and len(idx_sets) == 0: + # NOTE ixmp4 requires scalars to get value and unit upon creation, so the + # shim does that when `item_set_elements()` targets a scalar + return None + else: + repo = self._get_repo(s=s, type=type) + repo.create( + name=name, + constrained_to_indexsets=list(idx_sets), + column_names=list(idx_names) if idx_names else None, + ) + + def list_items(self, s: Scenario, type: Literal["set", "par", "equ"]) -> list[str]: + if type == "set": + indexset_repo = self._get_repo(s=s, type="indexset") + set_repo = self._get_repo(s=s, type=type) + return [item.name for item in indexset_repo.list()] + [ + item.name for item in set_repo.list() + ] + else: + repo = self._get_repo(s=s, type=type) + return [item.name for item in repo.list()] + + def _find_item( + self, + s: Scenario, + name: str, + types: tuple[Ixmp4ItemType, ...] = ( + "scalar", + "indexset", + "set", + "par", + "equ", + "var", + ), + ) -> tuple[ + Ixmp4ItemType, + Union[ + "Scalar", + "IndexSet", + "Table", + "Equation", + "Variable", + "Parameter", + ], + ]: + """Find item `name` in Scenario `s`. + + Parameters + ---------- + s : Scenario + The Scenario in which to search for `name`. + name : str + The name to search for. + types : tuple[`Ixmp4ItemType`, ...], optional + If provided, search only specific types for `name`. For example, + types=("indexset", "set"). Default: tuple of all types. + + Returns + ------- + tuple[`Ixmp4ItemType`, Scalar | IndexSet | Table | Parameter | Equation | Variable] + A tuple containing the string item type and the item itself. + + Raises + ------ + KeyError + If no item with `name` and type in `types` can be found in `s`. + """ # noqa: E501 + # NOTE this currently assumes that `name` will only be present once among + # Tables, Parameters, Equations, Variables. This is in line with the assumption + # made in the Java backend, but ixmp4 enforces no such constraint. + _type: Optional[Literal["scalar", "indexset", "set", "par", "equ", "var"]] = ( + None + ) + _item: Optional[ + Union[ + "Scalar", + "IndexSet", + "Table", + "Equation", + "Variable", + "Parameter", + ] + ] = None + for type in types: + repo = self._get_repo(s=s, type=type) + item_list = repo.list(name=name) + if ( + len(item_list) == 1 + ): # ixmp4 enforces names to be unique among individual item classes + _type = type + _item = item_list[0] + break + + if _item is None or _type is None: + raise KeyError(f"No item called '{name}' found on this Scenario!") + else: + return (_type, _item) + + @overload + def _get_item(self, s: Scenario, name: str, type: Literal["scalar"]) -> Scalar: ... + + @overload + def _get_item( + self, s: Scenario, name: str, type: Literal["indexset"] + ) -> IndexSet: ... + + @overload + def _get_item(self, s: Scenario, name: str, type: Literal["set"]) -> Table: ... + + @overload + def _get_item(self, s: Scenario, name: str, type: Literal["par"]) -> Parameter: ... + + @overload + def _get_item(self, s: Scenario, name: str, type: Literal["equ"]) -> Equation: ... + + @overload + def _get_item(self, s: Scenario, name: str, type: Literal["var"]) -> Variable: ... + + def _get_item( + self, s: Scenario, name: str, type: Ixmp4ItemType + ) -> Union[ + "Scalar", + "IndexSet", + "Table", + "Equation", + "Variable", + "Parameter", + ]: + """Retrieve item `name` with `type` from `s`. + + Returns + ------- + Scalar | IndexSet | Table | Parameter | Equation | Variable + Based on `type`. + """ + return self._get_repo(s=s, type=type).get(name=name) + + def _get_indexset_or_table( + self, s: Scenario, name: str + ) -> Union["IndexSet", "Table"]: + """Get `name` from Scenario `s`. + + Try first if `name` is an IndexSet. Get it as a Table if it isn't. + """ + try: + indexset_repo = self._get_repo(s=s, type="indexset") + return indexset_repo.get(name=name) + except IndexSet.NotFound: + table_repo = self._get_repo(s=s, type="set") + return table_repo.get(name=name) + + def item_index( + self, s: Scenario, name: str, sets_or_names: Literal["sets", "names"] + ) -> list[str]: + _, item = self._find_item(s=s, name=name) + # NOTE Using isinstance allows adequate attribute access + if ( + isinstance(item, IndexSet) + or isinstance(item, Scalar) + or (isinstance(item, (Variable, Equation)) and item.indexsets is None) + ): + return cast(list[str], []) + else: + if sets_or_names == "names" and item.column_names is None: + log.debug( + f"Requested {sets_or_names}, but these are None for item " + f"{item.name}, falling back on (Index)Set names!" + ) + assert item.indexsets # Could only be None for Variables & Equations + return ( + item.column_names + if item.column_names and sets_or_names == "names" + else item.indexsets + ) + + def _add_data_to_set( + self, + s: Scenario, + name: str, + key: Union[str, list[str]], + comment: Optional[str] = None, + ) -> None: + """Add data `key` to `name` in Scenario `s`. + + Parameters + ---------- + s : Scenario + The Scenario hosting the item. + name : str + The name of the item to add data to. + key : str | list[str] + The data to add. + + ATTENTION: if `key` is a str, we're assuming `name` means an IndexSet; + if `key` is a list, we're assuming `name` means a Table. + comment: str, optional + A message to store with the data addition. Unused by ixmp4. + """ + if comment: + log.warning( + "`comment` currently unused with ixmp4 when adding data to Tables." + ) + # Assumption: if key is just one value, we're dealing with an IndexSet + # NOTE E.g. westeros_addon_technologies in message_ix calls + # `scenario.add_set("addon", "po_turbine")` for a 1-D Table called "addon". This + # only works now because ixmp.Scenario.add_set() converts str keys for Tables to + # [key] before adding them. We would need to replicate that or adapt the + # decision logic here should we drop ixmp.Scenario. + if isinstance(key, str): + # NOTE ixmp_source silently ignores duplicate data; replicate here + # This could be improved by adding data without loading the indexset first, + # but this requires users to ensure their data are valid + indexset = self.index[s].optimization.indexsets.get(name=name) + if key not in indexset.data: + indexset.add(key) + else: + table = self.index[s].optimization.tables.get(name=name) + # TODO should we enforce in ixmp4 that when constrained_to_indexsets + # contains duplicate values, column_names must be provided? + keys = table.column_names or table.indexsets + data_to_add = pd.DataFrame({keys[i]: [key[i]] for i in range(len(key))}) + + # Silently ignore duplicate data, see NOTE above + data_to_add = data_to_add[~data_to_add.isin(values=table.data).all(axis=1)] + + table.add(data=data_to_add) + + def _create_scalar( + self, + s: Scenario, + name: str, + value: float, + unit: Optional[str], + comment: Optional[str] = None, + ) -> None: + """Create the Scalar `name` in Scenario `s`. + + Parameters + ---------- + s : Scenario + The Scenario hosting the Scalar. + name : str + The name of the Scalar. + value : float + The value of the Scalar. + unit : str, optional + The unit of the Scalar. + comment: str, optional + A message to explain what this Scalar means. + """ + scalar = self.index[s].optimization.scalars.create( + name=name, value=value, unit=unit + ) + if comment: + scalar.docs = comment + + def _add_data_to_parameter( + self, + s: Scenario, + name: str, + key: Union[str, list[str]], + value: float, + unit: str, + comment: Optional[str] = None, + ) -> None: + """Add data `key` to the Parameter `name` in Scenario `s`. + + Parameters + ---------- + s : Scenario + The Scenario hosting the Parameter. + name : str + The name of the Parameter. + key : str | list[str] + The data to add to the Parameter. + value : float + The value of the Parameter. + unit : str + The unit of the Parameter. + comment: str, optional + A message to store with the data addition. Unused by ixmp4. + """ + if comment: + log.warning( + "`comment` currently unused with ixmp4 when adding data to Parameters." + ) + parameter = self.index[s].optimization.parameters.get(name=name) + # TODO there's got to be a better way for handling possible lists + if isinstance(key, str): + key = [key] + + keys = parameter.column_names or parameter.indexsets + data_to_add: dict[str, Union[list[float], list[str]]] = { + keys[i]: [key[i]] for i in range(len(key)) + } + data_to_add["values"] = [value] + data_to_add["units"] = [unit] + + parameter.add(data=data_to_add) + + def item_set_elements( + self, + s: Scenario, + type: Literal["par", "set"], + name: str, + elements: Iterable[tuple[Any, Optional[float], Optional[str], Optional[str]]], + ) -> None: + for key, value, unit, comment in elements: + if type == "set": + self._add_data_to_set(s=s, name=name, key=key, comment=comment) + else: + if key is None: + assert isinstance(value, float), ( + "Creating a Scalar requires a value!" + ) + self._create_scalar( + s=s, name=name, value=value, unit=unit, comment=comment + ) + else: + assert isinstance(value, float), ( + "Adding data to a Parameter requires a value!" + ) + assert isinstance(unit, str), ( + "Adding data to a Parameter requires a unit!" + ) + self._add_data_to_parameter( + s=s, name=name, key=key, value=value, unit=unit, comment=comment + ) + + def _get_set_data( + self, s: Scenario, name: str, filters: Optional[dict[str, list[Any]]] = None + ) -> Union[pd.Series, pd.DataFrame]: + """Get the data stored in `name` in `s`. + + Parameters + ---------- + s : Scenario + The Scenario hosting the item. + name : str + The name of the item. + filters : dict[str, list[Any]], optional + Filters to apply to the data. If present, return only matching data. + Default: None. + """ + item = self._get_indexset_or_table(s=s, name=name) + + if isinstance(item, Table): + columns = item.column_names or item.indexsets + df = pd.DataFrame(item.data, columns=columns) + return df[df.isin(values=filters).all(axis=1)] if filters else df + else: + series = pd.Series(item.data) + return series[series.isin(values=filters[name])] if filters else series + + def item_get_elements( + self, + s: Scenario, + type: Literal["equ", "par", "set", "var"], + name: str, + filters: Optional[dict[str, Any]] = None, + ) -> Union[dict[str, Any], pd.Series, pd.DataFrame]: + if type == "set": + return self._get_set_data(s=s, name=name, filters=filters) + # TODO this is not handling scalars at the moment, but maybe try with type, + # except NotFound, try scalar? + else: + item = self._get_item(s=s, name=name, type=type) + + # TODO What about Scalars? How does message_ix load them? + + data = pd.DataFrame(item.data) + + if data.empty: + # Set correct columns + columns = item.column_names or item.indexsets + unindexed_columns = ( + ["value", "unit"] if type == "par" else ["lvl", "mrg"] + ) + if columns: + columns.extend(unindexed_columns) + + return pd.DataFrame( + columns=columns if bool(columns) else unindexed_columns + ) + + # Apply filters if requested + if filters: + # isin() won't consider int(700) to be in ['700'], etc + _ensure_filters_values_are_lists(filters=filters) + _align_dtypes_for_filters(filters=filters, data=data) + + data = data[data.isin(values=filters)[filters.keys()].all(axis=1)] + + # Rename columns + # NOTE the ixmp.report(er) expects these names, unfortunately; see + # ixmp.report.util::keys_for_quantity() + renames = ( + {"values": "value", "units": "unit"} + if type == "par" + else {"levels": "lvl", "marginals": "mrg"} + ) + data.rename(columns=renames, inplace=True) + + return data + + def item_delete_elements( + self, + s: Scenario, + type: Literal["par", "set"], + name: str, + keys: Iterable[Sequence[str]], + ) -> None: + if type == "set": + item = self._get_indexset_or_table(s=s, name=name) + + if isinstance(item, IndexSet): + # NOTE We might have to expose IndexSet._data_type to cast correctly + data = pd.DataFrame(keys, columns=[item.name]) + item.remove(data=cast(list[str], data[item.name].to_list())) + else: + # TODO can we assume that keys follow same order as indexsets/columns? + columns = item.column_names or item.indexsets + data = pd.DataFrame(keys, columns=columns) + item.remove(data=data) + else: + parameter = self._get_item(s=s, name=name, type="par") + columns = parameter.column_names or parameter.indexsets + data = pd.DataFrame(keys, columns=columns) + parameter.remove(data=data) + + def delete_item( + self, + s: Scenario, + type: Literal["set", "par", "equ", "indexset"], + name: str, + ) -> None: + # NOTE We need to allow 'set' and 'indexset' for type for backward compatibility + if type == "set": + # type will always be either "indexset" or "set" + type, _ = self._find_item(s=s, name=name, types=("indexset", "set")) # type: ignore[assignment] + + repo = self._get_repo(s=s, type=type) + repo.delete(item=name) + + # NOTE The name 'cat_`name`' is used for backward compatibility with the JDBC, where + # such names are hardcoded. 'cat' means 'category' and should be expanded for + # clarity in the future. + def cat_set_elements( + self, + ms: Scenario, + name: str, + cat: str, + keys: Union[str, Sequence[str]], + is_unique: bool, + ) -> None: + """Add data to a category mapping. + + For the ixmp4.Table or IndexSet `name`, define a category as a new IndexSet + called 'type_`name`' (if it doesn't exist already) and add `cat` to it. Then, + define a new Table 'cat_`name`' storing one column for `keys` and one for + 'categories'. + + Parameters + ---------- + name : str + Name of the category mapping Table. + cat : str + Name of the category within `name`. + keys : iterable of str or list of str + Keys to add to `cat`. + is_unique : bool + If :obj:`True`: + + - `keys` **must** contain only one key. + - The Backend **must** remove any existing member of `cat`, so that it has + only one element. + """ + column_name: Optional[str] = None + + # Categories can be based on IndexSets directly or on 1-d Tables + try: + # Most should be based on IndexSets, try that first + indexset = self.index[ms].optimization.indexsets.get(name=name) + indexset_name = indexset.name + except IndexSet.NotFound: + # We're dealing with a Table + table = self.index[ms].optimization.tables.get(name=name) + + # Ensure the Table's dimensions are correct before setting variables + assert len(table.indexsets) == 1 + indexset_name = table.indexsets[0] + column_name = table.name + + # Special treatment for 'technology' for historical reasons: + if name == "technology": + name = "tec" + # NOTE Of the default items, all categories except 'cat_addon' are based on + # IndexSets. 'cat_addon' is based on the 1-d Table 'addon', which is why it also + # doesn't follow the naming convention. Usually, the column names are + # 'type_' and '', but for 'cat_addon', it's 'type_addon' and + # 'technology_addon'. + elif name == "addon": + column_name = "technology_addon" + + # Get or create the 'type_name' indexset and 'cat_name' table + try: + category_indexset = self.index[ms].optimization.indexsets.get( + name=f"type_{name}" + ) + except IndexSet.NotFound: + category_indexset = self.index[ms].optimization.indexsets.create( + name=f"type_{name}" + ) + + try: + category_table = self.index[ms].optimization.tables.get(name=f"cat_{name}") + except Table.NotFound: + category_table = self.index[ms].optimization.tables.create( + name=f"cat_{name}", + constrained_to_indexsets=[indexset_name, category_indexset.name], + column_names=[column_name, category_indexset.name] + if column_name + else None, + ) + + # Convert for convenience + if isinstance(keys, str): + keys = [keys] + + # Ensure proper treatment when is_unique is True + if is_unique: + if len(keys) > 1: + raise ValueError("One can only add one element to a unique category!") + # Ensure data contains no data except that which we're going to add + # NOTE if category_table contains data linked to elements existing now, this + # will lead to DataValidationErrors when adding data to the table. Also, + # ixmp4 might safeguard against this when implementing remove() functions + if category_indexset.data: + # Remove all existing data so that only the single provided element will + # be stored in the indexset + category_indexset.remove(category_indexset.data) + + # Add data to both objects + if cat not in category_indexset.data: + category_indexset.add(data=cat) + data = {column_name: keys} if column_name else {indexset_name: keys} + data[category_indexset.name] = [cat] * len(keys) + + category_table.add(data=data) + + # TODO In cat_set_elements, we change e.g. cat_technology to cat_tec. Do we need the + # same here or do we expect user code to call this with name == "tec" if they're + # interested in "technology"? + def cat_get_elements(self, ms: Scenario, name: str, cat: str) -> list[str]: + data = pd.DataFrame( + self.index[ms].optimization.tables.get(name=f"cat_{name}").data + ) + # This assumes there are only two columns: 'type_{name}' and the category name + columns = data.columns.to_list() + columns.remove(f"type_{name}") + + return data[data[f"type_{name}"] == cat][columns[0]].astype(str).to_list() + + # TODO In cat_set_elements, we change e.g. cat_technology to cat_tec. Do we need the + # same here or do we expect user code to call this with name == "tec" if they're + # interested in "technology"? + def cat_list(self, ms: Scenario, name: str) -> list[str]: + category_indexset = self.index[ms].optimization.indexsets.get(f"type_{name}") + return cast(list[str], category_indexset.data) + + def set_data( + self, + ts: TimeSeries, + region: str, + variable: str, + data: dict[int, float], + unit: str, + subannual: str, + meta: bool, + ) -> None: + log.warning("Parameter `meta` for set_data() currently unused by ixmp4!") + log.warning("Parameter `subannual` for set_data() currently unused by ixmp4!") + + # Construct dataframe as ixmp4 expects it + years = data.keys() + values = data.values() + number_of_years = len(years) + regions = [region] * number_of_years + variables = [variable] * number_of_years + units = [unit] * number_of_years + _data = list(zip(years, values, regions, variables, units)) + + # Add timeseries dataframe + # TODO Is this really the only type we're interested in? + self.index[ts].iamc.add( + pd.DataFrame( + _data, columns=["step_year", "value", "region", "variable", "unit"] + ), + type=DataPoint.Type.ANNUAL, + ) + + def get_data( + self, + ts: TimeSeries, + region: Sequence[str], + variable: Sequence[str], + unit: Sequence[str], + year: Sequence[str], + ) -> Generator[tuple, Any, None]: + data = self.index[ts].iamc.tabulate( + region={"name__in": region}, + variable={"name__in": variable}, + unit={"name__in": unit}, + ) + + # Protect against empty data + if not data.empty: + # TODO depending on data["type"], a different column name will contain the + # "year" values: + # step_year if type is ANNUAL or CATEGORICAL + # step_category if type is CATEGORICAL + # step_datetime if type is DATETIME + # We're only adding ANNUAL above for now, so let's assume that + data = data.loc[data["step_year"].isin(year)] + + # Select only the columns we're interested in + # TODO the ixmp4 docstrings sounds like region, variable, and unit are not + # part of the df? + data = data[["region", "variable", "unit", "step_year", "value"]] + + # TODO Why would we iterate and yield tuples instead of returning the whole df? + for row in data.itertuples(index=False, name=None): + yield row + + # Handle timeslices + + # def set_timeslice(self, name: str, category: str, duration: float) -> None: + # return super().set_timeslice(name, category, duration) + + # Handle I/O + + def write_file( + self, path: PathLike, item_type: ItemType, **kwargs: Unpack[WriteKwargs] + ) -> None: + """Write Platform, TimeSeries, or Scenario data to file. + + IXMP4Backend supports writing to: + + - ``path='*.gdx', item_type=ItemType.SET | ItemType.PAR``. + - ``path='*.csv', item_type=TS``. The `default` keyword argument is + **required**. + + Other parameters + ---------------- + filters : dict of dict of str + Restrict items written. The following filters may be used: + + - model : list of str + - scenario : list of str | Scenario + - variable : list of str + - region: list of str + - unit: list of str + - default : bool + If :obj:`True`, only data from TimeSeries versions with + :meth:`.TimeSeries.set_as_default` are written. + - export_all_runs: bool + Whether to export all existing model+scenario run combinations. + + See also + -------- + .Backend.write_file + """ + try: + # Call the default implementation, e.g. for .xlsx + super().write_file(path, item_type, **kwargs) + except NotImplementedError: + pass + else: + return + + # NOTE Would like to use proper paths in signature already, but comply with base + # class for now + _path = Path(path) + + ts, filters = self._handle_rw_filters(kwargs["filters"]) + if _path.suffix == ".gdx" and item_type is ItemType.SET | ItemType.PAR: + # NOTE if we keep the TypedDicts and can't pop items, we might have to + # adjust more checks like this. Alternatively, use explicit keyword + # parameters for functions or convert kwargs to dicts in here to enable pop. + if len(filters) > 1: # pragma: no cover + raise NotImplementedError("write to GDX with filters") + elif not isinstance(ts, Scenario): # pragma: no cover + raise ValueError("write to GDX requires a Scenario object") + + write_run_to_gdx( + run=self.index[ts], + file_name=_path, + record_version_packages=kwargs["record_version_packages"], + container_data=kwargs["container_data"], + ) + elif _path.suffix == ".csv" and item_type is ItemType.TS: + models = filters.pop("model") + # NOTE this is what we get for not differentiating e.g. scenario vs + # scenarios in filters... + scenarios = set(cast(list[str], filters.pop("scenario"))) + variables = filters.pop("variable") + units = filters.pop("unit") + regions = filters.pop("region") + default = filters.pop("default") + # TODO (How) Should we include this? + # export_all_runs = filters.pop("export_all_runs") + + data = self._backend.runs.tabulate( + model={"name__in": models}, + scenario={"name__in": scenarios}, + iamc={ + "region": {"name__in": regions}, + "variable": {"name__in": variables}, + "unit": {"name__in": units}, + }, + default_only=default, + ) + data.to_csv(path_or_buf=_path) + + else: + raise NotImplementedError + + def read_file( + self, path: PathLike, item_type: ItemType, **kwargs: Unpack[ReadKwargs] + ) -> None: + """Read Platform, TimeSeries, or Scenario data from file. + + IXMP4Backend supports reading from: + + - ``path='*.gdx', item_type=ItemType.MODEL``. The keyword arguments + `check_solution`, `comment`, `equ_list`, and `var_list` are **required**. + + Other parameters + ---------------- + check_solution : bool + If True, raise an exception if the GAMS solver did not reach optimality. + (Only for MESSAGE-scheme Scenarios.) + comment : str + Comment added to Scenario when importing the solution. + equ_list : list of str + Equations to be imported. + var_list : list of str + Variables to be imported. + filters : dict of dict of str + Restrict items read. + + See also + -------- + .Backend.read_file + """ + try: + # Call the default implementation, e.g. for .xlsx + super().read_file(path, item_type, **kwargs) + except NotImplementedError: + pass + else: + return + + _path = Path(path) + + # TODO handle case when filters is not present + ts, _ = self._handle_rw_filters(kwargs["filters"]) + # Convert to normal dict to allow removal of keys + _kwargs = {k: v for (k, v) in kwargs.items() if k != "filters"} + + if _path.suffix == ".gdx" and item_type is ItemType.MODEL: + kw = {"check_solution", "comment", "equ_list", "var_list"} + + if not isinstance(ts, Scenario): + raise ValueError("read from GDX requires a Scenario object") + elif set(_kwargs.keys()) != kw: + raise ValueError( + f"keyword arguments {_kwargs.keys()} do not match required {kw}" + ) + + check_solution = bool(_kwargs.pop("check_solution")) + comment = str(_kwargs.pop("comment")) + + # message_ix/ixmp4 uses these lists to set the default items to read, so + # these will always be true + equ_list = cast(list[str], _kwargs.pop("equ_list")) + var_list = cast(list[str], _kwargs.pop("var_list")) + + read_gdx_to_run( + run=self.index[ts], + result_file=_path, + equ_list=equ_list, + var_list=var_list, + comment=comment, + check_solution=check_solution, + ) + + else: + raise NotImplementedError(path, item_type) + + # The below methods of base.Backend are not yet implemented + def _ni(self, *args, **kwargs): + raise NotImplementedError + + delete = _ni + delete_geo = _ni + delete_meta = _ni + get_doc = _ni + get_geo = _ni + get_meta = _ni + get_timeslices = _ni + last_update = _ni + remove_meta = _ni + set_doc = _ni + set_geo = _ni + set_meta = _ni + set_timeslice = _ni diff --git a/ixmp/backend/ixmp4_io.py b/ixmp/backend/ixmp4_io.py new file mode 100644 index 000000000..358cad7c5 --- /dev/null +++ b/ixmp/backend/ixmp4_io.py @@ -0,0 +1,431 @@ +import logging +from collections.abc import Iterable +from pathlib import Path +from typing import Literal, Optional, TypeVar, Union, cast + +import gams.transfer as gt +import pandas as pd +from ixmp4.core import Run +from ixmp4.core.optimization.base import Lister +from ixmp4.core.optimization.equation import Equation +from ixmp4.core.optimization.indexset import IndexSet, IndexSetRepository +from ixmp4.core.optimization.parameter import Parameter +from ixmp4.core.optimization.scalar import Scalar +from ixmp4.core.optimization.table import Table +from ixmp4.core.optimization.variable import Variable +from ixmp4.data.abstract.optimization.equation import Equation as AbstractEquation +from ixmp4.data.abstract.optimization.variable import Variable as AbstractVariable + +from ixmp.model.gams import gams_info +from ixmp.util.ixmp4 import ContainerData + +log = logging.getLogger(__name__) + +# Type variable that can be any one of these 6 types, but not a union of 2+ of them +Item4 = TypeVar("Item4", Equation, IndexSet, Parameter, Scalar, Table, Variable) + + +def _domain(item: Item4) -> Optional[list[str]]: + """Return domain for `item`. + + For IndexSets and Scalars, this is :obj:`None`. + + For all others, this is `item.indexsets`. + """ + if isinstance(item, (IndexSet, Scalar)): + return None + else: + return item.indexsets + + +def _records( + item: Item4, +) -> Union[ + float, + Union[list[float], list[int], list[str]], + dict[str, Union[list[float], list[int], list[str]]], + None, +]: + """Return records for `item`. + + For Scalars, this is `item.value`. + + For empty other items, this is :obj:`None`. + + For IndexSets and Tables, this is `item.data`. + + For Parameters, this is `item.data` without the column 'units'. + + For Equations and Variables, this is `item.data` + without the columns 'levels' and 'marginals'. + """ + if isinstance(item, Scalar): + return item.value + elif not len(item.data): + return None + + if isinstance(item, (IndexSet, Table)): + return item.data + + result = item.data + + # Pop items not to be stored + for name in { + Equation: ["levels", "marginals"], + Parameter: ["units"], + Variable: ["levels", "marginals"], + }[type(item)]: + result.pop(name) + + return result + + +def _ensure_correct_item_order(items: list[Item4], repo: Lister) -> list[Item4]: + """Reorder items to ensure the GDX file is written correctly. + + gamsapi stores all unique elements of all items in a single list internally. If item + A adds elements that item B shares, item B is not adding them again. If B's elements + are then requested, they are returned starting with those in A in their order, + disrupting expectations. + """ + if isinstance(repo, IndexSetRepository): + # Move all indexsets called 'type_*' to the end of the list + for type_indexset in list( + filter(lambda item: item.name.startswith("type_"), items) + ): + items.remove(type_indexset) + items.append(type_indexset) + + return items + + +def _align_records_and_domain( + item: Item4, records: dict[str, Union[list[float], list[int], list[str]]] +) -> dict[str, Union[list[float], list[int], list[str]]]: + """Align the order of `records.keys()` with domain of `item`.""" + # This function will only be called for these types + assert isinstance(item, (Table, Parameter, Variable, Equation)) + + # `records` may contain keys that are column_names, but not indexsets + domain_order = item.column_names or item.indexsets + + if not domain_order: + # This could happen for Variables or Equations + return records + else: + # Parameters contain values that we want to keep + if isinstance(item, Parameter): + domain_order.append("values") + + return {key: records[key] for key in domain_order} + + +def _update_item_in_container(container: gt.Container, item: ContainerData) -> None: + """Update `item` in `container`. + + Note that this overwrites existing records of `item`. + """ + # NOTE This is not updating the comment + + kwargs = dict(name=item.name, domain=item.domain, records=item.records) + + if item.kind in ("IndexSet", "Table"): + container.addSet(**kwargs) + elif item.kind in ("Scalar", "Parameter"): + container.addParameter(**kwargs) + elif item.kind == "Equation": + # NOTE This was hardcoded in ixmp_source the same way; not sure how the + # 'type' in the container affects anything + kwargs["type"] = "E" + container.addEquation(**kwargs) + else: # Variable + container.addVariable(**kwargs) + + +def _add_items_to_container( + container: gt.Container, items: list[ContainerData] +) -> None: + """Add or update `items` to/in `container`.""" + if not items: + return # Nothing to be done + + for item in items: + # Identify the corresponding gams.transfer class + klass = { + "IndexSet": gt.Set, + "Parameter": gt.Parameter, + "Scalar": gt.Parameter, + "Table": gt.Set, + "Variable": gt.Variable, + "Equation": gt.Equation, + }[item.kind] + + # NOTE A container may already contain some data from the MESSAGE-specific setup + # This seems to imply a UniqueConstraint over all item names, though. + if not container.hasSymbols(item.name): + # NOTE This was hardcoded in ixmp_source the same way; not sure how the + # 'type' in the container affects anything + kwargs = {"type": "E"} if item.kind == "Equation" else {} + + # Create an item instance linked to the container + klass( + container=container, + name=item.name, + domain=item.domain, + records=item.records, + # Optional, but must be str + description=item.docs or "", + **kwargs, + ) + else: + _update_item_in_container(container=container, item=item) + + +def _convert_ixmp4_items_to_containerdata(items: list[Item4]) -> list[ContainerData]: + """Convert list of ixmp4 `items` to ContainerData.""" + if not items: + return [] # Nothing to be done + + # Identify the ixmp4 class name + kind = cast( + Literal["Scalar", "IndexSet", "Table", "Parameter", "Equation", "Variable"], + type(items[0]).__name__, + ) + + container_items: list[ContainerData] = [] + + for item in items: + records = _records(item=item) + # The order of keys in 'domain' is used by gamsapi to overwrite the order of + # keys in 'records', so make sure they are aligned: + # NOTE ixmp4 ensures that all _records.keys() are in _domain and vice-versa + if isinstance(records, dict): + records = _align_records_and_domain(item=item, records=records) + + container_items.append( + ContainerData( + name=item.name, + kind=kind, + records=records, + domain=_domain(item), + docs=item.docs, + ) + ) + + return container_items + + +def _record_versions(container: gt.Container, packages: list[str]) -> None: + """Store Python package versions as set elements to be written to GDX. + + The values are stored in a 2-dimensional set named ``ixmp_version``, where the + first element is the package name, and the second is its version according to + :func:`importlib.metadata.version`). If the package is not installed, the + string "(not installed)" is stored. + """ + from importlib.metadata import PackageNotFoundError, version + + name = "ixmp_version" + + # Each tuple consists of (package_name, package_version) + versions: list[tuple[str, str]] = [] + + # Handle each identified package + for package in packages: + try: + # Retrieve the version; replace characters not supported by GAMS + package_version = version(package).replace(".", "-") + except PackageNotFoundError: + package_version = "(not installed)" # Not installed + + versions.append((package, package_version)) + + # Add Set to the container + container.addSet(name=name, domain=["*", "*"], records=versions) + + +def write_run_to_gdx( + run: Run, + file_name: Path, + container_data: list[ContainerData], + record_version_packages: list[str], + include_variables_and_equations: bool = False, +) -> None: + """Write data from the `run` to a GDX file at `file_name` via a GAMS Container. + + Parameters + ---------- + run : :class:`ixmp4.core.Run` + The Run in which the Scenario data is stored. + file_name: :obj:`Path` + The location at which the GDX file should be written. + container_data : list of :class:`ixmp.util.ixmp4.ContainerData`. + Data that should also be written to the GDX file + in the form of ContainerData. + record_version_packages : list of str + A list of package names for which versions will be stored in the GDX file. + include_variables_and_equations: bool, optional + Flag to indicate whether Variabels and Equations should be written to the GDX + file. + Default: :obj:`False`. + """ + + # TODO How to deal with [*] Set? That seems to be handled by GAMS automatically. + + # Define the container + container = gt.Container(system_directory=str(gams_info().system_dir)) + + repository: list[Lister] = [ + run.optimization.indexsets, + run.optimization.scalars, + run.optimization.tables, + run.optimization.parameters, + run.optimization.variables, + run.optimization.equations, + ] + idx = slice(None) if include_variables_and_equations else slice(-2) + for r in repository[idx]: + # Reorder items if necessary for GAMS to successfully read the GDX + ixmp4_items = _ensure_correct_item_order(items=r.list(), repo=r) + + # Convert ixmp4 items to ContainerData to streamline adding to container + container_items = _convert_ixmp4_items_to_containerdata(items=ixmp4_items) + + _add_items_to_container(container, container_items) + + # Add additional data *after* the required items to avoid confusing GAMS' internal + # Unique Element List + _add_items_to_container(container, container_data) + + _record_versions(container=container, packages=record_version_packages) + + container.write(write_to=file_name) + + +# NOTE since we currently only read Variables and Equations, this function only covers +# these cases +def _set_columns_to_read_from_records( + item: Union[AbstractVariable, AbstractEquation], +) -> list[str]: + """Gather all columns for `item` to read from GDX records.""" + # Prepare columns to select from container.data + # DF also includes lower, upper, scale + columns = ["levels", "marginals"] + item_columns = item.column_names or item.indexsets + if item_columns: + item_columns.extend(columns) + + return item_columns if item_columns else columns + + +# NOTE not sure we only need equations and variables; if we need others, abstracting one +# function for reading would not be as easy, since we might need different details, so +# I'm keeping them separate for now + + +def _read_variables_to_run( + container: gt.Container, run: Run, variables: Iterable[AbstractVariable] +) -> None: + """Read `variables` from `container` and store them in `run`.""" + for variable in variables: + columns_of_interest = _set_columns_to_read_from_records(item=variable) + + records = pd.DataFrame(container.data[variable.name].records) + + # Avoid touching the DB for empty data + if not records.empty: + # NOTE Overwriting the columns changes GAMS' "level" and "marginal" to the + # ixmp4-expected "levels" and "marginals" + # NOTE This assumes that the order the columns are defined in the gams code + # is equal to the order in item.column_names and item.indexsets + records.columns = pd.Index( + columns_of_interest + ["lower", "upper", "scale"] + ) + run.backend.optimization.variables.add_data( + variable_id=variable.id, data=records[columns_of_interest] + ) + + +def _read_equations_to_run( + container: gt.Container, run: Run, equations: Iterable[AbstractEquation] +) -> None: + """Read `equations` from `container` and store them in `run`.""" + for equation in equations: + columns_of_interest = _set_columns_to_read_from_records(item=equation) + + records = pd.DataFrame(container.data[equation.name].records) + + # Avoid touching the DB for empty data + if not records.empty: + # NOTE Overwriting the columns changes GAMS' "level" and "marginal" to the + # ixmp4-expected "levels" and "marginals" + # NOTE This assumes that the order the columns are defined in the gams code + # is equal to the order in item.column_names and item.indexsets + records.columns = pd.Index( + columns_of_interest + ["lower", "upper", "scale"] + ) + run.backend.optimization.equations.add_data( + equation_id=equation.id, data=records[columns_of_interest] + ) + + +def read_gdx_to_run( + run: Run, + result_file: Path, + equ_list: list[str], + var_list: list[str], + comment: str, + check_solution: bool, +) -> None: + """Read data from `result_file` to `run` via a GAMS Container. + + Parameters + ---------- + run : :class:`ixmp4.core.Run` + The Run in which the Scenario data is to be stored. + result_file: :obj:`Path` + The location from which the GDX file should be read. + equ_list : list of str + Names of Equations to read from `result_file`. + var_list : list of str + Names of Variables to read from `result_file`. + comment : str + Unused by ixmp4. + A comment to store when adding the data to `run`. + check_solution : bool + Unused by ixmp4. + A flag to indicate whether the solution should be checked + (for consistency, maybe?). + """ + # Warn about unused parameters + if comment: + log.warning(f"Ignoring comment {comment} for now; unused by ixmp4!") + if check_solution: + log.warning( + f"Ignoring check_solution={check_solution} for now; unused by ixmp4!" + ) + + # Create a GAMS Container from the `result_file` + container = gt.Container( + load_from=result_file, system_directory=str(gams_info().system_dir) + ) + + # Load requested Variables and read them to `run` + # NOTE This handles empty `var_list`, too, + # which is not necessary as long as any Variables are required in message_ix + variables = ( + run.backend.optimization.variables.list(run_id=run.id, name__in=var_list) + if len(var_list) + else run.backend.optimization.variables.list(run_id=run.id) + ) + _read_variables_to_run(container=container, run=run, variables=variables) + + # Load requested Equations and read them to `run` + # NOTE This handles empty `equ_list`, too, + # which is not necessary as long as any Equations are required in message_ix + equations = ( + run.backend.optimization.equations.list(run_id=run.id, name__in=equ_list) + if len(equ_list) + else run.backend.optimization.equations.list(run_id=run.id) + ) + _read_equations_to_run(container=container, run=run, equations=equations) diff --git a/ixmp/backend/jdbc.py b/ixmp/backend/jdbc.py index ac8c864dc..a9281daad 100644 --- a/ixmp/backend/jdbc.py +++ b/ixmp/backend/jdbc.py @@ -10,7 +10,7 @@ from functools import lru_cache from pathlib import Path, PurePosixPath from types import SimpleNamespace -from typing import Optional +from typing import Optional, cast from weakref import WeakKeyDictionary import jpype @@ -561,6 +561,7 @@ def read_file(self, path, item_type: ItemType, **kwargs): kwargs.pop("check_solution"), ) + # NOTE This test seems unnecessary with the 'elif' clause above if len(kwargs): raise ValueError(f"extra keyword arguments {kwargs}") @@ -605,7 +606,7 @@ def write_file(self, path, item_type: ItemType, **kwargs): ts, filters = self._handle_rw_filters(kwargs.pop("filters", {})) if path.suffix == ".gdx" and item_type is ItemType.SET | ItemType.PAR: - if len(filters): # pragma: no cover + if len(filters) > 1: # pragma: no cover raise NotImplementedError("write to GDX with filters") elif not isinstance(ts, Scenario): # pragma: no cover raise ValueError("write to GDX requires a Scenario object") @@ -614,7 +615,9 @@ def write_file(self, path, item_type: ItemType, **kwargs): self.jindex[ts].toGDX(str(path.parent), path.name, False) elif path.suffix == ".csv" and item_type is ItemType.TS: models = set(filters.pop("model")) - scenarios = set(filters.pop("scenario")) + # NOTE this is what we get for not differentiating e.g. scenario vs + # scenarios in filters... + scenarios = set(cast(list[str], filters.pop("scenario"))) variables = filters.pop("variable") units = filters.pop("unit") regions = filters.pop("region") diff --git a/ixmp/core/platform.py b/ixmp/core/platform.py index 7107773e0..b3142001c 100644 --- a/ixmp/core/platform.py +++ b/ixmp/core/platform.py @@ -1,7 +1,7 @@ import logging from collections.abc import Sequence from os import PathLike -from typing import TYPE_CHECKING, Optional, Union +from typing import TYPE_CHECKING, Literal, Optional, Union import numpy as np import pandas as pd @@ -9,6 +9,7 @@ from ixmp._config import config from ixmp.backend import BACKENDS, FIELDS, ItemType from ixmp.util import as_str_list +from ixmp.util.ixmp4 import WriteFiltersKwargs if TYPE_CHECKING: from ixmp.backend.base import Backend @@ -59,8 +60,13 @@ class Platform: "set_meta", ] + _units_to_warn_about: Optional[list[str]] = None + def __init__( - self, name: Optional[str] = None, backend: Optional[str] = None, **backend_args + self, + name: Optional[str] = None, + backend: Optional[Literal["ixmp4", "jdbc"]] = None, + **backend_args, ): if name is None: if backend is None and not len(backend_args): @@ -236,15 +242,15 @@ def export_timeseries_data( "Invalid arguments: export_all_runs cannot be used when providing a " "model or scenario." ) - filters = { - "model": as_str_list(model), - "scenario": as_str_list(scenario), - "variable": as_str_list(variable), - "unit": as_str_list(unit), - "region": as_str_list(region), - "default": default, - "export_all_runs": export_all_runs, - } + filters = WriteFiltersKwargs( + scenario=as_str_list(scenario), + model=as_str_list(model), + variable=as_str_list(variable), + unit=as_str_list(unit), + region=as_str_list(region), + default=default, + export_all_runs=export_all_runs, + ) self._backend.write_file(path, ItemType.TS, filters=filters) diff --git a/ixmp/core/scenario.py b/ixmp/core/scenario.py index fd81ef2e2..4cb1c89fe 100644 --- a/ixmp/core/scenario.py +++ b/ixmp/core/scenario.py @@ -41,7 +41,7 @@ class in :data:`.MODELS` is used to initialize items in the Scenario. """ #: Scheme of the Scenario. - scheme = None + scheme: Optional[str] = None def __init__( self, @@ -705,6 +705,7 @@ def remove_par(self, name: str, key=None) -> None: else: self._backend("item_delete_elements", "par", name, self._keys(name, key)) + # FIXME What ensures that filters has the correct type? def var(self, name: str, filters=None, **kwargs): """Return a dataframe of (filtered) elements for a specific variable. diff --git a/ixmp/core/timeseries.py b/ixmp/core/timeseries.py index f80492656..5dc4a7af3 100644 --- a/ixmp/core/timeseries.py +++ b/ixmp/core/timeseries.py @@ -158,6 +158,7 @@ def from_url( ts = cls(platform, **scenario_info) except Exception as e: if errors == "warn": + # FIXME ixmp4 errors might have empty e.args log.warning( f"{e.__class__.__name__}: {e.args[0]}\n" f"when loading {cls.__name__} from url: {repr(url)}" diff --git a/ixmp/model/gams.py b/ixmp/model/gams.py index 819516319..d098c383a 100644 --- a/ixmp/model/gams.py +++ b/ixmp/model/gams.py @@ -8,11 +8,14 @@ from pathlib import Path from subprocess import CalledProcessError, check_output, run from tempfile import TemporaryDirectory -from typing import Any, Optional +from typing import Any, Optional, Union from ixmp.backend import ItemType +from ixmp.backend.jdbc import JDBCBackend +from ixmp.core.scenario import Scenario from ixmp.model.base import Model, ModelError from ixmp.util import as_str_list +from ixmp.util.ixmp4 import ContainerData log = logging.getLogger(__name__) @@ -178,8 +181,27 @@ class GAMSModel(Model): record_version_packages : list of str, optional Names of Python packages to record versions. Default: :py:`["ixmp"]`. See :meth:`record_versions`. + container_data : list of :class:`ixmp.util.ixmp4.ContainerData`, optional + List of data to add to the GAMS Container used by the IXMP4Backend for GAMS I/O. + Default: empty list. """ # noqa: E501 + # Make attributes known to self + model_file: str + case: str + in_file: os.PathLike + out_file: os.PathLike + solve_args: list[str] + gams_args: list[str] + check_solution: bool + comment: Optional[str] + equ_list: Optional[list[str]] + var_list: Optional[list[str]] + quiet: bool + use_temp_dir: bool + record_version_packages: list[str] + container_data: list[ContainerData] + #: Model name. name = "default" @@ -199,6 +221,7 @@ class GAMSModel(Model): "quiet": False, "use_temp_dir": True, "record_version_packages": ["ixmp"], + "container_data": [], } def __init__(self, name_=None, **model_options): @@ -308,7 +331,7 @@ def __del__(self): # This appears to still fail on Windows. self.remove_temp_dir("at GAMSModel teardown") - def run(self, scenario): + def run(self, scenario: Scenario) -> None: """Execute the model. Among other steps: @@ -324,8 +347,18 @@ def run(self, scenario): # Store the scenario so its attributes can be referenced by format() self.scenario = scenario - # Record versions of packages listed in `record_version_packages` - self.record_versions() + # NOTE workaround delattr(self, "scenario"); differentiate backend types without + # moving that call + # If we get more backend types, we'll need to adjust this + backend_type = ( + "jdbc" + if isinstance(self.scenario.platform._backend, JDBCBackend) + else "ixmp4" + ) + + if backend_type == "jdbc": + # Record versions of packages listed in `record_version_packages` + self.record_versions() # Format or retrieve the model file option model_file = Path(self.format_option("model_file")) @@ -340,7 +373,7 @@ def run(self, scenario): # Assemble the full command: executable, model file, model-specific arguments, # and general GAMS arguments - command = ( + command: Union[str, list[str]] = ( ["gams", f'"{model_file}"'] + [self.format(arg) for arg in self.solve_args] + self.gams_args @@ -354,7 +387,12 @@ def run(self, scenario): delattr(self, "scenario") # Common argument for write_file and read_file - s_arg = dict(filters=dict(scenario=scenario)) + s_arg: dict[str, object] = dict(filters=dict(scenario=scenario)) + + # Instruct ixmp4 to record package versions + if backend_type == "ixmp4": + s_arg["record_version_packages"] = self.record_version_packages + s_arg["container_data"] = self.container_data try: # Write model data to file @@ -373,10 +411,14 @@ def run(self, scenario): "JDBCBackend" ) else: - # Remove ixmp_version set entirely - with scenario.transact(): - scenario.remove_set("ixmp_version") - + if backend_type == "jdbc": + # Remove ixmp_version set entirely + with scenario.transact(): + scenario.remove_set("ixmp_version") + else: # backend_type == "ixmp4" + # Clean up s_arg for use in read_file() + s_arg.pop("record_version_packages") + s_arg.pop("container_data") try: # Invoke GAMS run(command, shell=os.name == "nt", cwd=self.cwd, check=True) diff --git a/ixmp/report/reporter.py b/ixmp/report/reporter.py index 3eff09dd4..cab0ebe95 100644 --- a/ixmp/report/reporter.py +++ b/ixmp/report/reporter.py @@ -1,5 +1,5 @@ from itertools import chain, repeat -from typing import Union, cast +from typing import Literal, Union, cast import dask import pandas as pd @@ -53,7 +53,7 @@ def from_scenario(cls, scenario: Scenario, **kwargs) -> "Reporter": all_keys: list[Union[str, Key]] = [] # List of parameters, equations, and variables - quantities = chain( + quantities: chain[tuple[Literal["par", "equ", "var"], str]] = chain( zip(repeat("par"), sorted(scenario.par_list())), zip(repeat("equ"), sorted(scenario.equ_list())), zip(repeat("var"), sorted(scenario.var_list())), diff --git a/ixmp/report/util.py b/ixmp/report/util.py index 3a2430933..dd6e7622b 100644 --- a/ixmp/report/util.py +++ b/ixmp/report/util.py @@ -1,12 +1,18 @@ from functools import lru_cache, partial +from typing import TYPE_CHECKING, Literal, Union import pandas as pd from genno import Key from ixmp.report import common +if TYPE_CHECKING: + from genno.types import AnyQuantity -def dims_for_qty(data): + from ixmp.core.scenario import Scenario + + +def dims_for_qty(data: Union[list[str], pd.DataFrame]) -> list[str]: """Return the list of dimensions for *data*. If *data* is a :class:`pandas.DataFrame`, its columns are processed; @@ -28,7 +34,9 @@ def dims_for_qty(data): return [common.RENAME_DIMS.get(d, d) for d in dims] -def keys_for_quantity(ix_type, name, scenario): +def keys_for_quantity( + ix_type: Literal["par", "equ", "var"], name: str, scenario: "Scenario" +) -> list[tuple[Key, partial["AnyQuantity"], str, str]]: """Return keys for *name* in *scenario*.""" from .operator import data_for_quantity @@ -36,7 +44,7 @@ def keys_for_quantity(ix_type, name, scenario): dims = dims_for_qty(scenario.idx_names(name)) # Column for retrieving data - column = "value" if ix_type == "par" else "lvl" + column: Literal["mrg", "lvl", "value"] = "value" if ix_type == "par" else "lvl" # A computation to retrieve the data result = [ @@ -63,11 +71,11 @@ def keys_for_quantity(ix_type, name, scenario): @lru_cache(1) -def get_reversed_rename_dims(): +def get_reversed_rename_dims() -> dict[str, str]: return {v: k for k, v in common.RENAME_DIMS.items()} -def __getattr__(name: str): +def __getattr__(name: str) -> dict[str, str]: if name == "RENAME_DIMS": return common.RENAME_DIMS else: diff --git a/ixmp/testing/__init__.py b/ixmp/testing/__init__.py index 1eee1f923..8ada44571 100644 --- a/ixmp/testing/__init__.py +++ b/ixmp/testing/__init__.py @@ -35,16 +35,19 @@ import logging import os import shutil +import sys +from collections.abc import Generator from contextlib import contextmanager, nullcontext from copy import deepcopy from itertools import chain from pathlib import Path +from typing import Any, Literal import pint import pytest from click.testing import CliRunner -from ixmp import Platform, cli +from ixmp import BACKENDS, Platform, Scenario, cli from ixmp import config as ixmp_config from .data import ( @@ -85,11 +88,19 @@ "tmp_env", ] +# Parametrize platforms for jdbc and ixmp4 +backends = list(BACKENDS.keys()) + +# Provide a skip marker since ixmp4 is not published for Python 3.9 +min_ixmp4_version = pytest.mark.skipif( + sys.version_info < (3, 10), reason="ixmp4 requires Python 3.10 or higher" +) + # Pytest hooks -def pytest_addoption(parser): +def pytest_addoption(parser: pytest.Parser) -> None: """Add the ``--user-config`` command-line option to pytest.""" parser.addoption( "--ixmp-jvm-mem", @@ -103,7 +114,7 @@ def pytest_addoption(parser): ) -def pytest_sessionstart(session): +def pytest_sessionstart(session: pytest.Session) -> None: """Unset any configuration read from the user's directory.""" from ixmp.backend import jdbc @@ -117,11 +128,26 @@ def pytest_sessionstart(session): jdbc._GC_AGGRESSIVE = False -def pytest_report_header(config, start_path): +def pytest_report_header(config, start_path) -> str: """Add the ixmp configuration to the pytest report header.""" return f"ixmp config: {repr(ixmp_config.values)}" +# NOTE https://docs.pytest.org/en/latest/example/markers.html#marking-platform-specific-tests-with-pytest +# sound like what we need, but I couldn't quite get it to work. Instead, this is more +# following https://pytest-with-eric.com/introduction/pytest-generate-tests/ +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + """Parametrize tests for the two backend options.""" + if "backend" in metafunc.fixturenames: + markers = [m.name for m in metafunc.definition.iter_markers()] + + requested_backends = [marker for marker in markers if marker in backends] + + _backends = requested_backends if bool(requested_backends) else backends + + metafunc.parametrize("backend", _backends, indirect=True) + + # Session-scoped fixtures @@ -137,7 +163,7 @@ def invoke(self, *args, **kwargs): @pytest.fixture(scope="module") -def mp(test_mp): +def mp(test_mp: Platform) -> Generator[Platform, Any, None]: """A :class:`.Platform` containing test data. This fixture is **module** -scoped, and is used in :mod:`.test_platform`, @@ -149,23 +175,37 @@ def mp(test_mp): @pytest.fixture(scope="session") -def test_data_path(): +def test_data_path() -> Path: """Path to the directory containing test data.""" return Path(__file__).parents[1].joinpath("tests", "data") +# NOTE We need to declare this as module-scope explicitly; otherwise, pytest creates +# backend for pytest_generate_tests as function-scoped fixture automatically +@pytest.fixture(scope="module") +def backend(request): + return request.param + + @pytest.fixture(scope="module") -def test_mp(request, tmp_env, test_data_path): +def test_mp( + request: pytest.FixtureRequest, + tmp_env, + test_data_path, + backend: Literal["ixmp4", "jdbc"], +) -> Generator[Platform, Any, None]: """An empty :class:`.Platform` connected to a temporary, in-memory database. This fixture has **module** scope: the same Platform is reused for all tests in a module. """ - yield from _platform_fixture(request, tmp_env, test_data_path) + yield from _platform_fixture(request, tmp_env, test_data_path, backend=backend) @pytest.fixture(scope="session") -def tmp_env(pytestconfig, tmp_path_factory): +def tmp_env( + pytestconfig: pytest.Config, tmp_path_factory: pytest.TempPathFactory +) -> Generator[os._Environ[str], Any, None]: """Return the os.environ dict with the IXMP_DATA variable set. IXMP_DATA will point to a temporary directory that is unique to the test session. @@ -187,13 +227,13 @@ def tmp_env(pytestconfig, tmp_path_factory): @pytest.fixture(scope="session") -def tutorial_path(): +def tutorial_path() -> Path: """Path to the directory containing the tutorials.""" return Path(__file__).parents[2].joinpath("tutorial") @pytest.fixture(scope="session") -def ureg(): +def ureg() -> Generator[pint.UnitRegistry, Any, None]: """Application-wide units registry.""" registry = pint.get_application_registry() @@ -243,7 +283,12 @@ def protect_rename_dims(): @pytest.fixture(scope="function") -def test_mp_f(request, tmp_env, test_data_path): +def test_mp_f( + request: pytest.FixtureRequest, + tmp_env, + test_data_path, + backend: Literal["ixmp4", "jdbc"], +) -> Generator[Platform, Any, None]: """An empty :class:`Platform` connected to a temporary, in-memory database. This fixture has **function** scope: the same Platform is reused for one test @@ -253,7 +298,32 @@ def test_mp_f(request, tmp_env, test_data_path): -------- test_mp """ - yield from _platform_fixture(request, tmp_env, test_data_path) + yield from _platform_fixture(request, tmp_env, test_data_path, backend=backend) + + +# NOTE No type hint for Python 3.9 compliance +@pytest.fixture +def ixmp4_backend(test_mp: Platform): + from ixmp.backend.ixmp4 import IXMP4Backend + + assert isinstance(test_mp._backend, IXMP4Backend) + return test_mp._backend + + +@pytest.fixture +def scenario(test_mp: Platform, request: pytest.FixtureRequest) -> Scenario: + return Scenario( + mp=test_mp, + model=request.node.nodeid + "model", + scenario="scenario", + version="new", + ) + + +# NOTE No type hint for Python 3.9 compliance +@pytest.fixture +def run(ixmp4_backend, scenario: Scenario): + return ixmp4_backend.index[scenario] # Assertions @@ -384,19 +454,53 @@ def create_test_platform(tmp_path, data_path, name, **properties): # Private utilities -def _platform_fixture(request, tmp_env, test_data_path): +# NOTE The test suite doesn't need all units/regions defined by default +# Only those marked with 'tutorials' are needed for the MESSAGE tutorials +def _setup_ixmp4_platform(mp: Platform) -> None: + """Set up an ixmp4-backed Platform with things hardcoded in Java.""" + mp.add_unit("???", comment="As pre-defined in Java.") + mp.add_unit("GWa", comment="As pre-defined in Java.") + mp.add_unit("USD/kWa", comment="As pre-defined in Java.") + mp.add_unit("cases", comment="As pre-defined in Java.") # tutorials + mp.add_unit("kg", comment="As pre-defined in Java.") + mp.add_unit("km", comment="As pre-defined in Java.") # tutorials + mp.add_region(region="World", hierarchy="common") + + +def _platform_fixture( + request: pytest.FixtureRequest, + tmp_env, + test_data_path, + backend: Literal["jdbc", "ixmp4"], +) -> Generator[Platform, Any, None]: """Helper for :func:`test_mp` and other fixtures.""" # Long, unique name for the platform. # Remove '/' so that the name can be used in URL tests. platform_name = request.node.nodeid.replace("/", " ") # Add a platform - ixmp_config.add_platform( - platform_name, "jdbc", "hsqldb", url=f"jdbc:hsqldb:mem:{platform_name}" - ) + if backend == "jdbc": + ixmp_config.add_platform( + platform_name, backend, "hsqldb", url=f"jdbc:hsqldb:mem:{platform_name}" + ) + elif backend == "ixmp4": + import ixmp4.conf + from ixmp4.core.exceptions import PlatformNotUnique + + # Add to or get from ixmp4 an in-memory DB/Platform + dsn = "sqlite:///:memory:" + try: + ixmp4.conf.settings.toml.add_platform(name=platform_name, dsn=dsn) + except PlatformNotUnique: + pass + + # Add ixmp4 backend to ixmp platforms + ixmp_config.add_platform(platform_name, backend, _name=platform_name) # Launch Platform - mp = Platform(name=platform_name) + mp = Platform(name=platform_name, backend=backend) + if backend == "ixmp4": + _setup_ixmp4_platform(mp=mp) yield mp # Teardown: don't show log messages when destroying the platform, even if @@ -406,3 +510,6 @@ def _platform_fixture(request, tmp_env, test_data_path): # Remove from config ixmp_config.remove_platform(platform_name) + + if backend == "ixmp4": + ixmp4.conf.settings.toml.remove_platform(key=platform_name) diff --git a/ixmp/testing/data.py b/ixmp/testing/data.py index 8f551bf67..b328272e1 100644 --- a/ixmp/testing/data.py +++ b/ixmp/testing/data.py @@ -1,4 +1,5 @@ # Methods are in alphabetical order +import sys from itertools import product from math import ceil from typing import TYPE_CHECKING, Any, Optional @@ -182,6 +183,11 @@ def add_test_data(scen: Scenario): return t, t_foo, t_bar, x +def _add_required_units(mp: Platform) -> None: + mp.add_unit("USD") + mp.add_unit("???") + + def make_dantzig( mp: Platform, solve: bool = False, @@ -209,6 +215,12 @@ def make_dantzig( -------- .DantzigModel """ + if sys.version_info >= (3, 10): + from ixmp.backend.ixmp4 import IXMP4Backend + + if isinstance(mp._backend, IXMP4Backend): + _add_required_units(mp=mp) + # Add custom units and region for time series data try: mp.add_unit("USD/km") @@ -250,7 +262,7 @@ def make_dantzig( return scen -def populate_test_platform(platform): +def populate_test_platform(platform: Platform) -> None: """Populate `platform` with data for testing. Many of the tests in :mod:`ixmp.tests.core` depend on this set of data. diff --git a/ixmp/tests/backend/test_base.py b/ixmp/tests/backend/test_base.py index b84920bea..e12253353 100644 --- a/ixmp/tests/backend/test_base.py +++ b/ixmp/tests/backend/test_base.py @@ -135,6 +135,8 @@ def test_cache_invalidate(self, test_mp): backend.cache_invalidate(ts, "par", "baz", dict(x=["x1", "x2"], y=["y1", "y2"])) + # TODO IXMP4Backend needs to handle ._cache correctly + @pytest.mark.jdbc def test_del_ts(self, test_mp, request): """Test CachingBackend.del_ts().""" # Since CachingBackend is an abstract class, test it via JDBCBackend diff --git a/ixmp/tests/backend/test_io.py b/ixmp/tests/backend/test_io.py index 65c9cc3fc..f3b113150 100644 --- a/ixmp/tests/backend/test_io.py +++ b/ixmp/tests/backend/test_io.py @@ -1,8 +1,10 @@ +from pathlib import Path + import ixmp from ixmp.testing import add_random_model_data, models -def test_read_excel_big(test_mp, tmp_path): +def test_read_excel_big(test_mp: ixmp.Platform, tmp_path: Path) -> None: """Excel files with model items split across sheets can be read. https://github.com/iiasa/ixmp/pull/345. diff --git a/ixmp/tests/backend/test_ixmp4.py b/ixmp/tests/backend/test_ixmp4.py new file mode 100644 index 000000000..f55577eef --- /dev/null +++ b/ixmp/tests/backend/test_ixmp4.py @@ -0,0 +1,332 @@ +from pathlib import Path +from typing import Any, Literal, cast + +import pandas as pd +import pytest + +from ixmp import Scenario +from ixmp.backend import ItemType +from ixmp.testing import min_ixmp4_version + +pytestmark = min_ixmp4_version + + +def test__ensure_filters_values_are_lists() -> None: + from ixmp.backend.ixmp4 import _ensure_filters_values_are_lists + + filters = {"foo": [1, 2], "bar": 3} + expected = {"foo": [1, 2], "bar": [3]} + _ensure_filters_values_are_lists(filters=filters) + assert filters == expected + + +def test__align_dtypes_for_filters() -> None: + from ixmp.backend.ixmp4 import _align_dtypes_for_filters + + # This leads to dtypes 'int64' and 'object' + df = pd.DataFrame({"foo": [1, 2, 3], "bar": ["baz", "foo", "bar"]}) + # Filters' types are determined based on first item assuming one type per key + filters: dict[str, list[Any]] = {"foo": [1.0, 2.0, 3.0], "bar": [1, 2, 3]} + expected = {"foo": [1, 2, 3], "bar": ["1", "2", "3"]} + _align_dtypes_for_filters(filters=filters, data=df) + + assert filters == expected + + +# Overriding fixture that usually parametrizes test_mp +@pytest.fixture(scope="module") +def backend() -> Literal["ixmp4"]: + return "ixmp4" + + +# NOTE This marker usually parametrizes test creation, but with overriding `backend`, +# this might not be necessary. Keeping it for clarity. +@pytest.mark.ixmp4 +class TestIxmp4Functions: + """Test group for all functions touching ixmp4 directly.""" + + # NOTE Not writing a test for _index_and_set_attrs(). This is a helper function + # that's also used by JDBC, and is always called for Backend.init() and + # Backend.get(), so is tested already. + + def test__ni(self, ixmp4_backend) -> None: + with pytest.raises(NotImplementedError): + ixmp4_backend._ni() + + def test__get_repo(self, ixmp4_backend, scenario: Scenario) -> None: + from ixmp4.core.optimization.equation import EquationRepository + from ixmp4.core.optimization.indexset import IndexSetRepository + from ixmp4.core.optimization.parameter import ParameterRepository + from ixmp4.core.optimization.scalar import ScalarRepository + from ixmp4.core.optimization.table import TableRepository + from ixmp4.core.optimization.variable import VariableRepository + + repos: dict[Literal["indexset", "scalar", "set", "par", "equ", "var"], Any] = { + "indexset": IndexSetRepository, + "scalar": ScalarRepository, + "set": TableRepository, + "par": ParameterRepository, + "equ": EquationRepository, + "var": VariableRepository, + } + + # Test correct kind of instance is returned + for type, expected_repo in repos.items(): + repo = ixmp4_backend._get_repo(s=scenario, type=type) + assert isinstance(repo, expected_repo) + + def test__find_item(self, ixmp4_backend, scenario: Scenario) -> None: + # Test unknown item name raises + with pytest.raises(KeyError, match="No item called 'NotFound' found"): + ixmp4_backend._find_item(s=scenario, name="NotFound") + + # Test finding an item and its type + name = "foo" + _type: Literal["indexset"] = "indexset" + scenario.init_set(name=name) + return_type, return_item = ixmp4_backend._find_item( + s=scenario, name=name, types=tuple([_type]) + ) + assert (_type, name) == (return_type, return_item.name) + + def test__get_item(self, ixmp4_backend, scenario: Scenario) -> None: + # Test getting an item + name = "foo" + _type: Literal["indexset"] = "indexset" + scenario.init_set(name=name) + indexset = ixmp4_backend._get_item(s=scenario, name=name, type=_type) + assert indexset.name == name + + def test__get_indexset_or_table(self, ixmp4_backend, scenario: Scenario) -> None: + # Test getting an item of "unknown" type (either 'indexset' or 'table'); + # getting a Table first tests getting an IndexSet, too + indexset_name = "foo" + scenario.init_set(name=indexset_name) + table_name = "bar" + scenario.init_set(name=table_name, idx_sets=[indexset_name]) + table = ixmp4_backend._get_indexset_or_table(s=scenario, name=table_name) + assert table.name == table_name + + def test__add_data_to_set(self, ixmp4_backend, scenario: Scenario) -> None: + # Test adding to an Indexset and warning about `comment` + indexset_name = "Indexset" + scenario.init_set(name=indexset_name) + key = "foo" + ixmp4_backend._add_data_to_set( + s=scenario, name=indexset_name, key=key, comment="Test comment" + ) + indexset_data = scenario.set(indexset_name) + assert isinstance(indexset_data, pd.Series) + pd.testing.assert_series_equal(indexset_data, pd.Series([key])) + + # Test adding to a Table + table_name = "Table" + scenario.init_set(name=table_name, idx_sets=[indexset_name]) + ixmp4_backend._add_data_to_set(s=scenario, name=table_name, key=[key]) + # We can assume this data type for Tables + pd.testing.assert_frame_equal( + cast(pd.DataFrame, scenario.set(table_name)), + pd.DataFrame({indexset_name: [key]}), + ) + + def test__create_scalar(self, ixmp4_backend, scenario: Scenario) -> None: + # Test creating a scalar with a comment + name = "Scalar" + value = 3.141592653589793238462643383279502884197169399375105820 + comment = "Comment" + ixmp4_backend._create_scalar( + s=scenario, name=name, value=value, unit=None, comment=comment + ) + # NOTE We don't really have a way to retrieve Scalars from IXMP4Backend + scalar = ixmp4_backend.index[scenario].optimization.scalars.get(name=name) + assert scalar.value == value + assert scalar.docs == comment + + def test__add_data_to_parameter(self, ixmp4_backend, scenario: Scenario) -> None: + # Set up auxiliary items + unit_name = "Unit" + ixmp4_backend.set_unit(name=unit_name, comment="Create test unit") + indexset_name = "Indexset" + scenario.init_set(name=indexset_name) + key = "foo" + scenario.add_set(name=indexset_name, key=key) + + # Test creating a parameter with a comment and key conversion to list[str] + name = "Parameter" + value = 3.141592653589793238462643383279502884197169399375105820 + comment = "Comment" + scenario.init_par(name=name, idx_sets=[indexset_name]) + ixmp4_backend._add_data_to_parameter( + s=scenario, name=name, key=key, value=value, unit=unit_name, comment=comment + ) + parameter = ixmp4_backend.index[scenario].optimization.parameters.get(name=name) + assert parameter.data == { + indexset_name: [key], + "values": [value], + "units": [unit_name], + } + + def test__get_set_data(self, ixmp4_backend, scenario: Scenario) -> None: + indexset_name = "Indexset" + scenario.init_set(name=indexset_name) + table_name = "Table" + scenario.init_set(name=table_name, idx_sets=[indexset_name]) + + # Test getting empty data (other cases tested in integration tests) + indexset_data = ixmp4_backend._get_set_data(s=scenario, name=indexset_name) + # We can assume this return type for Indexsets + pd.testing.assert_series_equal(cast(pd.Series, indexset_data), pd.Series([])) + table_data = ixmp4_backend._get_set_data(s=scenario, name=table_name) + # We can assume this return type for Tables + pd.testing.assert_frame_equal( + cast(pd.DataFrame, table_data), pd.DataFrame(columns=[indexset_name]) + ) + + # Test some edge cases for standard functions + def test_handle_config(self, ixmp4_backend) -> None: + # Test raising for unhandled positional args + with pytest.raises(ValueError, match="Unhandled positional args"): + ixmp4_backend.handle_config(["test arg"], {"foo": "bar"}) + + # Test raising for missing required key + with pytest.raises(ValueError, match="Missing key '_name'"): + ixmp4_backend.handle_config([], {"foo": "bar"}) + + def test_set_node(self, ixmp4_backend, caplog: pytest.LogCaptureFixture) -> None: + from ixmp.backend.ixmp4 import log + + # Test logging warnings for unused/required parameters + parent = "Parent" + synonym = "Synonym" + with caplog.at_level("WARNING", logger=log.name): + ixmp4_backend.set_node( + name="Region", parent=parent, hierarchy=None, synonym=synonym + ) + + expected = [ + f"Discarding parent parameter {parent}; unused in ixmp4.", + f"Discarding synonym parameter {synonym}; unused in ixmp4.", + "IXMP4Backend.set_node() requires to specify 'hierarchy'! " + "Using 'None' as the meaningsless default.", + ] + assert caplog.messages == expected + + def test_clone( + self, ixmp4_backend, caplog: pytest.LogCaptureFixture, scenario: Scenario + ) -> None: + from ixmp.backend.ixmp4 import log + + # Test logging a warning for first_model_year + with caplog.at_level("WARNING", logger=log.name): + ixmp4_backend.clone( + s=scenario, + platform_dest=scenario.platform, + model=scenario.model, + scenario=scenario.scenario + "_clone", + annotation="not used", + keep_solution=False, + first_model_year=1, + ) + + expected = ( + "ixmp4-backed Scenarios don't support cloning from `first_model_year` only!" + ) + assert expected in caplog.messages + + def test_clear_solution( + self, ixmp4_backend, caplog: pytest.LogCaptureFixture, scenario: Scenario + ) -> None: + from ixmp.backend.ixmp4 import log + + # Test logging a warning for from_year + with caplog.at_level("WARNING", logger=log.name): + ixmp4_backend.clear_solution(s=scenario, from_year=1) + + expected = ( + "ixmp4 does not support removing the solution only after a certain year" + ) + assert expected in caplog.messages + + def test_run_id(self, ixmp4_backend, scenario: Scenario) -> None: + # NOTE Depending on what run_id() should actually fetch, this needs adapting + # scenario sets up a new Run, which has version 1 + assert ixmp4_backend.run_id(ts=scenario) == 1 + + def test_item_delete_elements(self, ixmp4_backend, scenario: Scenario) -> None: + # Prepare some data + run = ixmp4_backend.index[scenario] + indexset_data = "foo" + indexset = run.optimization.indexsets.create("Indexset") + indexset.add(data=indexset_data) + table_data = {indexset.name: [indexset_data]} + table = run.optimization.tables.create( + "Table", constrained_to_indexsets=[indexset.name] + ) + table.add(data=table_data) + + # Assert data is stored in scenario + set_data = scenario.set(name=table.name) + assert isinstance(set_data, pd.DataFrame) + assert not set_data.empty + + # Test data deletion for Tables + ixmp4_backend.item_delete_elements( + s=scenario, type="set", name=table.name, keys=[[indexset_data]] + ) + new_data = scenario.set(name=table.name) + assert isinstance(new_data, pd.DataFrame) + assert new_data.empty + + def test_delete_item(self, ixmp4_backend, scenario: Scenario) -> None: + # Create a 'set' to delete + run = ixmp4_backend.index[scenario] + indexset = run.optimization.indexsets.create("Indexset") + ixmp4_backend.delete_item(s=scenario, type="set", name=indexset.name) + + # Test there are no 'sets' on scenario anymore + assert ixmp4_backend.list_items(s=scenario, type="set") == [] + + def test_write_file(self, ixmp4_backend) -> None: + # Test raising an error for unknown file extension + with pytest.raises(NotImplementedError): + ixmp4_backend.write_file( + path=Path("none.txt"), item_type=ItemType.SET, filters={} + ) + + # Test raising with incorrect ItemType + with pytest.raises(NotImplementedError): + ixmp4_backend.write_file( + path=Path("none.gdx"), item_type=ItemType.EQU, filters={} + ) + + def test_read_file(self, ixmp4_backend, scenario: Scenario) -> None: + # Test raising an error for unknown file extension + with pytest.raises(NotImplementedError): + ixmp4_backend.read_file( + path=Path("none.txt"), item_type=ItemType.SET, filters={} + ) + + # Test raising with incorrect ItemType + with pytest.raises(NotImplementedError): + ixmp4_backend.read_file( + path=Path("none.gdx"), item_type=ItemType.EQU, filters={} + ) + + # Test raising when filters doesn't include a proper Scenario + with pytest.raises( + ValueError, match="read from GDX requires a Scenario object" + ): + ixmp4_backend.read_file( + path=Path("none.gdx"), item_type=ItemType.MODEL, filters={} + ) + + # TODO Should the ixmp4_backend get its own TypedDict for kwargs that only + # allows our expected values? If so, we probably need to adjust. + # Test raising with extra kwargs + with pytest.raises(ValueError, match="keyword arguments"): + ixmp4_backend.read_file( + path=Path("none.gdx"), + item_type=ItemType.MODEL, + init_items=True, + filters={"scenario": scenario}, + ) diff --git a/ixmp/tests/backend/test_ixmp4_io.py b/ixmp/tests/backend/test_ixmp4_io.py new file mode 100644 index 000000000..2fd71176e --- /dev/null +++ b/ixmp/tests/backend/test_ixmp4_io.py @@ -0,0 +1,388 @@ +from pathlib import Path +from typing import Literal, Union + +import pandas as pd +import pytest + +from ixmp.model.gams import gams_info +from ixmp.testing import min_ixmp4_version +from ixmp.util.ixmp4 import ContainerData + +pytestmark = min_ixmp4_version + + +# NOTE No return type for Python 3.9 compliance +@pytest.fixture +def container(): + from gams.transfer import Container + + return Container(system_directory=str(gams_info().system_dir)) + + +def test__record_versions(container) -> None: + from ixmp.backend.ixmp4_io import _record_versions + + # Test recording the package version of something that's always present + _record_versions(container=container, packages=["ixmp"]) + indexsets = container.getSets() + assert len(indexsets) == 1 + indexset = indexsets[0] + assert indexset.name == "ixmp_version" + assert indexset.domain == ["*", "*"] + + # Test recording the package version of something that's never present + _record_versions(container=container, packages=["not installed"]) + indexset = container.getSets()[0] + print(indexset.records) + pd.testing.assert_frame_equal( + indexset.records[["uni_0", "uni_1"]], + pd.DataFrame( + { + "uni_0": pd.Categorical(["not installed"], ordered=True), + "uni_1": pd.Categorical(["(not installed)"], ordered=True), + } + ), + ) + + +def test__update_item_in_container(container) -> None: + from ixmp.backend.ixmp4_io import _update_item_in_container + + item_list = [ + ContainerData( + name="table", kind="Table", records=["foo", "bar"], domain=["indexset"] + ), + ContainerData(name="scalar", kind="Scalar", records=3.14), + ContainerData(name="equation", kind="Equation", records=None), + ContainerData(name="variable", kind="Variable", records=None), + ] + + for item in item_list: + _update_item_in_container(container=container, item=item) + + assert all(container.hasSymbols(symbols=[item.name for item in item_list])) + + +def test__add_items_to_container(container) -> None: + from ixmp.backend.ixmp4_io import _add_items_to_container + + # Test handling of empty list + empty_list: list[ContainerData] = [] + _add_items_to_container(container=container, items=empty_list) + assert container.data == {} + + # Test adding new item + records = ["foo", "bar"] + item_list = [ContainerData(name="Indexset", kind="IndexSet", records=records)] + _add_items_to_container(container=container, items=item_list) + assert container.hasSymbols("Indexset") + indexsets = container.getSets() + assert len(indexsets) == 1 + indexset = indexsets[0] + assert indexset.records["uni"].to_list() == records + + # Test updating existing item + records = ["baz"] + item_list = [ContainerData(name="Indexset", kind="IndexSet", records=records)] + _add_items_to_container(container=container, items=item_list) + indexsets = container.getSets() + assert len(indexsets) == 1 + indexset = indexsets[0] + assert indexset.records["uni"].to_list() == records + + +# Overriding fixture that usually parametrizes test_mp +@pytest.fixture(scope="module") +def backend() -> Literal["ixmp4"]: + return "ixmp4" + + +# NOTE This marker usually parametrizes test creation, but with overriding `backend`, +# this might not be necessary. Keeping it for clarity. +@pytest.mark.ixmp4 +class TestIxmp4IOFunctions: + """Test group for all IO functions touching ixmp4 directly.""" + + def test__domain(self, run) -> None: + from ixmp.backend.ixmp4_io import _domain + + indexset = run.optimization.indexsets.create("Indexset") + table = run.optimization.tables.create( + "Table", constrained_to_indexsets=[indexset.name] + ) + + assert _domain(indexset) is None + assert _domain(table) == [indexset.name] + + def test__records(self, run) -> None: + from ixmp.backend.ixmp4_io import _records + + run.backend.units.create("unit") + + # Test records of a Scalar + scalar = run.optimization.scalars.create("Scalar", 3.14, "unit") + assert _records(scalar) == scalar.value + + # Test records of an empty Table + indexset = run.optimization.indexsets.create("Indexset") + table = run.optimization.tables.create( + "Table", constrained_to_indexsets=[indexset.name] + ) + assert _records(table) is None + + # Test records of an IndexSet + indexset.add(["foo", "bar"]) + assert _records(indexset) == indexset.data + + # Test records of a Parameter + parameter = run.optimization.parameters.create( + "Parameter", constrained_to_indexsets=[indexset.name] + ) + parameter.add( + {indexset.name: ["foo", "bar"], "values": [1, 2], "units": ["unit"] * 2} + ) + expected = parameter.data.copy() + expected.pop("units") + assert _records(parameter) == expected + + def test__ensure_correct_item_order(self, run) -> None: + from ixmp.backend.ixmp4_io import _ensure_correct_item_order + + years = run.optimization.indexsets.create("years") + nodes = run.optimization.indexsets.create("nodes") + type_years = run.optimization.indexsets.create("type_years") + type_nodes = run.optimization.indexsets.create("type_nodes") + + assert _ensure_correct_item_order( + items=[type_years, years, type_nodes, nodes], + repo=run.optimization.indexsets, + ) == [years, nodes, type_years, type_nodes] + + def test__align_records_and_domain(self, run) -> None: + from ixmp.backend.ixmp4_io import _align_records_and_domain + + # Test item without domain order + equation = run.optimization.equations.create("Equation") + expected: dict[str, Union[list[float], list[int], list[str]]] = { + "foo": [1, 2, 3] + } + assert _align_records_and_domain(item=equation, records=expected) == expected + + # Test sorting of records + indexset_1 = run.optimization.indexsets.create("Indexset 1") + indexset_2 = run.optimization.indexsets.create("Indexset 2") + parameter = run.optimization.parameters.create( + "Parameter", constrained_to_indexsets=[indexset_1.name, indexset_2.name] + ) + records: dict[str, Union[list[float], list[int], list[str]]] = { + indexset_2.name: [1, 2, 3], + "values": [1.0, 2.0, 3.0], + indexset_1.name: ["1", "2", "3"], + } + expected = { + indexset_1.name: ["1", "2", "3"], + indexset_2.name: [1, 2, 3], + "values": [1.0, 2.0, 3.0], + } + + assert _align_records_and_domain(item=parameter, records=records) == expected + + def test__convert_ixmp4_items_to_containerdata(self, run) -> None: + from ixmp.backend.ixmp4_io import _convert_ixmp4_items_to_containerdata + + # Test handling of empty list + empty_list = _convert_ixmp4_items_to_containerdata(items=[]) + assert len(empty_list) == 0 + + # Test adding an Indexset (covers no new cases, but we need to index a Table) + indexset = run.optimization.indexsets.create("Indexset") + indexset.add(data=["foo", "bar", "baz"]) + container_list = _convert_ixmp4_items_to_containerdata(items=[indexset]) + assert len(container_list) == 1 + indexset_data = container_list[0] + assert indexset.name == indexset_data.name + assert indexset.data == indexset_data.records + + # Test adding an item where records are a dict + table = run.optimization.tables.create( + "Table", constrained_to_indexsets=[indexset.name] + ) + table.add(data={indexset.name: ["baz", "foo", "bar"]}) + container_list = _convert_ixmp4_items_to_containerdata(items=[table]) + table_data = container_list[0] + assert table.name == table_data.name + assert table.data == table_data.records + + def test_write_run_to_gdx(self, run, tmp_path: Path) -> None: + from gams.transfer import Container + + from ixmp.backend.ixmp4_io import write_run_to_gdx + + # Test minimal call targeting all kinds of items + # NOTE All other functions are unit-tested already and other tests call write() + # with actual data, so we probably don't need to replicate here + file_path = tmp_path / "test_write.gdx" + write_run_to_gdx( + run=run, + file_name=file_path, + container_data=[], + record_version_packages=["ixmp"], + include_variables_and_equations=True, + ) + container = Container( + load_from=tmp_path / "test_write.gdx", + system_directory=str(gams_info().system_dir), + ) + assert len(container.data) == 1 + assert "ixmp_version" in container.data.keys() + + def test__set_columns_to_read_from_records(self, run) -> None: + from ixmp.backend.ixmp4_io import _set_columns_to_read_from_records + + default_columns = ["levels", "marginals"] + + # Test an Equation without dimensions + equation = run.backend.optimization.equations.create( + run_id=run.id, name="Equation" + ) + columns = _set_columns_to_read_from_records(item=equation) + assert columns == default_columns + + # Test an indexed Variable + indexset = run.optimization.indexsets.create("Indexset") + variable = run.backend.optimization.variables.create( + run_id=run.id, name="Variable", constrained_to_indexsets=[indexset.name] + ) + columns = _set_columns_to_read_from_records(item=variable) + assert columns == [indexset.name] + default_columns + + # Test an Equation with dimension names + equation_2 = run.backend.optimization.equations.create( + run_id=run.id, + name="Equation 2", + constrained_to_indexsets=[indexset.name], + column_names=["Column"], + ) + columns = _set_columns_to_read_from_records(item=equation_2) + assert columns == ["Column"] + default_columns + + # NOTE Keeping the read_equ()/read_var() tests separate because the read_*() + # functions are separate + + def test__read_variables_to_run(self, run, container) -> None: + from ixmp.backend.ixmp4_io import _read_variables_to_run + + indexset = run.optimization.indexsets.create("Indexset") + indexset.add(data=["foo", "bar", "baz"]) + variable = run.backend.optimization.variables.create( + run_id=run.id, name="Variable", constrained_to_indexsets=[indexset.name] + ) + + # NOTE A read GDX always contains 'level', 'marginal', 'lower', 'upper', 'scale' + # The first two will be changed to plural, the latter three ignored + records = { + indexset.name: ["foo", "bar", "baz"], + "level": [1.0, 2.0, 3.0], + "marginal": [0, 0, 0], + "lower": [0, 0, 0], + "upper": [0, 0, 0], + "scale": [0, 0, 0], + } + container.addVariable(name="Variable", domain=[indexset.name], records=records) + + _read_variables_to_run(container=container, run=run, variables=[variable]) + expected = { + indexset.name: records[indexset.name], + "levels": records["level"], + "marginals": records["marginal"], + } + variable = run.backend.optimization.variables.get( + run_id=run.id, name=variable.name + ) + assert variable.data == expected + + def test__read_equations_to_run(self, run, container) -> None: + from ixmp.backend.ixmp4_io import _read_equations_to_run + + indexset = run.optimization.indexsets.create("Indexset") + indexset.add(data=["foo", "bar", "baz"]) + equation = run.backend.optimization.equations.create( + run_id=run.id, name="Equation", constrained_to_indexsets=[indexset.name] + ) + + # NOTE A read GDX always contains 'level', 'marginal', 'lower', 'upper', 'scale' + # The first two will be changed to plural, the latter three ignored + records = { + indexset.name: ["foo", "bar", "baz"], + "level": [1.0, 2.0, 3.0], + "marginal": [0, 0, 0], + "lower": [0, 0, 0], + "upper": [0, 0, 0], + "scale": [0, 0, 0], + } + container.addEquation( + name="Equation", type="E", domain=[indexset.name], records=records + ) + + _read_equations_to_run(container=container, run=run, equations=[equation]) + expected = { + indexset.name: records[indexset.name], + "levels": records["level"], + "marginals": records["marginal"], + } + equation = run.backend.optimization.equations.get( + run_id=run.id, name=equation.name + ) + assert equation.data == expected + + def test_read_gdx_to_run(self, run, tmp_path: Path) -> None: + from ixmp.backend.ixmp4_io import read_gdx_to_run, write_run_to_gdx + + # NOTE Names without space to produce "valid GAMS names" + variable_1 = run.backend.optimization.variables.create( + run_id=run.id, name="Variable1" + ) + variable_2 = run.backend.optimization.variables.create( + run_id=run.id, name="Variable2" + ) + variable_3 = run.backend.optimization.variables.create( + run_id=run.id, name="Variable3" + ) + records: dict[str, Union[list[float], list[int], list[str]]] = { + "level": [1.0], + "marginal": [0], + "lower": [0], + "upper": [0], + "scale": [0], + } + file_path = tmp_path / "test_read.gdx" + write_run_to_gdx( + run=run, + file_name=file_path, + container_data=[ + ContainerData(name=var.name, kind="Variable", records=records) + for var in [variable_1, variable_2, variable_3] + ], + record_version_packages=["ixmp"], + include_variables_and_equations=True, + ) + var_list = [variable_1.name, variable_3.name] + + # Test minimal confirmable call (with otherwise unused parameters) + read_gdx_to_run( + run=run, + result_file=file_path, + equ_list=[], + var_list=var_list, + comment="Test comment", + check_solution=True, + ) + for var in run.optimization.variables.list(): + # 'variable_2' is not supposed to be read in + expected = ( + {} + if var.name == variable_2.name + else {"levels": records["level"], "marginals": records["marginal"]} + ) + + assert var.data == expected diff --git a/ixmp/tests/backend/test_jdbc.py b/ixmp/tests/backend/test_jdbc.py index 3642ee80c..b0e722b9e 100644 --- a/ixmp/tests/backend/test_jdbc.py +++ b/ixmp/tests/backend/test_jdbc.py @@ -61,6 +61,8 @@ def test_close_default_logging(test_mp_f, capfd): assert captured.out == "" +# NOTE IXMP4Backend's close_db() is a noop +@pytest.mark.jdbc def test_close_increased_logging(test_mp_f, capfd): """Platform.close_db() doesn't throw needless exceptions.""" # Use the session-scoped fixture to avoid affecting other tests in this file @@ -217,6 +219,8 @@ def test_gc(self, monkeypatch, be): be.gc() +# TODO IXMP4Backend needs to handle change_scalar() correctly +@pytest.mark.jdbc def test_exceptions(test_mp): """Ensure that Python exceptions are raised for some actions.""" s = ixmp.Scenario(test_mp, "model name", "scenario name", "new") @@ -366,6 +370,8 @@ def exception_verbose_true(): ixmp.backend.jdbc._EXCEPTION_VERBOSE = tmp # Restore value +# FIMXE This raises a RunNotFound on IXMP4Backend +@pytest.mark.jdbc def test_verbose_exception(test_mp, exception_verbose_true): # Exception stack trace is logged for debugging with pytest.raises(RuntimeError) as exc_info: @@ -654,6 +660,8 @@ def test_reload_cycle( memory_usage("shutdown") +# TODO Not yet implemented by IXMP4Backend +@pytest.mark.jdbc def test_docs(test_mp, request): scen = make_dantzig(test_mp, request=request) # test model docs diff --git a/ixmp/tests/core/test_meta.py b/ixmp/tests/core/test_meta.py index 7d91f7825..dda74d564 100644 --- a/ixmp/tests/core/test_meta.py +++ b/ixmp/tests/core/test_meta.py @@ -3,6 +3,7 @@ # behaviour they actually test. import copy +from typing import Any, Generator import pytest @@ -25,7 +26,7 @@ @pytest.fixture(scope="function") -def mp(test_mp_f): +def mp(test_mp_f: ixmp.Platform) -> Generator[ixmp.Platform, Any, None]: """A test Platform. The platform contains one time series with the "dantizg" model & scenario name from @@ -41,311 +42,312 @@ def mp(test_mp_f): yield test_mp_f -@pytest.mark.parametrize("meta", META_ENTRIES) -def test_set_meta_missing_argument(mp, meta): - with pytest.raises(ValueError): - mp.set_meta(meta) - with pytest.raises(ValueError): - mp.set_meta(meta, model=DANTZIG["model"], version=0) - with pytest.raises(ValueError): - mp.set_meta(meta, scenario=DANTZIG["scenario"], version=0) - - -@pytest.mark.parametrize("meta", META_ENTRIES) -def test_set_get_meta(mp, meta): - """Assert that storing+retrieving meta yields expected values.""" - mp.set_meta(meta, model=DANTZIG["model"]) - obs = mp.get_meta(model=DANTZIG["model"]) - assert obs == meta - - -@pytest.mark.parametrize("meta", META_ENTRIES) -def test_unique_meta(mp, meta): - """ - When setting a meta category on two distinct levels, a uniqueness error is - expected. - """ - scenario = ixmp.Scenario(mp, **DANTZIG, version="new") - scenario.commit("save dummy scenario") - mp.set_meta(meta, model=DANTZIG["model"]) - expected = ( - r"The meta category .* is already used at another level: " - r"model canning problem, scenario null, version null" - ) - with pytest.raises(Exception, match=expected): - mp.set_meta(meta, **DANTZIG, version=scenario.version) - scen = ixmp.Scenario(mp, **DANTZIG) - with pytest.raises(Exception, match=expected): +# TODO IXMP4Backend needs to handle meta data +@pytest.mark.jdbc +class TestMeta: + @pytest.mark.parametrize("meta", META_ENTRIES) + def test_set_meta_missing_argument(self, mp: ixmp.Platform, meta) -> None: + with pytest.raises(ValueError): + mp.set_meta(meta) + with pytest.raises(ValueError): + mp.set_meta(meta, model=DANTZIG["model"], version=0) + with pytest.raises(ValueError): + mp.set_meta(meta, scenario=DANTZIG["scenario"], version=0) + + @pytest.mark.parametrize("meta", META_ENTRIES) + def test_set_get_meta(self, mp: ixmp.Platform, meta) -> None: + """Assert that storing+retrieving meta yields expected values.""" + mp.set_meta(meta, model=DANTZIG["model"]) + obs = mp.get_meta(model=DANTZIG["model"]) + assert obs == meta + + @pytest.mark.parametrize("meta", META_ENTRIES) + def test_unique_meta(self, mp: ixmp.Platform, meta) -> None: + """ + When setting a meta category on two distinct levels, a uniqueness error is + expected. + """ + scenario = ixmp.Scenario(mp, **DANTZIG, version="new") + scenario.commit("save dummy scenario") + mp.set_meta(meta, model=DANTZIG["model"]) + expected = ( + r"The meta category .* is already used at another level: " + r"model canning problem, scenario null, version null" + ) + with pytest.raises(Exception, match=expected): + mp.set_meta(meta, **DANTZIG, version=scenario.version) + scen = ixmp.Scenario(mp, **DANTZIG) + with pytest.raises(Exception, match=expected): + scen.set_meta(meta) + # changing the category value type of an entry should also raise an error + meta = {"sample_entry": 3} + mp.set_meta(meta, **DANTZIG) + meta["sample_entry"] = "test-string" + expected = ( + r"The meta category .* is already used at another level: " + r"model canning problem, scenario standard, version null" + ) + with pytest.raises(Exception, match=expected): + mp.set_meta(meta, **DANTZIG, version=scenario.version) + + @pytest.mark.parametrize("meta", META_ENTRIES) + def test_set_get_meta_equals(self, mp: ixmp.Platform, meta) -> None: + initial_meta = mp.get_meta(scenario=DANTZIG["scenario"]) + mp.set_meta(meta, model=DANTZIG["model"]) + obs_meta = mp.get_meta(scenario=DANTZIG["scenario"]) + assert obs_meta == initial_meta + + @pytest.mark.parametrize("meta", META_ENTRIES) + def test_unique_meta_model_scenario(self, mp: ixmp.Platform, meta) -> None: + """ + When setting a meta key for a Model, it shouldn't be possible to set it + for a Model+Scenario then. + """ + mp.set_meta(meta, model=DANTZIG["model"]) + expected = r"The meta category .* is already used at another level: " + with pytest.raises(Exception, match=expected): + mp.set_meta(meta, **DANTZIG) + + # Setting this meta category on a new model should fail too + dantzig2 = { + "model": "canning problem 2", + "scenario": "standard", + } + mp.add_model_name(dantzig2["model"]) + expected = r"The meta category .* is already used at another level: " + with pytest.raises(Exception, match=expected): + mp.set_meta(meta, **dantzig2) + + @pytest.mark.parametrize("meta", META_ENTRIES) + def test_get_meta_strict(self, mp: ixmp.Platform, meta) -> None: + """ + Set meta indicators on several model/scenario/version levels and test + the 'strict' parameter of get_meta(). + """ + # set meta on various levels + model_meta = { + "model_int": 3, + "model_string": "string_value", + "model_bool": False, + } + scenario_meta = { + "scenario_int": 3, + "scenario_string": "string_value", + "scenario_bool": False, + } + meta2 = { + "sample_int2": 3, + "sample_string2": "string_value2", + "sample_bool2": False, + } + meta3 = { + "sample_int3": 3, + "sample_string3": "string_value3", + "sample_bool3": False, + "mixed3": ["string", 0.01, 2, True], + } + meta_scen = { + "sample_int4": 3, + "sample_string4": "string_value4", + "sample_bool4": False, + "mixed4": ["string", 0.01, 2, True], + } + scenario2 = "standard 2" + model2 = "canning problem 2" + mp.add_scenario_name(scenario2) + mp.add_model_name(model2) + dantzig2 = { + "model": model2, + "scenario": "standard", + } + dantzig3 = { + "model": model2, + "scenario": scenario2, + } + mp.set_meta(model_meta, model=DANTZIG["model"]) + mp.set_meta(scenario_meta, scenario=DANTZIG["scenario"]) + mp.set_meta(meta, **DANTZIG) + mp.set_meta(meta2, **dantzig2) + mp.set_meta(meta3, **dantzig3) + scen = ixmp.Scenario(mp, **DANTZIG, version="new") + scen.commit("save dummy scenario") + scen.set_meta(meta_scen) + + # Retrieve and validate meta indicators + # model + obs1 = mp.get_meta(model=DANTZIG["model"]) + assert obs1 == model_meta + # scenario + obs2 = mp.get_meta(scenario=DANTZIG["scenario"], strict=True) + assert obs2 == scenario_meta + # model+scenario + obs3 = mp.get_meta(**DANTZIG) + exp3 = copy.copy(meta) + exp3.update(model_meta) + exp3.update(scenario_meta) + assert obs3 == exp3 + # model+scenario, strict + obs3_strict = mp.get_meta(**DANTZIG, strict=True) + assert obs3_strict == meta + assert obs3 != obs3_strict + + # second model+scenario combination + obs4 = mp.get_meta(**dantzig2) + exp4 = copy.copy(meta2) + exp4.update(scenario_meta) + assert obs4 == exp4 + # second model+scenario combination, strict + obs4_strict = mp.get_meta(**dantzig2, strict=True) + assert obs4_strict == meta2 + assert obs4 != obs4_strict + + # second model+scenario combination + obs5 = mp.get_meta(**dantzig3) + exp5 = copy.copy(meta3) + assert obs5 == exp5 + + # model+scenario+version + obs6 = mp.get_meta(**DANTZIG, version=scen.version) + exp6 = copy.copy(meta_scen) + exp6.update(meta) + exp6.update(model_meta) + exp6.update(scenario_meta) + assert obs6 == exp6 + obs6_strict = mp.get_meta( + DANTZIG["model"], DANTZIG["scenario"], scen.version, strict=True + ) + assert obs6_strict == meta_scen + + @pytest.mark.parametrize("meta", META_ENTRIES) + def test_unique_meta_scenario(self, mp: ixmp.Platform, meta) -> None: + """ + When setting a meta key on a specific Scenario run, setting the same key + on an higher level (Model or Model+Scenario) should fail. + """ + scen = ixmp.Scenario(mp, **DANTZIG) scen.set_meta(meta) - # changing the category value type of an entry should also raise an error - meta = {"sample_entry": 3} - mp.set_meta(meta, **DANTZIG) - meta["sample_entry"] = "test-string" - expected = ( - r"The meta category .* is already used at another level: " - r"model canning problem, scenario standard, version null" - ) - with pytest.raises(Exception, match=expected): - mp.set_meta(meta, **DANTZIG, version=scenario.version) - - -@pytest.mark.parametrize("meta", META_ENTRIES) -def test_set_get_meta_equals(mp, meta): - initial_meta = mp.get_meta(scenario=DANTZIG["scenario"]) - mp.set_meta(meta, model=DANTZIG["model"]) - obs_meta = mp.get_meta(scenario=DANTZIG["scenario"]) - assert obs_meta == initial_meta - - -@pytest.mark.parametrize("meta", META_ENTRIES) -def test_unique_meta_model_scenario(mp, meta): - """ - When setting a meta key for a Model, it shouldn't be possible to set it - for a Model+Scenario then. - """ - mp.set_meta(meta, model=DANTZIG["model"]) - expected = r"The meta category .* is already used at another level: " - with pytest.raises(Exception, match=expected): + # add a second scenario and verify that setting+getting Meta works + scen2 = ixmp.Scenario(mp, **DANTZIG, version="new") + scen2.commit("save dummy scenario") + scen2.set_meta(meta) + assert scen2.get_meta() == scen.get_meta() + + expected = ( + r"The meta category .* is already used at another level: " + r"model canning problem, scenario standard, " + ) + with pytest.raises(Exception, match=expected): + mp.set_meta(meta, **DANTZIG) + with pytest.raises(Exception, match=expected): + mp.set_meta(meta, model=DANTZIG["model"]) + + def test_meta_partial_overwrite(self, mp: ixmp.Platform) -> None: + meta1 = { + "sample_string": 3.0, + "another_string": "string_value", + "sample_bool": False, + } + meta2 = { + "sample_string": 5.0, + "yet_another_string": "hello", + "sample_bool": True, + } + scen = ixmp.Scenario(mp, **DANTZIG) + scen.set_meta(meta1) + scen.set_meta(meta2) + expected = copy.copy(meta1) + expected.update(meta2) + obs = scen.get_meta() + assert obs == expected + + def test_remove_meta(self, mp: ixmp.Platform) -> None: + meta = {"sample_int": 3.0, "another_string": "string_value"} + remove_key = "another_string" mp.set_meta(meta, **DANTZIG) + mp.remove_meta(remove_key, **DANTZIG) + expected = copy.copy(meta) + del expected[remove_key] + obs = mp.get_meta(**DANTZIG) + assert expected == obs + + def test_remove_invalid_meta(self, mp: ixmp.Platform) -> None: + """ + Removing nonexisting meta entries or None shouldn't result in any meta + being removed. Providing None should give a ValueError. + """ + mp.set_meta(SAMPLE_META, **DANTZIG) + with pytest.raises(ValueError): + mp.remove_meta(None, **DANTZIG) + mp.remove_meta("nonexisting_category", **DANTZIG) + mp.remove_meta([], **DANTZIG) + obs = mp.get_meta(**DANTZIG) + assert obs == SAMPLE_META + + def test_set_and_remove_meta_scenario(self, mp: ixmp.Platform) -> None: + """ + Test partial overwriting and meta deletion on scenario level. + """ + meta1 = {"sample_string": 3.0, "another_string": "string_value"} + meta2 = {"sample_string": 5.0, "yet_another_string": "hello"} + remove_key = "another_string" + + scen = ixmp.Scenario(mp, **DANTZIG) + scen.set_meta(meta1) + scen.set_meta(meta2) + expected = copy.copy(meta1) + expected.update(meta2) + obs = scen.get_meta() + assert expected == obs + + scen.remove_meta(remove_key) + del expected[remove_key] + obs = scen.get_meta() + assert obs == expected + + def test_scenario_delete_meta_warning(self, mp: ixmp.Platform) -> None: + """ + Scenario.delete_meta works but raises a deprecation warning. + + This test can be removed once Scenario.delete_meta is removed. + """ + scen = ixmp.Scenario(mp, **DANTZIG) + meta = {"sample_int": 3, "sample_string": "string_value"} + remove_key = "sample_string" - # Setting this meta category on a new model should fail too - dantzig2 = { - "model": "canning problem 2", - "scenario": "standard", - } - mp.add_model_name(dantzig2["model"]) - expected = r"The meta category .* is already used at another level: " - with pytest.raises(Exception, match=expected): - mp.set_meta(meta, **dantzig2) - - -@pytest.mark.parametrize("meta", META_ENTRIES) -def test_get_meta_strict(mp, meta): - """ - Set meta indicators on several model/scenario/version levels and test - the 'strict' parameter of get_meta(). - """ - # set meta on various levels - model_meta = {"model_int": 3, "model_string": "string_value", "model_bool": False} - scenario_meta = { - "scenario_int": 3, - "scenario_string": "string_value", - "scenario_bool": False, - } - meta2 = {"sample_int2": 3, "sample_string2": "string_value2", "sample_bool2": False} - meta3 = { - "sample_int3": 3, - "sample_string3": "string_value3", - "sample_bool3": False, - "mixed3": ["string", 0.01, 2, True], - } - meta_scen = { - "sample_int4": 3, - "sample_string4": "string_value4", - "sample_bool4": False, - "mixed4": ["string", 0.01, 2, True], - } - scenario2 = "standard 2" - model2 = "canning problem 2" - mp.add_scenario_name(scenario2) - mp.add_model_name(model2) - dantzig2 = { - "model": model2, - "scenario": "standard", - } - dantzig3 = { - "model": model2, - "scenario": scenario2, - } - mp.set_meta(model_meta, model=DANTZIG["model"]) - mp.set_meta(scenario_meta, scenario=DANTZIG["scenario"]) - mp.set_meta(meta, **DANTZIG) - mp.set_meta(meta2, **dantzig2) - mp.set_meta(meta3, **dantzig3) - scen = ixmp.Scenario(mp, **DANTZIG, version="new") - scen.commit("save dummy scenario") - scen.set_meta(meta_scen) - - # Retrieve and validate meta indicators - # model - obs1 = mp.get_meta(model=DANTZIG["model"]) - assert obs1 == model_meta - # scenario - obs2 = mp.get_meta(scenario=DANTZIG["scenario"], strict=True) - assert obs2 == scenario_meta - # model+scenario - obs3 = mp.get_meta(**DANTZIG) - exp3 = copy.copy(meta) - exp3.update(model_meta) - exp3.update(scenario_meta) - assert obs3 == exp3 - # model+scenario, strict - obs3_strict = mp.get_meta(**DANTZIG, strict=True) - assert obs3_strict == meta - assert obs3 != obs3_strict - - # second model+scenario combination - obs4 = mp.get_meta(**dantzig2) - exp4 = copy.copy(meta2) - exp4.update(scenario_meta) - assert obs4 == exp4 - # second model+scenario combination, strict - obs4_strict = mp.get_meta(**dantzig2, strict=True) - assert obs4_strict == meta2 - assert obs4 != obs4_strict - - # second model+scenario combination - obs5 = mp.get_meta(**dantzig3) - exp5 = copy.copy(meta3) - assert obs5 == exp5 - - # model+scenario+version - obs6 = mp.get_meta(**DANTZIG, version=scen.version) - exp6 = copy.copy(meta_scen) - exp6.update(meta) - exp6.update(model_meta) - exp6.update(scenario_meta) - assert obs6 == exp6 - obs6_strict = mp.get_meta( - DANTZIG["model"], DANTZIG["scenario"], scen.version, strict=True - ) - assert obs6_strict == meta_scen - - -@pytest.mark.parametrize("meta", META_ENTRIES) -def test_unique_meta_scenario(mp, meta): - """ - When setting a meta key on a specific Scenario run, setting the same key - on an higher level (Model or Model+Scenario) should fail. - """ - scen = ixmp.Scenario(mp, **DANTZIG) - scen.set_meta(meta) - # add a second scenario and verify that setting+getting Meta works - scen2 = ixmp.Scenario(mp, **DANTZIG, version="new") - scen2.commit("save dummy scenario") - scen2.set_meta(meta) - assert scen2.get_meta() == scen.get_meta() - - expected = ( - r"The meta category .* is already used at another level: " - r"model canning problem, scenario standard, " - ) - with pytest.raises(Exception, match=expected): - mp.set_meta(meta, **DANTZIG) - with pytest.raises(Exception, match=expected): + scen.set_meta(meta) + with pytest.warns(DeprecationWarning): + scen.delete_meta(remove_key) + expected = copy.copy(meta) + del expected[remove_key] + obs = scen.get_meta() + assert obs == expected + + def test_meta_arguments(self, mp: ixmp.Platform) -> None: + """Set scenario meta with key-value arguments""" + meta = {"sample_int": 3} + scen = ixmp.Scenario(mp, **DANTZIG) + scen.set_meta(meta) + # add a second scenario and verify that setting Meta for it works + scen2 = ixmp.Scenario(mp, **DANTZIG, version="new") + scen2.commit("save dummy scenario") + scen2.set_meta(*meta.popitem()) + assert scen.get_meta() == scen2.get_meta() + + def test_update_meta_lists(self, mp: ixmp.Platform) -> None: + """Set metadata categories having list/array values.""" + SAMPLE_META = {"list_category": ["a", "b", "c"]} + mp.set_meta(SAMPLE_META, model=DANTZIG["model"]) + obs = mp.get_meta(model=DANTZIG["model"]) + assert obs == SAMPLE_META + # try updating meta + SAMPLE_META = {"list_category": ["a", "e", "f"]} + mp.set_meta(SAMPLE_META, model=DANTZIG["model"]) + obs = mp.get_meta(model=DANTZIG["model"]) + assert obs == SAMPLE_META + + def test_meta_mixed_list(self, mp: ixmp.Platform) -> None: + """Set metadata categories having list/array values.""" + meta = {"mixed_category": ["string", 0.01, True]} mp.set_meta(meta, model=DANTZIG["model"]) - - -def test_meta_partial_overwrite(mp): - meta1 = { - "sample_string": 3.0, - "another_string": "string_value", - "sample_bool": False, - } - meta2 = {"sample_string": 5.0, "yet_another_string": "hello", "sample_bool": True} - scen = ixmp.Scenario(mp, **DANTZIG) - scen.set_meta(meta1) - scen.set_meta(meta2) - expected = copy.copy(meta1) - expected.update(meta2) - obs = scen.get_meta() - assert obs == expected - - -def test_remove_meta(mp): - meta = {"sample_int": 3.0, "another_string": "string_value"} - remove_key = "another_string" - mp.set_meta(meta, **DANTZIG) - mp.remove_meta(remove_key, **DANTZIG) - expected = copy.copy(meta) - del expected[remove_key] - obs = mp.get_meta(**DANTZIG) - assert expected == obs - - -def test_remove_invalid_meta(mp): - """ - Removing nonexisting meta entries or None shouldn't result in any meta - being removed. Providing None should give a ValueError. - """ - mp.set_meta(SAMPLE_META, **DANTZIG) - with pytest.raises(ValueError): - mp.remove_meta(None, **DANTZIG) - mp.remove_meta("nonexisting_category", **DANTZIG) - mp.remove_meta([], **DANTZIG) - obs = mp.get_meta(**DANTZIG) - assert obs == SAMPLE_META - - -def test_set_and_remove_meta_scenario(mp): - """ - Test partial overwriting and meta deletion on scenario level. - """ - meta1 = {"sample_string": 3.0, "another_string": "string_value"} - meta2 = {"sample_string": 5.0, "yet_another_string": "hello"} - remove_key = "another_string" - - scen = ixmp.Scenario(mp, **DANTZIG) - scen.set_meta(meta1) - scen.set_meta(meta2) - expected = copy.copy(meta1) - expected.update(meta2) - obs = scen.get_meta() - assert expected == obs - - scen.remove_meta(remove_key) - del expected[remove_key] - obs = scen.get_meta() - assert obs == expected - - -def test_scenario_delete_meta_warning(mp): - """ - Scenario.delete_meta works but raises a deprecation warning. - - This test can be removed once Scenario.delete_meta is removed. - """ - scen = ixmp.Scenario(mp, **DANTZIG) - meta = {"sample_int": 3, "sample_string": "string_value"} - remove_key = "sample_string" - - scen.set_meta(meta) - with pytest.warns(DeprecationWarning): - scen.delete_meta(remove_key) - expected = copy.copy(meta) - del expected[remove_key] - obs = scen.get_meta() - assert obs == expected - - -def test_meta_arguments(mp): - """Set scenario meta with key-value arguments""" - meta = {"sample_int": 3} - scen = ixmp.Scenario(mp, **DANTZIG) - scen.set_meta(meta) - # add a second scenario and verify that setting Meta for it works - scen2 = ixmp.Scenario(mp, **DANTZIG, version="new") - scen2.commit("save dummy scenario") - scen2.set_meta(*meta.popitem()) - assert scen.get_meta() == scen2.get_meta() - - -def test_update_meta_lists(mp): - """Set metadata categories having list/array values.""" - SAMPLE_META = {"list_category": ["a", "b", "c"]} - mp.set_meta(SAMPLE_META, model=DANTZIG["model"]) - obs = mp.get_meta(model=DANTZIG["model"]) - assert obs == SAMPLE_META - # try updating meta - SAMPLE_META = {"list_category": ["a", "e", "f"]} - mp.set_meta(SAMPLE_META, model=DANTZIG["model"]) - obs = mp.get_meta(model=DANTZIG["model"]) - assert obs == SAMPLE_META - - -def test_meta_mixed_list(mp): - """Set metadata categories having list/array values.""" - meta = {"mixed_category": ["string", 0.01, True]} - mp.set_meta(meta, model=DANTZIG["model"]) - obs = mp.get_meta(model=DANTZIG["model"]) - assert obs == meta + obs = mp.get_meta(model=DANTZIG["model"]) + assert obs == meta diff --git a/ixmp/tests/core/test_platform.py b/ixmp/tests/core/test_platform.py index bca620420..4bcbf2caa 100644 --- a/ixmp/tests/core/test_platform.py +++ b/ixmp/tests/core/test_platform.py @@ -2,8 +2,10 @@ import logging import re +from collections.abc import Generator +from pathlib import Path from sys import getrefcount -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Literal from weakref import getweakrefcount import pandas as pd @@ -13,37 +15,59 @@ import ixmp from ixmp.backend import FIELDS -from ixmp.testing import DATA, assert_logs, models +from ixmp.testing import DATA, assert_logs, min_ixmp4_version, models if TYPE_CHECKING: - from ixmp import Platform + pass class TestPlatform: - def test_init(self): + def test_init0(self) -> None: with pytest.raises( - ValueError, match=re.escape("backend class 'foo' not among ['jdbc']") + ValueError, + match=re.escape("backend class 'foo' not among"), ): - ixmp.Platform(backend="foo") + # Testing the wrong type on purpose + ixmp.Platform(backend="foo") # type: ignore[arg-type] # name="default" is used, referring to "local" mp = ixmp.Platform() assert "local" == mp.name - def test_getattr(self, test_mp): + # NOTE Can't use 'backend' due to duplicate parametrization + @pytest.mark.parametrize( + "_backend, backend_args", + ( + ("jdbc", dict(driver="hsqldb", url="jdbc:hsqldb:mem:TestPlatform")), + pytest.param("ixmp4", dict(), marks=min_ixmp4_version), + ), + ) + def test_init1( + self, _backend: Literal["jdbc", "ixmp4"], backend_args: dict[str, str] + ) -> None: + # Platform can be instantiated + ixmp.Platform(backend=_backend, **backend_args) + + def test_getattr(self, test_mp: ixmp.Platform) -> None: """Test __getattr__.""" with pytest.raises(AttributeError): test_mp.not_a_direct_backend_method + def test_scenario_list(self, mp: ixmp.Platform) -> None: + scenario = mp.scenario_list() + assert isinstance(scenario, pd.DataFrame) + @pytest.fixture -def log_level_mp(test_mp): +def log_level_mp(test_mp: ixmp.Platform) -> Generator[ixmp.Platform, Any, None]: """A fixture that preserves the log level of *test_mp*.""" tmp = test_mp.get_log_level() yield test_mp test_mp.set_log_level(tmp) +# TODO Not sure why 'NOTSET' fails on IXMP4Backend +@pytest.mark.jdbc @pytest.mark.parametrize( "level, exc", [ @@ -57,7 +81,7 @@ def log_level_mp(test_mp): ("FOO", ValueError), ], ) -def test_log_level(log_level_mp, level, exc): +def test_log_level(log_level_mp: ixmp.Platform, level: str, exc) -> None: """Log level can be set and retrieved.""" if exc is None: log_level_mp.set_log_level(level) @@ -67,12 +91,14 @@ def test_log_level(log_level_mp, level, exc): log_level_mp.set_log_level(level) -def test_scenario_list(mp): +def test_scenario_list(mp: ixmp.Platform) -> None: scenario = mp.scenario_list(model="Douglas Adams")["scenario"] assert scenario[0] == "Hitchhiker" -def test_export_timeseries_data(mp: "Platform", tmp_path) -> None: +# TODO Not sure why this fails on IXMP4Backend +@pytest.mark.jdbc +def test_export_timeseries_data(mp: ixmp.Platform, tmp_path: Path) -> None: path = tmp_path / "export.csv" mp.export_timeseries_data(path, model="Douglas Adams", unit="???", region="World") @@ -87,7 +113,7 @@ def test_export_timeseries_data(mp: "Platform", tmp_path) -> None: assert_frame_equal(exp, obs) -def test_export_ts_wrong_params(test_mp, tmp_path): +def test_export_ts_wrong_params(test_mp: ixmp.Platform, tmp_path: Path) -> None: """Platform.export_timeseries_data to raise error with wrong parameters.""" path = tmp_path / "export.csv" with raises(ValueError, match="Invalid arguments"): @@ -100,7 +126,9 @@ def test_export_ts_wrong_params(test_mp, tmp_path): ) -def test_export_ts_of_all_runs(mp, tmp_path): +# TODO Not sure why this fails on IXMP4Backend +@pytest.mark.jdbc +def test_export_ts_of_all_runs(mp: ixmp.Platform, tmp_path: Path) -> None: """Export timeseries of all runs.""" path = tmp_path / "export.csv" @@ -114,11 +142,15 @@ def test_export_ts_of_all_runs(mp, tmp_path): mp.export_timeseries_data( path, unit="???", region="World", default=True, export_all_runs=True ) + # Find the one default version of the expected scenario + exp_default_version = mp.scenario_list( + model=models["h2g2"]["model"], scen=models["h2g2"]["scenario"] + )["version"].item() obs = pd.read_csv(path, index_col=False, header=0) exp = ( DATA[0] - .assign(**models["h2g2"], version=2, subannual="Year", meta=0) + .assign(**models["h2g2"], version=exp_default_version, subannual="Year", meta=0) .rename(columns=lambda c: c.upper()) .reindex(columns=FIELDS["write_file"]) ) @@ -130,10 +162,15 @@ def test_export_ts_of_all_runs(mp, tmp_path): path, unit="???", region="World", default=False, export_all_runs=True ) obs = pd.read_csv(path, index_col=False, header=0) - assert 4 == len(obs) + + # We only have 1 model, scenario, region, variable, unit, so we expect lines equal + # to number of versions (equals default version number) times number of years: + expected = exp_default_version * len(DATA[0]["year"]) # Usually 4 + + assert expected == len(obs) -def test_export_timeseries_data_empty(mp, tmp_path): +def test_export_timeseries_data_empty(mp: ixmp.Platform, tmp_path: Path) -> None: """Dont export data if given models/scenarios do not have any runs.""" path = tmp_path / "export.csv" model = "model-no-run" @@ -145,16 +182,16 @@ def test_export_timeseries_data_empty(mp, tmp_path): assert 0 == len(pd.read_csv(path, index_col=False, header=0)) -def test_unit_list(test_mp): +def test_unit_list(test_mp: ixmp.Platform) -> None: units = test_mp.units() assert ("cases" in units) is True -def test_add_unit(test_mp): +def test_add_unit(test_mp: ixmp.Platform) -> None: test_mp.add_unit("test", "just testing") -def test_regions(test_mp): +def test_regions(test_mp: ixmp.Platform) -> None: regions = test_mp.regions() # Result has the expected columns @@ -166,7 +203,9 @@ def test_regions(test_mp): assert all([list(obs.loc[0]) == ["World", None, "World", "common"]]) -def test_add_region(test_mp): +# NOTE IXMP4(Backend) doesn't store `Parent`; instead, it returns the region name itself +@pytest.mark.jdbc +def test_add_region(test_mp: ixmp.Platform) -> None: # Region can be added test_mp.add_region("foo", "bar", "World") @@ -176,7 +215,9 @@ def test_add_region(test_mp): assert all([list(obs.loc[0]) == ["foo", None, "World", "bar"]]) -def test_add_region_synonym(test_mp): +# NOTE IXMP4Backend doesn't handle synonyms, it might not ever +@pytest.mark.jdbc +def test_add_region_synonym(test_mp: ixmp.Platform) -> None: test_mp.add_region("foo", "bar", "World") test_mp.add_region_synonym("foo2", "foo") regions = test_mp.regions() @@ -192,7 +233,9 @@ def test_add_region_synonym(test_mp): assert_frame_equal(obs, exp) -def test_timeslices(test_mp): +# TODO Not yet implemented on IXMP4Backend +@pytest.mark.jdbc +def test_timeslices(test_mp: ixmp.Platform) -> None: timeslices = test_mp.timeslices() obs = timeslices[timeslices.category == "Common"] # result has all attributes of time slice @@ -201,7 +244,9 @@ def test_timeslices(test_mp): assert all([list(obs.iloc[0]) == ["Year", "Common", 1.0]]) -def test_add_timeslice(test_mp): +# TODO Not yet implemented on IXMP4Backend +@pytest.mark.jdbc +def test_add_timeslice(test_mp: ixmp.Platform) -> None: test_mp.add_timeslice("January, 1st", "Days", 1.0 / 366) timeslices = test_mp.timeslices() obs = timeslices[timeslices.category == "Days"] @@ -211,7 +256,9 @@ def test_add_timeslice(test_mp): assert all([list(obs.iloc[0]) == ["January, 1st", "Days", 1.0 / 366]]) -def test_add_timeslice_duplicate(caplog, test_mp): +# TODO Not yet implemented on IXMP4Backend +@pytest.mark.jdbc +def test_add_timeslice_duplicate(caplog, test_mp: ixmp.Platform) -> None: test_mp.add_timeslice("foo_slice", "foo_category", 0.2) # Adding same name with different duration raises an error @@ -224,7 +271,7 @@ def test_add_timeslice_duplicate(caplog, test_mp): test_mp.add_timeslice("foo_slice", "bar_category", 0.2) -def test_weakref(): +def test_weakref() -> None: """Weak references allow Platforms to be del'd while Scenarios live.""" mp = ixmp.Platform( backend="jdbc", @@ -267,11 +314,11 @@ def test_weakref(): # *s* is garbage-collected at this point -def test_add_model_name(test_mp): +def test_add_model_name(test_mp: ixmp.Platform) -> None: test_mp.add_model_name("new_model_name") assert "new_model_name" in test_mp.get_model_names() -def test_add_scenario_name(test_mp): +def test_add_scenario_name(test_mp: ixmp.Platform) -> None: test_mp.add_scenario_name("new_scenario_name") assert "new_scenario_name" in test_mp.get_scenario_names() diff --git a/ixmp/tests/core/test_scenario.py b/ixmp/tests/core/test_scenario.py index f2b17b1ef..53d09f258 100644 --- a/ixmp/tests/core/test_scenario.py +++ b/ixmp/tests/core/test_scenario.py @@ -45,6 +45,8 @@ class TestScenario: """Tests of :class:`ixmp.Scenario`.""" # Initialize Scenario + # NOTE IXMP4-backed Scenarios start with version = 1 + @pytest.mark.jdbc def test_init(self, test_mp, scen_empty): # Empty scenario has version == 0 assert scen_empty.version == 0 @@ -76,16 +78,26 @@ class Scenario(ixmp.Scenario): ): Scenario(test_mp, model="foo", scenario="bar", cache=False) - def test_default_version(self, mp): + def test_default_version(self, mp: ixmp.Platform) -> None: scen = ixmp.Scenario(mp, **models["dantzig"]) - assert scen.version == 2 + scenario_df = mp.scenario_list( + model=models["dantzig"]["model"], scen=models["dantzig"]["scenario"] + ) + assert len(scenario_df["version"]) == 1 + assert scen.version == scenario_df["version"].item() + # NOTE IXMP4(Backend) doesn't raise the same error/message as expected here + @pytest.mark.jdbc def test_from_url(self, mp, caplog): url = f"ixmp://{mp.name}/Douglas Adams/Hitchhiker" # Default version is loaded scen, mp = ixmp.Scenario.from_url(url) - assert scen.version == 1 + scenario_df = mp.scenario_list( + model=models["h2g2"]["model"], scen=models["h2g2"]["scenario"] + ) + assert len(scenario_df["version"]) == 1 + assert scen.version == scenario_df["version"].item() # Giving an invalid version with errors='raise' raises an exception expected = ( @@ -118,6 +130,8 @@ def test_clone(self, mp): obs = scen2.set("h") npt.assert_array_equal(obs, ["test"]) + # FIXME IXMP4Backend needs to handle change_scalar correctly + @pytest.mark.jdbc def test_clone_edit(self, scen): scen2 = scen.clone(keep_solution=False) scen2.check_out() @@ -131,6 +145,8 @@ def test_clone_edit(self, scen): assert scen2.scalar("f") == {"unit": "USD/km", "value": 95} # Initialize items + # NOTE IXMP4Backend doesn't handle commits yet + @pytest.mark.jdbc def test_init_set(self, scen): """Test ixmp.Scenario.init_set().""" @@ -149,6 +165,9 @@ def test_init_set(self, scen): with pytest.raises(ValueError, match="'foo' already exists"): scen.init_set("foo") + # FIXME IXMP4Backend recognized 'foo' as having no idx_sets, likely due to the + # assumption of all names of all items being unique. We should drop that, it seems. + @pytest.mark.jdbc def test_init_par(self, scen) -> None: scen = scen.clone(keep_solution=False) scen.check_out() @@ -170,6 +189,8 @@ def test_init_scalar(self, scen): scen2.commit("adding a scalar 'g'") # Existence checks + # TODO IXMP4Backend doesn't handle scalars correctly yet + @pytest.mark.jdbc def test_has_par(self, scen): assert scen.has_par("f") assert not scen.has_par("m") @@ -182,6 +203,8 @@ def test_has_var(self, scen): assert scen.has_var("x") assert not scen.has_var("y") + # TODO IXMP4Backend doesn't handle scalars correctly yet + @pytest.mark.jdbc def test_scalar(self, scen): assert scen.scalar("f") == {"unit": "USD/km", "value": 90} @@ -226,6 +249,8 @@ def test_add_par(self, scen, args, kwargs): scen.check_out() scen.add_par(*args, **kwargs) + # TODO IXMP4Backend should support this, I think + @pytest.mark.jdbc def test_add_par2(self, scen): scen = scen.clone(keep_solution=False) scen.check_out() @@ -255,6 +280,8 @@ def test_idx(self, scen): assert scen.idx_sets("d") == ["i", "j"] assert scen.idx_names("d") == ["i", "j"] + # FIXME IXMP4-backed par doesn't have 0 as a key; avoid this hardcoding + @pytest.mark.jdbc def test_par(self, scen): # Parameter data can be retrieved with filters df = scen.par("d", filters={"i": ["seattle"]}) @@ -264,6 +291,8 @@ def test_par(self, scen): with pytest.warns(DeprecationWarning, match="ignored kwargs"): scen.par("d", i=["seattle"]) + # FIXME IXMP4Backend is missing an item, likely the scalar f again + @pytest.mark.jdbc def test_items0(self, scen): # Without filters iterator = scen.items() @@ -294,6 +323,9 @@ def test_items0(self, scen): assert i == 1 + # FIXME For test case 3, IXMP4Backend somehow also lists 'foo' (likely defined in + # the fixture); in test case 2 it's missing f (the scalar) + @pytest.mark.jdbc @pytest.mark.parametrize( "item_type, indexed_by, exp", ( @@ -333,6 +365,8 @@ def test_var(self, scen): # Marginals npt.assert_array_almost_equal(df["mrg"], [0, 0, 0.036]) + # TODO IXMP4Backend is not handling _cache correctly + @pytest.mark.jdbc def test_load_scenario_data(self, mp): """load_scenario_data() caches all data.""" scen = ixmp.Scenario(mp, **models["dantzig"]) @@ -366,6 +400,9 @@ def test_load_scenario_data_clear_cache(self, monkeypatch, mp): scen.load_scenario_data() # I/O + # TODO For IXMP4Backend, this somehow triggers a GAMS-related error, while this test + # should never touch GAMS + @pytest.mark.jdbc def test_excel_io(self, scen, scen_empty, tmp_path, caplog): tmp_path /= "output.xlsx" @@ -448,6 +485,9 @@ def test_excel_io(self, scen, scen_empty, tmp_path, caplog): # Succeeds with add_units=True s.read_excel(tmp_path, add_units=True, init_items=True) + # NOTE IXMP4-backed Scenarios should not call remove_solution() if they don't have + # one + @pytest.mark.jdbc def test_solve(self, tmp_path, scen): from subprocess import run @@ -483,6 +523,8 @@ def test_solve(self, tmp_path, scen): assert "'notapackage'.'(not installed)'" in result.stdout.decode() # Combined tests + # NOTE Not yet implemented on IXMP4Backend + @pytest.mark.jdbc def test_meta(self, mp, test_dict): scen = ixmp.Scenario(mp, **models["dantzig"], version=1) for k, v in test_dict.items(): @@ -513,6 +555,8 @@ def test_meta(self, mp, test_dict): with pytest.raises(ValueError, match="Cannot use value"): scen.set_meta("test_string", complex(1, 1)) + # NOTE Not yet implemented on IXMP4Backend + @pytest.mark.jdbc def test_meta_bulk(self, mp, test_dict): scen = ixmp.Scenario(mp, **models["dantzig"], version=1) scen.set_meta(test_dict) @@ -560,6 +604,9 @@ def test_gh_210(scen_empty): assert all(foo_data.columns == columns) +# NOTE IXMP4(Backend) raises an OptimizationDataValidationError if 'bar' is missing from +# index set 'i' instead of a ValueError +@pytest.mark.jdbc def test_set(scen_empty) -> None: """Test ixmp.Scenario.add_set(), .set(), and .remove_set().""" scen = scen_empty @@ -690,6 +737,9 @@ def test_filter_str(scen_empty): assert_frame_equal(exp[["s", "value"]], obs[["s", "value"]]) +# FIXME Calling scen.solve(change_distance, ...) triggers a bug in the GAMS code +# IXMP4Backend might need to avoid similar calls (which properties exactly?) +@pytest.mark.jdbc def test_solve_callback(test_mp, request): """Test the callback argument to Scenario.solve(). diff --git a/ixmp/tests/core/test_timeseries.py b/ixmp/tests/core/test_timeseries.py index f7af7263b..3dc12fc97 100644 --- a/ixmp/tests/core/test_timeseries.py +++ b/ixmp/tests/core/test_timeseries.py @@ -132,6 +132,8 @@ def test_default(self, mp, ts): # Original TimeSeries is no longer default assert not ts.is_default() + # NOTE IXMP4Backend doesn't handle commits yet, so run_id is 1 immediately + @pytest.mark.jdbc def test_run_id(self, ts): # New, un-committed TimeSeries has run_id of -1 assert ts.run_id() == -1 @@ -140,6 +142,8 @@ def test_run_id(self, ts): ts.commit("") assert ts.run_id() > 0 and isinstance(ts.run_id(), int) + # NOTE Not yet implemented on IXMP4Backend + @pytest.mark.jdbc def test_last_update(self, ts): # New, un-committed TimeSeries has no last update date assert ts.last_update() is None @@ -174,6 +178,9 @@ def test_discard_changes(self, ts): assert 0 == len(ts.timeseries()) + # FIXME IXMP4Backend seems to handle timeseries() incorrectly; the last call returns + # 0 rows + @pytest.mark.jdbc @pytest.mark.parametrize("format", ["long", "wide"]) def test_get(self, ts, format): data = DATA[0] if format == "long" else wide(DATA[0]) @@ -211,6 +218,9 @@ def test_get_year(self, ts, year_arg): # year filters with integer values are handled correctly (iiasa/ixmp#440) ts.timeseries(year=year_arg) + # FIXME IXMP4Backend seems to handle timeseries() incorrectly; the last call returns + # 0 rows + @pytest.mark.jdbc @pytest.mark.parametrize("format", ["long", "wide"]) def test_edit(self, mp, ts, format): """Tests that data can be overwritten.""" @@ -248,6 +258,9 @@ def test_edit(self, mp, ts, format): exp = wide(exp) assert_frame_equal(exp, ts.timeseries(**args)) + # NOTE IXMP4Backend requires version-specifier or setting a default for the + # corresponding Run since Run doesn't store `version` + @pytest.mark.jdbc @pytest.mark.parametrize("cls", [TimeSeries, Scenario]) def test_edit_with_region_synonyms(self, mp, ts, cls): info = dict(model=ts.model, scenario=ts.scenario) @@ -268,6 +281,8 @@ def test_edit_with_region_synonyms(self, mp, ts, cls): assert_frame_equal(expected(DATA[2050], ts), ts.timeseries()) + # NOTE Not yet implemented on IXMP4Backend + @pytest.mark.jdbc # TODO parametrize format as wide/long @pytest.mark.parametrize( "commit", @@ -315,6 +330,8 @@ def test_remove(self, mp, ts, commit): # Result is empty assert ts.timeseries().empty + # NOTE Not yet implemented on IXMP4Backend + @pytest.mark.jdbc def test_transact_discard(self, caplog, mp, ts): caplog.set_level(logging.INFO, "ixmp.util") @@ -345,6 +362,8 @@ def test_transact_discard(self, caplog, mp, ts): # Geodata + # NOTE Not yet implemented on IXMP4Backend + @pytest.mark.jdbc def test_add_geodata(self, ts): # Empty TimeSeries includes no geodata assert_frame_equal(DATA["geo"].loc[[False, False, False]], ts.get_geodata()) @@ -357,6 +376,8 @@ def test_add_geodata(self, ts): obs = ts.get_geodata().sort_values("year").reset_index(drop=True) assert_frame_equal(DATA["geo"], obs) + # NOTE Not yet implemented on IXMP4Backend + @pytest.mark.jdbc @pytest.mark.parametrize( "rows", [(1,), (1, 2), (0, 1, 2)], ids=["single", "multiple", "all"] ) @@ -420,10 +441,10 @@ def test_add_timeseries_with_extra_col(self, caplog, ts, format): ts.add_timeseries(data) # TODO: add check that warning message is displayed ts.commit("") - assert [ - "Dropped extra column(s) ['climate_model'] from data" - ] == caplog.messages + assert "Dropped extra column(s) ['climate_model'] from data" in caplog.messages + # NOTE Not yet implemented on IXMP4Backend + @pytest.mark.jdbc def test_new_timeseries_as_iamc(self, test_mp): # TODO rescue use of subannual= here @@ -458,6 +479,8 @@ def test_new_timeseries_error(self, test_mp): # column `unit` is missing pytest.raises(ValueError, scen.add_timeseries, df) + # NOTE Not yet implemented on IXMP4Backend + @pytest.mark.jdbc def test_new_subannual_timeseries_as_iamc(self, mp): mp.add_timeslice("Summer", "Season", 1.0 / 4) scen = TimeSeries(mp, *models["h2g2"], version="new", annotation="fo") @@ -495,11 +518,15 @@ def test_new_subannual_timeseries_as_iamc(self, mp): # setting False raises an error because subannual data exists pytest.raises(ValueError, scen.timeseries, subannual=False) + # NOTE Not yet implemented on IXMP4Backend + @pytest.mark.jdbc def test_fetch_empty_geodata(self, mp): scen = TimeSeries(mp, *models["h2g2"], version="new", annotation="fo") empty = scen.get_geodata() assert_geodata(empty, DATA["geo"].loc[[False, False, False]]) + # NOTE Not yet implemented on IXMP4Backend + @pytest.mark.jdbc def test_remove_multiple_geodata(self, mp): scen = TimeSeries(mp, *models["h2g2"], version="new", annotation="fo") scen.add_geodata(DATA["geo"]) diff --git a/ixmp/tests/report/test_operator.py b/ixmp/tests/report/test_operator.py index 04ddfc95f..f48b7328b 100644 --- a/ixmp/tests/report/test_operator.py +++ b/ixmp/tests/report/test_operator.py @@ -45,6 +45,8 @@ def test_from_url(test_mp, request) -> None: assert ts.url == result.url +# FIXME Not sure why introducing the Computer fails for IXMP4Backend +@pytest.mark.jdbc def test_get_remove_ts(caplog, test_mp, request) -> None: ts = make_dantzig(test_mp, request=request) @@ -107,6 +109,8 @@ def test_map_as_qty() -> None: assert_qty_equal(exp, result) +# FIXME IXMP4Backend seems to return d with unexpected ordering +@pytest.mark.jdbc def test_update_scenario(caplog, test_mp, request) -> None: scen = make_dantzig(test_mp, request=request) scen.check_out() @@ -161,6 +165,8 @@ def test_update_scenario(caplog, test_mp, request) -> None: assert_frame_equal(scen.par("d"), data) +# FIXME Nots ure why this fails on IXMP4Backend +@pytest.mark.jdbc def test_store_ts(request, caplog, test_mp) -> None: # Computer and target scenario c = Computer() diff --git a/ixmp/tests/report/test_reporter.py b/ixmp/tests/report/test_reporter.py index 22e9cdf35..dd49dc101 100644 --- a/ixmp/tests/report/test_reporter.py +++ b/ixmp/tests/report/test_reporter.py @@ -124,7 +124,7 @@ def test_platform_units(test_mp, caplog, ureg) -> None: scen.add_par("x", x) with assert_logs( - caplog, "x: mixed units ['kg', 'USD/pkm'] discarded", at_level=logging.INFO + caplog, ["x: mixed units", "kg", "USD/pkm", "discarded"], at_level=logging.INFO ): rep.get(x_key) diff --git a/ixmp/tests/test_cli.py b/ixmp/tests/test_cli.py index 982414319..5a17c6746 100644 --- a/ixmp/tests/test_cli.py +++ b/ixmp/tests/test_cli.py @@ -27,6 +27,8 @@ def test_versiontype(): vt.convert("xx", None, None) +# FIXME This should work for IXMP4Backend, I think +@pytest.mark.jdbc def test_main(ixmp_cli, test_mp, tmp_path): # Name of a temporary file that doesn't exist tmp_path /= "temp.properties" @@ -96,7 +98,8 @@ def test_config(ixmp_cli): def test_list(ixmp_cli, test_mp): - cmd = ["list", "--match", "foo"] + # NOTE Some other test may leak scenarios onto test_mp + cmd = ["list", "--match", "no-scenario-named-foo"] # 'list' without specifying a platform/scenario is a UsageError result = ixmp_cli.invoke(cmd) @@ -113,7 +116,7 @@ def test_list(ixmp_cli, test_mp): 0 (model, scenario) combination(s) 0 total scenarios """ - ) + ), result.output def test_platform(ixmp_cli, tmp_path): @@ -121,7 +124,7 @@ def test_platform(ixmp_cli, tmp_path): def call(*args, exit_0=True): result = ixmp_cli.invoke(["platform"] + list(map(str, args))) - assert not exit_0 or result.exit_code == 0 + assert not exit_0 or result.exit_code == 0, result.output return result # The default platform is 'local' @@ -217,6 +220,8 @@ def call(*args, exit_0=True): call("copy", f"p1-{test_specific_name}", f"p3-{test_specific_name}") +# TODO Version 1 for IXMP4Backend returns an empty DF for scen.timeseries() +@pytest.mark.jdbc def test_import_ts(ixmp_cli, test_mp, test_data_path): # Ensure the 'canning problem'/'standard' TimeSeries exists populate_test_platform(test_mp) @@ -326,6 +331,10 @@ def test_excel_io(ixmp_cli, test_mp, tmp_path): assert result.exit_code == 0, result.output +# FIXME IXMP4Backend requires a version to be specified when loading a Scenario +# or a default to be set for the corresponding Run before +# since Run doesn't store `version` +@pytest.mark.jdbc def test_excel_io_filters(ixmp_cli, test_mp, tmp_path): populate_test_platform(test_mp) tmp_path /= "dantzig.xlsx" @@ -381,6 +390,8 @@ def test_show_versions(ixmp_cli): assert result.exit_code == 0, result.output +# FIXME This is again the parameter error for IXMP4Backend and the DantzigModel +@pytest.mark.jdbc def test_solve(ixmp_cli, test_mp): populate_test_platform(test_mp) cmd = [ diff --git a/ixmp/tests/test_integration.py b/ixmp/tests/test_integration.py index caf5d6b88..014a8c26b 100644 --- a/ixmp/tests/test_integration.py +++ b/ixmp/tests/test_integration.py @@ -12,6 +12,8 @@ TS_DF_CLEARED.loc[0, 2005] = np.nan +# FIXME IXMP4Backend seems to return columns with no/wrong names for scen.timeseries() +@pytest.mark.jdbc def test_run_clone(caplog, test_mp, request): caplog.set_level(logging.WARNING) @@ -66,6 +68,8 @@ def test_run_clone(caplog, test_mp, request): ) +# FIXME Fix IXMP4Backend return value for s.var()["lvl"] so that np.isnan() accepts it +@pytest.mark.jdbc def test_run_remove_solution(test_mp, request): # create a new instance of the transport problem and solve it mp = test_mp diff --git a/ixmp/tests/test_model.py b/ixmp/tests/test_model.py index 83259563d..73c3106e2 100644 --- a/ixmp/tests/test_model.py +++ b/ixmp/tests/test_model.py @@ -23,6 +23,9 @@ class M1(Model): M1() +# FIXME IXMP4Backend doesn't handle scenario.change_scalar() correctly: +# We can't just create() it, it might already exist, so then we need update() +@pytest.mark.jdbc def test_model_initialize(test_mp, caplog, request): # Model.initialize runs on an empty Scenario s = make_dantzig(test_mp, request=request) @@ -111,6 +114,9 @@ class TestGAMSModel: def dantzig(self, test_mp, request): yield make_dantzig(test_mp, request=request) + # FIXME The name_ seems to be handled correctly, but IXMP4Backend struggles with + # make_dantzig().clone().solve() somehow. + @pytest.mark.jdbc @pytest.mark.parametrize("char", r'<>"/\|?*') def test_filename_invalid_char(self, dantzig, char): """Model can be solved with invalid character names.""" @@ -122,6 +128,8 @@ def test_filename_invalid_char(self, dantzig, char): # the GAMSModel.model_name attribute, and in turn the GDX file names used. s.solve(name_=name, quiet=True) + # FIXME IXMP4Backend should support this + @pytest.mark.jdbc @pytest.mark.parametrize( "kwargs", [ diff --git a/ixmp/tests/test_util.py b/ixmp/tests/test_util.py index 05f12e37e..29c608f55 100644 --- a/ixmp/tests/test_util.py +++ b/ixmp/tests/test_util.py @@ -72,6 +72,8 @@ def test_diff_identical(test_mp, request): assert exp_name == name and len(df) == N +# FIXME I don't see why IXMP4Backend shouldn't support this, but it's failing. +@pytest.mark.jdbc def test_diff_data(test_mp, request): """diff() when Scenarios contain the same items, but different data.""" scen_a = make_dantzig(test_mp, request=request) @@ -156,6 +158,9 @@ def test_diff_items(test_mp, request): pass # No check of the contents +# TODO IXMP4Backend doesn't handle retrieval of scalars correctly yet; +# but look here for a test case! +@pytest.mark.jdbc def test_discard_on_error(caplog, test_mp, request): caplog.set_level(logging.INFO, "ixmp.util") @@ -293,6 +298,8 @@ def test_format_scenario_list(test_mp_f): ) == util.format_scenario_list(mp, as_url=True) +# IXMP4Backend doesn't have proper commits yet, so these never raise RuntimeErrors +@pytest.mark.jdbc def test_maybe_commit(caplog, test_mp): s = Scenario(test_mp, "maybe_commit", "maybe_commit", version="new") diff --git a/ixmp/util/ixmp4.py b/ixmp/util/ixmp4.py new file mode 100644 index 000000000..80d2d7399 --- /dev/null +++ b/ixmp/util/ixmp4.py @@ -0,0 +1,58 @@ +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Literal, Optional, Union + +import pandas as pd + +# TODO Import this from typing when dropping Python 3.11 +from typing_extensions import TypedDict + +if TYPE_CHECKING: + from ixmp.core.scenario import Scenario + + +# These are based on existing calls within ixmp +class WriteFiltersKwargs(TypedDict, total=False): + scenario: "Scenario | list[str]" + model: list[str] + variable: list[str] + unit: list[str] + region: list[str] + default: bool + export_all_runs: bool + + +@dataclass +class ContainerData: + name: str + kind: Literal["IndexSet", "Table", "Scalar", "Parameter", "Equation", "Variable"] + records: Optional[ + Union[ + float, + list[int], + list[float], + list[str], + dict[str, Union[list[float], list[int], list[str]]], + pd.DataFrame, + ] + ] + domain: Optional[list[str]] = None + docs: Optional[str] = None + + +class WriteKwargs(TypedDict, total=False): + filters: WriteFiltersKwargs + record_version_packages: list[str] + container_data: list[ContainerData] + + +class ReadKwargs(TypedDict, total=False): + filters: WriteFiltersKwargs + firstyear: Optional[Any] + lastyear: Optional[Any] + add_units: bool + init_items: bool + commit_steps: bool + check_solution: bool + comment: str + equ_list: list[str] + var_list: list[str] diff --git a/pyproject.toml b/pyproject.toml index 13989297e..f538a48ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,10 +57,16 @@ docs = [ "sphinx_rtd_theme", "sphinxcontrib-bibtex", ] +# Change ixmp4 to new release after https://github.com/iiasa/ixmp4/pull/164 is merged +ixmp4 = [ + "ixmp4 @ git+https://github.com/iiasa/ixmp4@enh/remove-linked-items-when-removing-indexset-items", + "gamsapi[core,transfer] >= 45.7.0", +] report = ["genno[compat,graphviz]"] tutorial = ["jupyter"] tests = [ "ixmp[report,tutorial]", + "ixmp[ixmp4]; python_version > '3.9'", "memory_profiler", "nbclient >= 0.5", "pytest >= 5", @@ -79,7 +85,10 @@ exclude_also = [ # Imports only used by type checkers "if TYPE_CHECKING:", ] -omit = ["ixmp/util/sphinx_linkcode_github.py"] +omit = [ + "ixmp/util/sphinx_linkcode_github.py", + ".venv/*", +] [tool.mypy] files = [ @@ -109,6 +118,8 @@ addopts = """ markers = [ "rixmp: test of the ixmp R interface.", "performance: ixmp performance test.", + "jdbc: tests exclusive to JDBCBackend.", + "ixmp4: tests exclusive to IXMP4Backend.", ] tmp_path_retention_policy = "none"