Implement ixmp4 shim

.github/workflows/pytest.yaml

@@ -30,7 +30,7 @@ jobs:
         - "3.12"
         - "3.13"  # Latest supported by ixmp
         gams-version:
-        - "43.4.1"  # First version including a macOS arm64 distribution
+        - "45.7.0" # Oldest version of gamsapi allowed by pyproject.toml
 
         exclude:
         # Specific version combinations that are invalid / not to be used

.pre-commit-config.yaml

@@ -7,6 +7,7 @@ repos:
     additional_dependencies:
     - genno
     - GitPython
+    - "ixmp4 @ git+https://github.com/iiasa/ixmp4@enh/remove-linked-items-when-removing-indexset-items"
     - nbclient
     - pandas-stubs
     - pytest

RELEASE_NOTES.rst

@@ -1,8 +1,15 @@
-.. Next release
-.. ============
+Next release
+============
+
+All changes
+-----------
+
+- Add :class:`.IXMP4Backend` as an alternative to :class:`.JDBCBackend` (:pull:`552`).
+  Please note:
+
+  - This requires ixmp4, which is only compatible with Python 3.10 and above.
+  - :class:`.IXMP4Backend` is still missing several features and documentation, which will be added in subsequent PRs.
 
-.. All changes
-.. -----------
 
 .. _v3.10.0:
 

doc/api-backend.rst

@@ -61,6 +61,15 @@ Provided backends
 
 .. autofunction:: ixmp.backend.jdbc.start_jvm
 
+.. currentmodule:: ixmp.backend.ixmp4
+
+.. warning::
+   `ixmp4 <https://github.com/iiasa/ixmp4/>`_ only supports Python 3.10 and above. 
+   Please ensure you are using a sufficiently recent Python version if you want to use IXMP4Backend.
+   If you are restricted to Python 3.9 and below, please use JDBCBackend instead.
+
+.. autoclass:: ixmp.backend.ixmp4.IXMP4Backend
+
 
 .. currentmodule:: ixmp.backend
 

ixmp/__init__.py

@@ -48,6 +48,11 @@
 # Register Backends provided by ixmp
 BACKENDS["jdbc"] = JDBCBackend
 
+if sys.version_info >= (3, 10):
+    from ixmp.backend.ixmp4 import IXMP4Backend
+
+    BACKENDS["ixmp4"] = IXMP4Backend
+
 # Register Models provided by ixmp
 MODELS.update(
     {

ixmp/_config.py

@@ -68,6 +68,14 @@ def _platform_default():
             "driver": "hsqldb",
             "path": next(_iter_config_paths())[1].joinpath("localdb", "default"),
         },
+        "ixmp4-local": {
+            "class": "ixmp4",
+            "dsn": (
+                "sqlite:///"
+                f"{Path.home().joinpath('.local', 'share', 'ixmp4', 'databases')}"
+                "/ixmp4-local.sqlite3"
+            ),
+        },
     }
 
 

ixmp/backend/base.py

@@ -9,11 +9,15 @@
 
 import pandas as pd
 
+# TODO Import this from typing when dropping Python 3.11
+from typing_extensions import Unpack
+
 from ixmp.backend import ItemType
 from ixmp.backend.io import s_read_excel, s_write_excel, ts_read_file
 from ixmp.core.platform import Platform
 from ixmp.core.scenario import Scenario
 from ixmp.core.timeseries import TimeSeries
+from ixmp.util.ixmp4 import ReadKwargs, WriteFiltersKwargs, WriteKwargs
 
 
 class Backend(ABC):
@@ -204,12 +208,13 @@ def set_node(
         """
 
     @abstractmethod
-    def get_nodes(self) -> Iterable[tuple[str, Optional[str], str, str]]:
+    def get_nodes(self) -> Iterable[tuple[str, Optional[str], Optional[str], str]]:
         """Iterate over all nodes stored on the Platform.
 
         Yields
         -------
         tuple
+
             The members of each tuple are:
 
             ========= =========== ===
@@ -378,7 +383,9 @@ def get_units(self) -> list[str]:
         set_unit
         """
 
-    def read_file(self, path: PathLike, item_type: ItemType, **kwargs) -> None:
+    def read_file(
+        self, path: PathLike, item_type: ItemType, **kwargs: Unpack[ReadKwargs]
+    ) -> None:
         """OPTIONAL: Read Platform, TimeSeries, or Scenario data from file.
 
         A backend **may** implement read_file for one or more combinations of the `path`
@@ -418,16 +425,21 @@ def read_file(self, path: PathLike, item_type: ItemType, **kwargs) -> None:
         --------
         write_file
         """
-        s, filters = self._handle_rw_filters(kwargs.pop("filters", {}))
+        filters = kwargs["filters"] if "filters" in kwargs else WriteFiltersKwargs()
+        s, _ = self._handle_rw_filters(filters=filters)
+        _kwargs = {k: v for (k, v) in kwargs.items() if k != "filters"}
+
         path = Path(path)
         if path.suffix in (".csv", ".xlsx") and item_type is ItemType.TS and s:
-            ts_read_file(s, path, **kwargs)
+            ts_read_file(s, path, **_kwargs)
         elif path.suffix == ".xlsx" and item_type is ItemType.MODEL and s:
-            s_read_excel(self, s, path, **kwargs)
+            s_read_excel(self, s, path, **_kwargs)
         else:
             raise NotImplementedError
 
-    def write_file(self, path: PathLike, item_type: ItemType, **kwargs) -> None:
+    def write_file(
+        self, path: PathLike, item_type: ItemType, **kwargs: Unpack[WriteKwargs]
+    ) -> None:
         """OPTIONAL: Write Platform, TimeSeries, or Scenario data to file.
 
         A backend **may** implement write_file for one or more combinations of the
@@ -460,12 +472,19 @@ def write_file(self, path: PathLike, item_type: ItemType, **kwargs) -> None:
         """
         # Use the "scenario" filter to retrieve the Scenario `s` to be written; reappend
         # any other filters
-        s, kwargs["filters"] = self._handle_rw_filters(kwargs.pop("filters", {}))
+        filters = kwargs["filters"] if "filters" in kwargs else WriteFiltersKwargs()
+        s, kwargs["filters"] = self._handle_rw_filters(filters=filters)
 
         xlsx_types = (ItemType.SET | ItemType.PAR, ItemType.MODEL)
         path = Path(path)
         if path.suffix == ".xlsx" and item_type in xlsx_types and s:
-            s_write_excel(self, s, path, item_type, **kwargs)
+            s_write_excel(
+                be=self,
+                s=s,
+                path=path,
+                item_type=item_type,
+                filters=kwargs["filters"],
+            )
         else:
             # All other combinations of arguments
             raise NotImplementedError
@@ -581,18 +600,20 @@ def preload(self, ts: TimeSeries) -> None:
         """OPTIONAL: Load `ts` data into memory."""
 
     @staticmethod
-    def _handle_rw_filters(filters: dict) -> tuple[Optional[TimeSeries], dict]:
+    def _handle_rw_filters(
+        filters: WriteFiltersKwargs,
+    ) -> tuple[Optional[TimeSeries], WriteFiltersKwargs]:
         """Helper for :meth:`read_file` and :meth:`write_file`.
 
         The `filters` argument is unpacked if the 'scenarios' key is a single
         :class:`.TimeSeries` object. A 2-tuple is returned of the object (or
         :obj:`None`) and the remaining filters.
         """
-        ts = None
+        ts: Optional[TimeSeries] = None
         filters = copy(filters)
         try:
             if isinstance(filters["scenario"], TimeSeries):
-                ts = filters.pop("scenario")
+                ts = filters["scenario"]
         except KeyError:
             pass  # Don't modify filters at all
 
@@ -820,7 +841,7 @@ def has_solution(self, s: Scenario) -> bool:
         """
 
     @abstractmethod
-    def list_items(self, s: Scenario, type: str) -> list[str]:
+    def list_items(self, s: Scenario, type: Literal["set", "par", "equ"]) -> list[str]:
         """Return a list of names of items of `type`.
 
         Parameters
@@ -832,7 +853,7 @@ def list_items(self, s: Scenario, type: str) -> list[str]:
     def init_item(
         self,
         s: Scenario,
-        type: str,
+        type: Literal["set", "par", "equ", "var"],
         name: str,
         idx_sets: Sequence[str],
         idx_names: Optional[Sequence[str]],
@@ -860,7 +881,9 @@ def init_item(
         """
 
     @abstractmethod
-    def delete_item(self, s: Scenario, type: str, name: str) -> None:
+    def delete_item(
+        self, s: Scenario, type: Literal["set", "par", "equ"], name: str
+    ) -> None:
         """Remove an item `name` of `type`.
 
         Parameters
@@ -871,7 +894,9 @@ def delete_item(self, s: Scenario, type: str, name: str) -> None:
         """
 
     @abstractmethod
-    def item_index(self, s: Scenario, name: str, sets_or_names: str) -> list[str]:
+    def item_index(
+        self, s: Scenario, name: str, sets_or_names: Literal["sets", "names"]
+    ) -> list[str]:
         """Return the index sets or names of item `name`.
 
         Parameters
@@ -932,7 +957,7 @@ def item_get_elements(
     def item_set_elements(
         self,
         s: Scenario,
-        type: str,
+        type: Literal["par", "set"],
         name: str,
         elements: Iterable[tuple[Any, Optional[float], Optional[str], Optional[str]]],
     ) -> None:
@@ -974,7 +999,9 @@ def item_set_elements(
         """
 
     @abstractmethod
-    def item_delete_elements(self, s: Scenario, type: str, name: str, keys) -> None:
+    def item_delete_elements(
+        self, s: Scenario, type: Literal["par", "set"], name: str, keys
+    ) -> None:
         """Remove elements of item `name`.
 
         Parameters