Test against ixmp4

.github/workflows/pytest.yaml

@@ -16,11 +16,27 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+defaults:
+  run:
+    shell: bash
+
 env:
-  GAMS_VERSION: 43.4.1   # First version including a macOS arm64 distribution
+  GAMS_VERSION: 45.7.0 # Oldest version of gamsapi allowed by pyproject.toml
   depth: 100  # Must be large enough to include the most recent release
   label: "safe to test"  # Label that must be applied to run on PRs from forks
   python-version: "3.13"  # For non-matrixed jobs
+  # Install:
+  # - ixmp: from its `main` branch.
+  # - all other dependencies from PyPI.
+  #
+  # To test against other code, such as other branches for open PRs, temporarily
+  # add lines as appropriate. These lines SHOULD NOT be merged to `main`.
+  #
+  # - ixmp4: pending https://github.com/iiasa/ixmp4/pull/171.
+  #   TODO Remove once the PR is merged and a new version is released.
+  upstream-versions: |
+    "ixmp4 @ git+https://github.com/iiasa/ixmp4@enh/remove-linked-items-when-removing-indexset-items; python_version > '3.9'" \
+    "ixmp @ git+https://github.com/iiasa/ixmp.git@main" \
 
 jobs:
   check:
@@ -113,12 +129,7 @@ jobs:
         license: ${{ secrets.GAMS_LICENSE }}
 
     - name: Install the package and dependencies
-      # By default, the below installs ixmp from the main branch. To run against
-      # other code, e.g. other branches for open PRs), temporarily edit as
-      # appropriate. DO NOT merge such changes to `main`.
-      run: |
-        uv pip install --upgrade "ixmp @ git+https://github.com/iiasa/ixmp.git@main"
-        uv pip install .[tests]
+      run: uv pip install --upgrade ${{ env.upstream-versions }} .[tests]
 
     - name: Run test suite using pytest
       env:
@@ -130,7 +141,6 @@ jobs:
           --color=yes --durations=20 -rA --verbose \
           --cov-report=xml \
           --numprocesses=auto --dist=loadgroup
-      shell: bash
 
     - name: Upload test coverage to Codecov.io
       uses: codecov/codecov-action@v5
@@ -169,7 +179,6 @@ jobs:
     - name: Set RETICULATE_PYTHON
       # Retrieve the Python executable set up above
       run: echo "RETICULATE_PYTHON=$(uv python find)" >> "$GITHUB_ENV"
-      shell: bash
 
     - uses: r-lib/actions/setup-r@v2
       id: setup-r
@@ -193,12 +202,7 @@ jobs:
         license: ${{ secrets.GAMS_LICENSE }}
 
     - name: Install the package and dependencies
-      # By default, the below installs ixmp from the main branch. To run against
-      # other code, e.g. other branches for open PRs), temporarily edit as
-      # appropriate. DO NOT merge such changes to `main`.
-      run: |
-        uv pip install --upgrade "ixmp @ git+https://github.com/iiasa/ixmp.git@main"
-        uv pip install .[tests]
+      run: uv pip install --upgrade ${{ env.upstream-versions }} .[tests]
 
     - name: Install R dependencies and tutorial requirements
       run: |
@@ -213,7 +217,6 @@ jobs:
           --color=yes --durations=20 -rA --verbose \
           --cov-report=xml \
           --numprocesses=auto --dist=loadgroup
-      shell: bash
 
     - name: Upload test coverage to Codecov.io
       uses: codecov/codecov-action@v5

RELEASE_NOTES.rst

@@ -28,6 +28,8 @@ Users **should**:
 All changes
 -----------
 
+- Some MESSAGEix :doc:`tutorials <tutorials>` are runnable with the :class:`.IXMP4Backend` introduced in :mod:`ixmp` version 3.11 (:pull:`894`).
+  See `Support roadmap for ixmp4 <https://github.com/iiasa/message_ix/discussions/939>`__ for details.
 - Adjust use of :ref:`type_tec <mapping-sets>` in :ref:`equation_emission_equivalence` (:pull:`930`, :issue:`929`, :pull:`935`).
 
   This change reduces the size of the ``EMISS`` variable,

ci/rtd-requirements.txt

@@ -1 +1,11 @@
+# Requirements for documentation build on RTD
+#
+# These *should* generally align with env.upstream-versions
+# in .github/workflows/pytest.yaml
+
+# Temporary, pending https://github.com/iiasa/ixmp4/pull/171.
+# TODO Remove once the PR is merged and a new version is released.
+git+https://github.com/iiasa/ixmp4@enh/remove-linked-items-when-removing-indexset-items
+
+gamsapi [core,transfer] >= 45.7.0
 git+https://github.com/iiasa/ixmp.git@main#egg=ixmp

message_ix/core.py

@@ -3,15 +3,20 @@
 from collections.abc import Iterable, Mapping, Sequence
 from functools import lru_cache, partial
 from itertools import chain, product, zip_longest
-from typing import Optional, Union
+from typing import Optional, TypeVar, Union
 from warnings import warn
 
 import ixmp
 import numpy as np
 import pandas as pd
 from ixmp.backend import ItemType
+from ixmp.backend.jdbc import JDBCBackend
 from ixmp.util import as_str_list, maybe_check_out, maybe_commit
 
+from message_ix.util.ixmp4 import on_ixmp4backend
+
+# from message_ix.util.scenario_data import PARAMETERS
+
 log = logging.getLogger(__name__)
 
 # Also print warnings to stderr
@@ -62,7 +67,7 @@ def __init__(
     # Utility methods used by .equ(), .par(), .set(), and .var()
 
     @lru_cache()
-    def _year_idx(self, name):
+    def _year_idx(self, name: str):
         """Return a sequence of (idx_set, idx_name) for 'year'-indexed dims.
 
         Since item dimensionality does not change, the the return value is
@@ -78,7 +83,10 @@ def _year_idx(self, name):
             )
         )
 
-    def _year_as_int(self, name, df):
+    data_type = TypeVar("data_type", pd.Series, pd.DataFrame, dict)
+
+    # NOTE super().equ() etc hint that pd objects return, but they may also return dict
+    def _year_as_int(self, name: str, data: data_type) -> data_type:
         """Convert 'year'-indexed columns of *df* to :obj:`int` dtypes.
 
         :meth:`_year_idx` is used to retrieve a sequence of (idx_set, idx_name)
@@ -90,12 +98,18 @@ def _year_as_int(self, name, df):
         year_idx = self._year_idx(name)
 
         if len(year_idx):
-            return df.astype({col_name: "int" for _, col_name in year_idx})
+            assert isinstance(data, pd.DataFrame)
+            # NOTE With the IXMP4Backend, we call scenario.par() for parameters that
+            # might be empty, which fails df.astype() below.
+            if data.empty:
+                return data
+            return data.astype({col_name: "int" for _, col_name in year_idx})
         elif name == "year":
             # The 'year' set itself
-            return df.astype(int)
+            assert isinstance(data, pd.Series)
+            return data.astype(int)
         else:
-            return df
+            return data
 
     # Override ixmp methods to convert 'year'-indexed columns to int
 
@@ -251,6 +265,28 @@ def add_par(
         # accepts int for "year"-like dimensions. Proxy the call to avoid type check
         # failures.
         # TODO Move this upstream, to ixmp
+
+        if on_ixmp4backend(self):
+            from message_ix.util.scenario_setup import check_existence_of_units
+
+            # Check for existence of required units
+            # NOTE these checks are similar to those in super().add_par(), but we
+            # need them here for access to self
+            # NOTE it seems to me that only dict or pd.DataFrame key_or_data could
+            # contain 'unit' already, else it's supplied by keyword or defaults to
+            # "???"
+            if isinstance(key_or_data, dict):
+                _data = pd.DataFrame.from_dict(key_or_data, orient="columns")
+            elif isinstance(key_or_data, pd.DataFrame):
+                _data = key_or_data
+            else:
+                _data = pd.DataFrame()
+
+            if "unit" not in _data.columns:
+                _data["unit"] = unit or "???"
+
+            check_existence_of_units(platform=self.platform, data=_data)
+
         super().add_par(name, key_or_data, value, unit, comment)  # type: ignore [arg-type]
 
     add_par.__doc__ = ixmp.Scenario.add_par.__doc__
@@ -323,6 +359,7 @@ def recurse(k, v, parent="World"):
             recurse(k, v)
 
         self.add_set("node", nodes)
+        # TODO do we handle levels being added multiple times correctly for ixmp4?
         self.add_set("lvl_spatial", levels)
         self.add_set("map_spatial_hierarchy", hierarchy)
 
@@ -420,8 +457,11 @@ def add_horizon(
         # Add the year set elements and first model year
         year = sorted(year)
         self.add_set("year", year)
+
+        # Avoid removing default data on IXMP4Backend
+        is_unique = True if isinstance(self.platform._backend, JDBCBackend) else False
         self.add_cat(
-            "year", "firstmodelyear", firstmodelyear or year[0], is_unique=True
+            "year", "firstmodelyear", firstmodelyear or year[0], is_unique=is_unique
         )
 
         # Calculate the duration of all periods
@@ -560,7 +600,7 @@ def vintage_and_active_years(
             vintages = sorted(
                 self.par(
                     "technical_lifetime",
-                    filters={"node_loc": ya_args[0], "technology": ya_args[1]},
+                    filters={"node_loc": [ya_args[0]], "technology": [ya_args[1]]},
                 )["year_vtg"].unique()
             )
             ya_max = max(vintages) if tl_only else np.inf
@@ -585,6 +625,7 @@ def vintage_and_active_years(
 
         # Minimum value for year_act
         if "in_horizon" in kwargs:
+            # FIXME I don't receive this in the tutorials
             warn(
                 "'in_horizon' argument to .vintage_and_active_years() will be removed "
                 "in message_ix>=4.0. Use .query(…) instead per documentation examples.",
@@ -842,3 +883,16 @@ def rename(self, name: str, mapping: Mapping[str, str], keep: bool = False) -> N
             self.remove_set(name, list(mapping.keys()))
 
         maybe_commit(self, commit, f"Rename {name!r} using mapping {mapping}")
+
+    def commit(self, comment: str) -> None:
+        from message_ix.util.scenario_setup import compose_maps
+
+        # JDBCBackend calls these functions as part of every commit, but they have moved
+        # to message_ix because they handle message-specific data
+
+        # The sanity checks fail for some tests (e.g. 'node' being empty)
+        # ensure_required_indexsets_have_data(scenario=self)
+
+        compose_maps(self)
+
+        return super().commit(comment)

message_ix/model/includes/period_parameter_assignment.gms

@@ -18,7 +18,7 @@ Sets
 
 Parameter
     duration_period_sum(year_all,year_all2) number of years between two periods ('year_all' must precede 'year_all2')
-    duration_time_rel(time,time2)         relative duration of subannual time slice 'time2' relative to parent 'time' (only for 'time' specified in set 'time_relative')')
+    duration_time_rel(time,time2)         relative duration of subannual time slice 'time2' relative to parent 'time' (only for 'time' specified in set 'time_relative')
     elapsed_years(year_all)    elapsed years since the start of the model horizon (not including 'year_all' period)
     remaining_years(year_all)  remaining years until the end of the model horizon (including last period)
     year_order(year_all)       order for members of set 'year_all'