From b27c2302d91deba3ee000b42afa33aa72d0776a8 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 27 Aug 2021 11:49:15 +0200 Subject: [PATCH 1/9] Type .tests.test_tutorials to satisfy mypy --- message_ix/tests/test_tutorials.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/message_ix/tests/test_tutorials.py b/message_ix/tests/test_tutorials.py index f4b99578a..5e407c80e 100644 --- a/message_ix/tests/test_tutorials.py +++ b/message_ix/tests/test_tutorials.py @@ -1,5 +1,6 @@ import sys from shutil import copyfile +from typing import List, Tuple import numpy as np import pytest @@ -17,7 +18,7 @@ # 3. Dictionary with extra keyword arguments to run_notebook(). # FIXME check objective function of the rest of tutorials. -tutorials = [ +tutorials: List[Tuple] = [ # IPython kernel ( ("westeros", "westeros_baseline"), From 4d8aa0d72f1930058a83d5237dbdeb19c84cabd0 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 27 Aug 2021 11:49:29 +0200 Subject: [PATCH 2/9] Add .tools.check() --- message_ix/tools/__init__.py | 5 ++ message_ix/tools/_check.py | 158 +++++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+) create mode 100644 message_ix/tools/_check.py diff --git a/message_ix/tools/__init__.py b/message_ix/tools/__init__.py index e69de29bb..c133f0764 100644 --- a/message_ix/tools/__init__.py +++ b/message_ix/tools/__init__.py @@ -0,0 +1,5 @@ +from ._check import check + +__all__ = [ + "check", +] diff --git a/message_ix/tools/_check.py b/message_ix/tools/_check.py new file mode 100644 index 000000000..3639c4f09 --- /dev/null +++ b/message_ix/tools/_check.py @@ -0,0 +1,158 @@ +from functools import partial +from logging import ERROR as FAIL +from logging import NOTSET as PASS +from logging import WARNING +from typing import Any, Dict, Iterable, Tuple, Union + +from message_ix.reporting import Reporter + +CHECKS = [] + + +def append(func): + CHECKS.append(func) + + +class Result: + """Container class for a check result and associated messages.""" + + desc: str + result: int + message: str = "" + + def __init__( + self, desc: str, result: int, message_or_lines: Union[str, Iterable[str]] = "" + ): + self.description = desc + self.result = ( + {True: PASS, False: FAIL}.get(result, result) + if isinstance(result, bool) + else result + ) + self.message = ( + message_or_lines + if isinstance(message_or_lines, str) + else "\n".join(message_or_lines) + ) + + def __bool__(self): + return self.result is PASS + + def __str__(self): + if self.result is PASS: + return f"PASS {self.description}" + else: + result_str = {WARNING: "WARNING", FAIL: "FAIL"}.get(self.result) + return f"{result_str} {self.description}:\n{self.message}" + + def __repr__(self): + return str(self) + + +def check(scenario, config=None) -> Dict[str, Tuple[bool, Any]]: + """Check that data in `scenario` is consistent with the MESSAGE(-MACRO) formulation. + + :func:`check` applies multiple checks, including: + + 1. ``technical_lifetime`` (maybe also ``var_cost`` missing for corresponding with + ``input``/``output`` values. + 2. Period “gaps” in data. + + Other checks that could be added include: + """ + # Create a Reporter + config = config or dict() + rep = Reporter.from_scenario(scenario, solved=False, **config) + + # Apply each function to the reporter; collect names and keys for check results + keys = [func(rep) for func in CHECKS] + + # Add a key that collects all check results + rep.add("check all", keys) + + # Trigger all checks + results = rep.get("check all") + + # Prepare an overall Result object that gives overall success or failure + results.insert(0, Result("Overall", all(map(bool, results)))) + + return results + + +@append +def var_cost(rep): + def _(vc, input, output): + # TODO implement the check + return Result("var_cost missing for corresponding input/output", PASS) + + key = {k: rep.full_key(k) for k in "var_cost input output".split()} + return rep.add( + "check var_cost", + _, + key["var_cost"], + key["input"], + key["output"], + ) + + +def check_gaps(qty, years, dim): + """Check `qty` for gaps in data along `dim`.""" + result = PASS + messages = [] + + # List of dimensions other than `dim` + dims = list(qty.dims) + dims.pop(dims.index(dim)) + + # Find the nulls in `qty`, then iterate over groups of `dims` + for key, group in qty.isnull().groupby(dims): + # print(key, group) + + # Set of years for which `qty` is not null, i.e. data exists + seen = set() + for idx, null in group.drop(dims).to_series().items(): + if not null: + seen.add(idx) + + # Elements from `years` that don't appear in `seen` + gaps = list( + filter(lambda y: min(seen) < y < max(seen) and y not in seen, years) + ) + + if len(gaps): + # At least 1 gap; format a message + result = WARNING + key_str = " ".join(f"{d}={k}" for d, k in zip(dims, key)) + messages.extend( + [ + f"- at indices: {key_str}", + f" for {dim}: {min(seen)} < {repr(gaps)} < {max(seen)}", + ] + ) + + return Result(f"Data gaps in '{qty.name}'", result, messages) + + +@append +def gaps_input(rep): + return rep.add( + "check gaps in input", partial(check_gaps, dim="ya"), rep.full_key("input"), "y" + ) + + +@append +def gaps_tl(rep): + return rep.add( + "check gaps in technical_lifetime", + partial(check_gaps, dim="yv"), + rep.full_key("technical_lifetime"), + "y", + ) + + +# @append +def map_tec(rep): + def _(scen): + return Result("map_tec is present", FAIL, repr(scen.set("map_tec"))) + + return rep.add("map_tec", _, "scenario") From c6e5270224d7c3c185e2dcc93a37391da31603b3 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 27 Aug 2021 11:50:17 +0200 Subject: [PATCH 3/9] Quiet logging in Reporter.from_scenario when solved=False --- message_ix/reporting/__init__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/message_ix/reporting/__init__.py b/message_ix/reporting/__init__.py index a726fa1d5..d46f7484d 100644 --- a/message_ix/reporting/__init__.py +++ b/message_ix/reporting/__init__.py @@ -189,11 +189,14 @@ def from_scenario(cls, scenario, **kwargs): solved = scenario.has_solution() if not solved: - log.warning( - f'Scenario "{scenario.model}/{scenario.scenario}" has no solution' - ) - log.warning("Some reporting may not function as expected") - fail_action = logging.DEBUG + if kwargs.pop("solved", True): + log.warning( + f'Scenario "{scenario.model}/{scenario.scenario}" has no solution' + ) + log.warning("Some reporting may not function as expected") + fail_action = logging.DEBUG + else: + fail_action = logging.NOTSET else: fail_action = "raise" From 870bfce9d373d8434315801e36d7d1f253c4e65e Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 27 Aug 2021 11:52:34 +0200 Subject: [PATCH 4/9] Add tests of .tools.check() --- message_ix/tests/tools/test_check.py | 45 ++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 message_ix/tests/tools/test_check.py diff --git a/message_ix/tests/tools/test_check.py b/message_ix/tests/tools/test_check.py new file mode 100644 index 000000000..9ccb8c4b1 --- /dev/null +++ b/message_ix/tests/tools/test_check.py @@ -0,0 +1,45 @@ +import pytest + +from message_ix import Scenario, make_df +from message_ix.testing import make_dantzig, make_westeros +from message_ix.tools import check + + +def test_check_dantzig(test_mp): + scen = make_dantzig(test_mp) + + # Checks all True + results = check(scen) + assert results[0] + + +def test_check_westeros(test_mp): + scen = make_westeros(test_mp) + + # Minimal config to make Westeros reportable + config = {"units": {"replace": {"-": ""}}} + + # Checks all pass + results = check(scen, config=config) + assert results[0] + + # Delete one value + to_delete = make_df( + "input", + node_loc="Westeros", + technology="bulb", + year_vtg=690, + year_act=710, + mode="standard", + node_origin="Westeros", + commodity="electricity", + level="final", + time="year", + time_origin="year", + ).dropna(axis=1) + with scen.transact(): + scen.remove_par("input", to_delete) + + # Checks fail + results = check(scen, config=config) + assert not results[0] From 6594133d7673dcddbe0ba8d3f2f34124b7592424 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 27 Aug 2021 12:16:37 +0200 Subject: [PATCH 5/9] Add test_check_existing() --- message_ix/tests/tools/test_check.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/message_ix/tests/tools/test_check.py b/message_ix/tests/tools/test_check.py index 9ccb8c4b1..075111416 100644 --- a/message_ix/tests/tools/test_check.py +++ b/message_ix/tests/tools/test_check.py @@ -43,3 +43,27 @@ def test_check_westeros(test_mp): # Checks fail results = check(scen, config=config) assert not results[0] + + +@pytest.mark.parametrize( + "url, config", + [ + # ("ixmp://platform/model/scenario#version", dict()), + ], +) +def test_check_existing(url, config): + """Check existing scenarios. + + For local use only: extend the list of parameters, above, but do not commit + additions to ``main``. + """ + # import pint + # from iam_units import registry + # + # pint.set_application_registry(registry) + + scen, mp = Scenario.from_url(url) + results = check(scen, config=config) + + # Checks all pass + assert results[0], "\n".join(map(str, results)) From fca5ae5a5e48fb1a6acc56911640333bb678602e Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 27 Aug 2021 13:08:23 +0200 Subject: [PATCH 6/9] Add check for #503 --- message_ix/tests/tools/test_check.py | 32 ++++++++++++++++++++++ message_ix/tools/_check.py | 41 ++++++++++++++++++++++++++-- 2 files changed, 70 insertions(+), 3 deletions(-) diff --git a/message_ix/tests/tools/test_check.py b/message_ix/tests/tools/test_check.py index 075111416..0cdda5f4a 100644 --- a/message_ix/tests/tools/test_check.py +++ b/message_ix/tests/tools/test_check.py @@ -45,6 +45,38 @@ def test_check_westeros(test_mp): assert not results[0] +def test_check_tl_integer(test_mp): + scen = make_westeros(test_mp) + + # Minimal config to make Westeros reportable + config = {"units": {"replace": {"-": ""}}} + + # Change one value + tl = "technical_lifetime" + with scen.transact(): + scen.add_par( + tl, + make_df( + tl, + node_loc="Westeros", + technology="bulb", + year_vtg=700, + value=1.1, + unit="y", + ), + ) + + # Checks fail + results = check(scen, config=config) + assert not results[0] + + assert """FAIL Non-integer values for technical_lifetime: +See https://github.com/iiasa/message_ix/issues/503. +- 1.1 at indices: nl=Westeros t=bulb yv=700""" in map( + str, results + ) + + @pytest.mark.parametrize( "url, config", [ diff --git a/message_ix/tools/_check.py b/message_ix/tools/_check.py index 3639c4f09..c87879b7c 100644 --- a/message_ix/tools/_check.py +++ b/message_ix/tools/_check.py @@ -1,4 +1,5 @@ from functools import partial +from inspect import getdoc from logging import ERROR as FAIL from logging import NOTSET as PASS from logging import WARNING @@ -11,12 +12,13 @@ def append(func): CHECKS.append(func) + return func class Result: """Container class for a check result and associated messages.""" - desc: str + description: str result: int message: str = "" @@ -95,6 +97,10 @@ def _(vc, input, output): ) +def key_str(dims, key): + return " ".join(f"{d}={k}" for d, k in zip(dims, key)) + + def check_gaps(qty, years, dim): """Check `qty` for gaps in data along `dim`.""" result = PASS @@ -122,10 +128,9 @@ def check_gaps(qty, years, dim): if len(gaps): # At least 1 gap; format a message result = WARNING - key_str = " ".join(f"{d}={k}" for d, k in zip(dims, key)) messages.extend( [ - f"- at indices: {key_str}", + f"- at indices: {key_str(dims, key)}", f" for {dim}: {min(seen)} < {repr(gaps)} < {max(seen)}", ] ) @@ -150,6 +155,36 @@ def gaps_tl(rep): ) +def munge_docstring(func): + """Prepare Result.description and Result.messages from ``func.__doc__``.""" + lines = getdoc(func).split("\n") + return lines[0].rstrip("."), lines[2:] + + +@append +def tl_integer(rep): + """Non-integer values for technical_lifetime. + + See https://github.com/iiasa/message_ix/issues/503. + """ + + def _(tl): + # Quick check: convert all values to integer and look for changed values + mask = tl.astype(int) != tl + result = FAIL if mask.any() else PASS + + desc, messages = munge_docstring(tl_integer) + + # Process item-wise, only if there was some failure + for key, non_int in mask.groupby(list(tl.dims)) if result is FAIL else (): + if non_int.item(): + messages.append(f"- {tl.loc[key]} at indices: {key_str(tl.dims, key)}") + + return Result(desc, result, messages) + + return rep.add("check tl integer", _, rep.full_key("technical_lifetime")) + + # @append def map_tec(rep): def _(scen): From bcf56c97f0ea2570029aeb83549e30763e17a970 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 27 Aug 2021 13:50:39 +0200 Subject: [PATCH 7/9] Document .tools.check() --- doc/index.rst | 2 - doc/tools/check.rst | 27 +++++++++++ message_ix/tools/_check.py | 93 +++++++++++++++++++++++++++----------- 3 files changed, 93 insertions(+), 29 deletions(-) create mode 100644 doc/tools/check.rst diff --git a/doc/index.rst b/doc/index.rst index 4a123dce0..23c08c95a 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -48,8 +48,6 @@ Then, continue with the: tutorials - - .. _core: Mathematical specification diff --git a/doc/tools/check.rst b/doc/tools/check.rst new file mode 100644 index 000000000..9d675f5d5 --- /dev/null +++ b/doc/tools/check.rst @@ -0,0 +1,27 @@ +Pre-solve checks +**************** + +.. currentmodule:: message_ix.tools + +.. autofunction:: check + +.. currentmodule:: message_ix.tools._check + +.. _checks-summary: + +.. automodule:: message_ix.tools._check + :members: gaps_input, gaps_tl, tl_integer + + .. autoclass:: Result + + Individual checks + ================= + + Gaps in certain parameters (:func:`gaps_*` check functions) can cause the GAMS implementation to treat that specific technology to be disabled in one period, between other periods where it *is* enabled. + This may prevent solution of the model. + + .. autosummary:: + + gaps_input + gaps_tl + tl_integer diff --git a/message_ix/tools/_check.py b/message_ix/tools/_check.py index c87879b7c..048f96d05 100644 --- a/message_ix/tools/_check.py +++ b/message_ix/tools/_check.py @@ -3,23 +3,45 @@ from logging import ERROR as FAIL from logging import NOTSET as PASS from logging import WARNING -from typing import Any, Dict, Iterable, Tuple, Union +from typing import Dict, Iterable, List, Optional, Union -from message_ix.reporting import Reporter +from message_ix import Reporter, Scenario +#: List of all check functions. CHECKS = [] +# Utility functions + def append(func): + """Add `func` to `CHECKS`.""" CHECKS.append(func) return func +def key_str(dims, key): + return " ".join(f"{d}={k}" for d, k in zip(dims, key)) + + +def munge_docstring(func): + """Prepare Result.description and Result.messages from ``func.__doc__``.""" + lines = getdoc(func).split("\n") + return lines[0].rstrip("."), lines[2:] + + class Result: - """Container class for a check result and associated messages.""" + """Container class for a check result and associated messages. + Use ``str(result)`` for a complete, formatted description of the result. + """ + + #: 1-line description of the check. description: str + + #: Outcome of the check. Either 0 (PASS), 30 (WARNING), or 40 (FAIL). result: int + + #: Descriptive message when :attr:`result` is not PASS. message: str = "" def __init__( @@ -45,22 +67,44 @@ def __str__(self): return f"PASS {self.description}" else: result_str = {WARNING: "WARNING", FAIL: "FAIL"}.get(self.result) - return f"{result_str} {self.description}:\n{self.message}" + return f"{result_str} {self.description}" + ( + f":\n{self.message}" if self.message else "" + ) def __repr__(self): return str(self) -def check(scenario, config=None) -> Dict[str, Tuple[bool, Any]]: +def check(scenario: Scenario, config: Optional[Dict] = None) -> List[Result]: """Check that data in `scenario` is consistent with the MESSAGE(-MACRO) formulation. - :func:`check` applies multiple checks, including: - - 1. ``technical_lifetime`` (maybe also ``var_cost`` missing for corresponding with - ``input``/``output`` values. - 2. Period “gaps” in data. + :func:`check` applies multiple checks; see :ref:`the list of checks + `. + + Parameters + ---------- + scenario : + Scenario to be checked, possibly without a model solution. + config : optional + Configuration for :class:`~message_ix.Reporter`. + + Returns + ------- + list of :class:`.Result` + The first object is an "overall" result: PASS iff all the other checks PASS. + Subsequent object are results for individual checks. + + Example + ------- + >>> from message_ix.tools import check + >>> results = check(scen) + >>> print(" ".join(map(str, results)) + PASS Overall + PASS var_cost missing for corresponding input/output + PASS Data gaps in ``input`` along the year_act dimension + PASS Data gaps in ``technical_lifetime`` along the year_vtg dimension + PASS Non-integer values for ``technical_lifetime`` - Other checks that could be added include: """ # Create a Reporter config = config or dict() @@ -81,6 +125,9 @@ def check(scenario, config=None) -> Dict[str, Tuple[bool, Any]]: return results +# Individual checks + + @append def var_cost(rep): def _(vc, input, output): @@ -97,10 +144,6 @@ def _(vc, input, output): ) -def key_str(dims, key): - return " ".join(f"{d}={k}" for d, k in zip(dims, key)) - - def check_gaps(qty, years, dim): """Check `qty` for gaps in data along `dim`.""" result = PASS @@ -140,6 +183,7 @@ def check_gaps(qty, years, dim): @append def gaps_input(rep): + """Data gaps in ``input`` along the year_act dimension.""" return rep.add( "check gaps in input", partial(check_gaps, dim="ya"), rep.full_key("input"), "y" ) @@ -147,6 +191,7 @@ def gaps_input(rep): @append def gaps_tl(rep): + """Data gaps in ``technical_lifetime`` along the year_vtg dimension.""" return rep.add( "check gaps in technical_lifetime", partial(check_gaps, dim="yv"), @@ -155,15 +200,9 @@ def gaps_tl(rep): ) -def munge_docstring(func): - """Prepare Result.description and Result.messages from ``func.__doc__``.""" - lines = getdoc(func).split("\n") - return lines[0].rstrip("."), lines[2:] - - @append def tl_integer(rep): - """Non-integer values for technical_lifetime. + """Non-integer values for ``technical_lifetime``. See https://github.com/iiasa/message_ix/issues/503. """ @@ -186,8 +225,8 @@ def _(tl): # @append -def map_tec(rep): - def _(scen): - return Result("map_tec is present", FAIL, repr(scen.set("map_tec"))) - - return rep.add("map_tec", _, "scenario") +# def map_tec(rep): +# def _(scen): +# return Result("map_tec is present", FAIL, repr(scen.set("map_tec"))) +# +# return rep.add("map_tec", _, "scenario") From 9fbbc94842a9baaaa83577532b8e814f7f55300c Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 27 Aug 2021 13:51:05 +0200 Subject: [PATCH 8/9] Use a fixture in test_check.py --- message_ix/tests/tools/test_check.py | 35 ++++++++++++---------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/message_ix/tests/tools/test_check.py b/message_ix/tests/tools/test_check.py index 0cdda5f4a..3c2b07351 100644 --- a/message_ix/tests/tools/test_check.py +++ b/message_ix/tests/tools/test_check.py @@ -13,16 +13,16 @@ def test_check_dantzig(test_mp): assert results[0] -def test_check_westeros(test_mp): - scen = make_westeros(test_mp) +@pytest.fixture +def westeros(test_mp): + yield make_westeros(test_mp) - # Minimal config to make Westeros reportable - config = {"units": {"replace": {"-": ""}}} - # Checks all pass - results = check(scen, config=config) - assert results[0] +# Minimal config to make Westeros reportable +WESTEROS_CONFIG = {"units": {"replace": {"-": ""}}} + +def test_gaps_input(westeros): # Delete one value to_delete = make_df( "input", @@ -37,24 +37,19 @@ def test_check_westeros(test_mp): time="year", time_origin="year", ).dropna(axis=1) - with scen.transact(): - scen.remove_par("input", to_delete) + with westeros.transact(): + westeros.remove_par("input", to_delete) # Checks fail - results = check(scen, config=config) + results = check(westeros, config=WESTEROS_CONFIG) assert not results[0] -def test_check_tl_integer(test_mp): - scen = make_westeros(test_mp) - - # Minimal config to make Westeros reportable - config = {"units": {"replace": {"-": ""}}} - +def test_check_tl_integer(westeros): # Change one value tl = "technical_lifetime" - with scen.transact(): - scen.add_par( + with westeros.transact(): + westeros.add_par( tl, make_df( tl, @@ -67,10 +62,10 @@ def test_check_tl_integer(test_mp): ) # Checks fail - results = check(scen, config=config) + results = check(westeros, config=WESTEROS_CONFIG) assert not results[0] - assert """FAIL Non-integer values for technical_lifetime: + assert """FAIL Non-integer values for ``technical_lifetime``: See https://github.com/iiasa/message_ix/issues/503. - 1.1 at indices: nl=Westeros t=bulb yv=700""" in map( str, results From 4dae5fb6be4609ebbe642817c24c437a6a5ea319 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 27 Aug 2021 13:52:08 +0200 Subject: [PATCH 9/9] Add #506 to release notes --- RELEASE_NOTES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index e0a375dc8..ace9c5d70 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -4,6 +4,7 @@ Next release All changes ----------- +- Add a tool for :doc:`tools/check` (:pull:`506`). - Adjust test suite for pyam v1.1.0 compatibility (:pull:`499`). - Add Westeros :doc:`tutorial ` on historical parameters (:pull:`478`). - Update reference for activity and capacity soft constraints (:pull:`474`).