Merge pull request #991 from iiasa/issue/988

Fix 2 reporting bugs

Author: khaeru

RELEASE_NOTES.rst

@@ -47,6 +47,11 @@ All changes
   - :mod:`message_ix.message` includes :class:`.MESSAGE`.
   - :mod:`message_ix.message_macro` includes :class:`.MESSAGE_MACRO`.
 
+- Improve :class:`.Reporter` and its documentation (:pull:`991`).
+
+  - Handle older scenarios—for instance, those without |input_cap|—in :meth:`.Reporter.from_scenario` (:issue:`988`).
+  - Expand and test keys for multiple methods of calculating :ref:`reporter-historical` (:issue:`989`).
+
 - Document the :ref:`minimum version of Java <install-java>` required for :class:`ixmp.JDBCBackend <ixmp.backend.jdbc.JDBCBackend>` (:pull:`962`).
 - Document :ref:`how to run a local PostgreSQL instance <install-postgres>`
   for local testing using :class:`ixmp.IXMP4Backend <ixmp.backend.ixmp4.IXMP4Backend>` (:pull:`981`).

doc/conf.py

@@ -184,7 +184,7 @@
 gh_ref = "main" if ".dev" in version else f"v{version}"
 
 extlinks = {
-    "issue": ("https://github.com/iiasa/message_ix/issue/%s", "#%s"),
+    "issue": ("https://github.com/iiasa/message_ix/issues/%s", "#%s"),
     "pull": ("https://github.com/iiasa/message_ix/pull/%s", "PR #%s"),
     "tut": (f"https://github.com/iiasa/message_ix/blob/{gh_ref}/tutorial/%s", None),
 }

doc/reporting.rst

@@ -40,9 +40,7 @@ idna==3.7
     # via requests
 imagesize==1.4.1
     # via sphinx
-importlib-metadata==8.7.0
-    # via ixmp
-ixmp @ git+https://github.com/iiasa/ixmp.git@1730afd24238b4dad88a89c2d58d0fed2cd19dd0
+ixmp @ git+https://github.com/iiasa/ixmp.git@51ccb3276ea1d85a437ad14232890f3f3e8dec70
     # via -r requirements.in
 jinja2==3.1.6
     # via sphinx
@@ -165,5 +163,3 @@ xarray==2025.7.1
     # via
     #   genno
     #   ixmp
-zipp==3.23.0
-    # via importlib-metadata

doc/requirements.txt

@@ -5,6 +5,7 @@
 from contextlib import contextmanager
 from copy import copy
 from dataclasses import InitVar, dataclass, field
+from functools import cache
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
@@ -15,6 +16,7 @@
 if TYPE_CHECKING:
     from logging import LogRecord
 
+    from genno import Key
     from ixmp.types import InitializeItemsKwargs
 
 
@@ -68,9 +70,11 @@
     "yr": ("year", "year_rel"),
     "yv": ("year", "year_vtg"),
 }
+# Inverse mapping
+DIMS_INVERSE = {v[1]: k for k, v in DIMS.items()}
 
 
-@dataclass
+@dataclass(unsafe_hash=True)
 class Item:
     """Description of an :mod:`ixmp` item: equation, parameter, set, or variable.
 
@@ -118,6 +122,18 @@ def ix_type(self) -> str:
         """
         return str(self.type.name).lower()
 
+    @property
+    @cache
+    def key(self) -> "Key":
+        """:class:`genno.Key` for this Item in a :class:`.Reporter`.
+
+        Read-only.
+        """
+        from genno import Key
+
+        dims = [DIMS_INVERSE.get(d, d) for d in self.dims or self.coords]
+        return Key(self.name, dims)
+
     def to_dict(self) -> "InitializeItemsKwargs":
         """Return the :class:`dict` representation used internally in :mod:`ixmp`."""
         result: "InitializeItemsKwargs" = dict(

message_ix/common.py

@@ -1,30 +1,30 @@
 import logging
 from collections.abc import Mapping
 from functools import lru_cache, partial
+from itertools import product
 from operator import itemgetter
 from typing import TYPE_CHECKING, cast
 
+from genno import ComputationError, Key, KeyExistsError, Keys, MissingKeyError
 from genno.operator import broadcast_map
-from ixmp.report import (
-    ComputationError,
-    Key,
-    KeyExistsError,
-    MissingKeyError,
-    Quantity,
-    configure,
-)
+from ixmp.backend import ItemType
+from ixmp.model import get_model
+from ixmp.report import Quantity, configure
 from ixmp.report import Reporter as IXMPReporter
 
 from message_ix.common import DIMS
 
 from .pyam import collapse_message_cols
 
 if TYPE_CHECKING:
+    from message_ix.common import GAMSModel
+
     from .pyam import CollapseMessageColsKw
 
 __all__ = [
     "ComputationError",
     "Key",
+    "Keys",
     "KeyExistsError",
     "MissingKeyError",
     "Quantity",
@@ -61,6 +61,9 @@
     ("map_tec", "map_as_qty", "cat_tec", "t"),
     ("map_year", "map_as_qty", "cat_year", "y"),
     #
+    # Derived set contents
+    ("current:ya-yv", "yv_ya_current", "y"),
+    #
     # Products
     ("out", "mul", "output", "ACT"),
     ("in", "mul", "input", "ACT"),
@@ -69,6 +72,8 @@
     ("rel", "mul", "relation_activity", "ACT"),
     ("emi", "mul", "emission_factor", "ACT"),
     ("inv", "mul", "inv_cost", "CAP_NEW"),
+    ("inv::historical", "mul", "inv_cost", "historical_activity"),
+    ("inv::ref", "mul", "inv_cost", "ref_activity"),
     ("fom", "mul", "fix_cost", "CAP"),
     ("vom", "mul", "var_cost", "ACT"),
     ("land_out", "mul", "land_output", "LAND"),
@@ -154,6 +159,8 @@
 @lru_cache(1)
 def get_tasks() -> list[tuple[tuple, Mapping]]:
     """Return a list of tasks describing MESSAGE reporting calculations."""
+    from message_ix.message import MESSAGE
+
     # Assemble queue of items to add. Each element is a 2-tuple of (positional, keyword)
     # arguments for Reporter.add()
     to_add: list[tuple[tuple, Mapping]] = []
@@ -162,15 +169,46 @@ def get_tasks() -> list[tuple[tuple, Mapping]]:
 
     for t in TASKS0:
         if len(t) == 2 and isinstance(t[1], dict):
-            # (args, kwargs) → update kwargs with strict
-            t[1].update(strict)
-            to_add.append(cast(tuple[tuple, Mapping], t))
+            # (args, kwargs) → use strict unless already set
+            to_add.append((t[0], strict | t[1]))
         else:
             # args only → use strict as kwargs
             to_add.append((t, strict))
 
-    # Conversions to IAMC data structure and pyam objects
+    # Products using {historical,ref}_activity instead of ACT
+    for (name, io), act in product(
+        [
+            ("emi", "emission_factor"),
+            ("in", "input"),
+            ("out", "output"),
+            ("vom", "var_cost"),
+        ],
+        ["historical", "ref"],
+    ):
+        # Construct some keys
+        k_io = MESSAGE.items[io].key
+        k_act = MESSAGE.items[f"{act}_activity"].key
+        k_share = Key("share", k_act.dims, f"{name}+{act}") * "yv"
+        k = Key(name, sorted(set(k_io.dims + k_act.dims)), act)
+        desc = f"share of contribution of each vy to {k_act.name} in each ya"
+
+        to_add.extend(
+            [
+                ((k["full"], "mul", k_io, k_act), strict),
+                ((k["current"], "mul", k["full"], "current:ya-yv"), strict),
+                ((k_share, "missing_data"), dict(key=k_share, description=desc)),
+                (
+                    (k["weighted+full"], "mul", k["full"], k_share),
+                    dict(sums=False) | strict,
+                ),
+                (
+                    (k["weighted"] / "yv", "sum", k["weighted+full"]),
+                    dict(dimensions=["yv"], sums=True) | strict,
+                ),
+            ]
+        )
 
+    # Conversions to IAMC data structure and pyam objects
     for qty, collapse_kw in PYAM_CONVERT:
         # Column to set as year dimension + standard renames from MESSAGE to IAMC dims
         rename = {
@@ -213,9 +251,9 @@ def from_scenario(cls, scenario, **kwargs) -> "Reporter":
         .Reporter
             A reporter for `scenario`.
         """
-        solved = scenario.has_solution()
+        import genno
 
-        if not solved:
+        if not scenario.has_solution():
             log.warning(
                 f'Scenario "{scenario.model}/{scenario.scenario}" has no solution'
             )
@@ -227,6 +265,22 @@ def from_scenario(cls, scenario, **kwargs) -> "Reporter":
         # Invoke the ixmp method
         rep = cast("Reporter", super().from_scenario(scenario, **kwargs))
 
+        # Handle missing parameters
+        missing = set()
+        for name, item in cast("GAMSModel", get_model(scenario.scheme)).items.items():
+            if item.type is not ItemType.PAR or item.key in rep:
+                continue
+            rep.add(item.key, lambda: genno.Quantity())
+            missing.add(item.name)
+
+        if missing:
+            log.warning(
+                f"Scenario {scenario.url!r} is missing {len(missing)} parameter(s):"
+                + "\n- ".join(sorted({""} | missing))
+                + "\n…possibly added by a newer version of message_ix. These keys will "
+                "return empty Quantity()."
+            )
+
         # Add the MESSAGEix calculations
         rep.add_tasks(fail_action)
 

message_ix/report/__init__.py

@@ -1,18 +1,21 @@
 from collections.abc import Mapping
 from typing import TYPE_CHECKING, Literal, overload
 
+import genno
 import pandas as pd
 
 from message_ix.util import make_df
 
 if TYPE_CHECKING:
-    from genno.types import AnyQuantity
+    from genno.types import AnyQuantity, KeyLike
 
 __all__ = [
     "as_message_df",
+    "missing_data",
     "model_periods",
     "plot_cumulative",
     "stacked_bar",
+    "yv_ya_current",
 ]
 
 
@@ -80,6 +83,16 @@ def as_message_df(qty, name, dims, common, wrap=True):
     return {name: df} if wrap else df
 
 
+def missing_data(key: "KeyLike", description: str = "") -> None:
+    """Raise an exception for missing user-supplied data."""
+    raise RuntimeError(
+        f"Must supply data for {key!r}. Replace {key!r} in the Reporter with a task "
+        "returning genno.Quantity with the same dimensions"
+        + (f" representing {description}" if description else "")
+        + "."
+    )
+
+
 def model_periods(y: list[int], cat_year: pd.DataFrame) -> list[int]:
     """Return the elements of `y` beyond the firstmodelyear of `cat_year`."""
     return list(
@@ -200,3 +213,9 @@ def stacked_bar(
     ax.legend(loc="center left", bbox_to_anchor=(1.0, 0.5))
 
     return ax
+
+
+def yv_ya_current(years: list[int]) -> "AnyQuantity":
+    """2-dimensional Quantity with value 1.0 where :math:`y^V = y^A`."""
+    idx = pd.MultiIndex.from_tuples([(y, y) for y in years], names=["yv", "ya"])
+    return genno.Quantity(pd.Series(1.0, index=idx))