Add #451 to docs, whatsnew

Author: khaeru

doc/api/report/index.rst

@@ -139,13 +139,37 @@ API reference
 .. automodule:: message_ix_models.report
    :members:
 
+   General-purpose code:
+
    .. autosummary::
 
       Config
+      defaults
       prepare_reporter
       register
       report
 
+   The following submodules prepare reporting of specific measures or quantities:
+
+   .. autosummary::
+      :toctree: _autosummary
+      :template: autosummary-module.rst
+      :recursive:
+
+      extraction
+
+.. currentmodule:: message_ix_models.report.key
+
+Keys
+----
+
+.. automodule:: message_ix_models.report.key
+   :members:
+
+   Pre-set :class:`genno.Key` instances
+   and :class:`genno.Keys` in this module can be referenced across modules,
+   instead of hand-typing the same strings in multiple places.
+
 .. currentmodule:: message_ix_models.report.plot
 
 Plots
@@ -167,17 +191,25 @@ Operators
 
    .. autosummary::
 
+      broadcast_wildcard
+      call
       codelist_to_groups
       compound_growth
       exogenous_data
       filter_ts
       from_url
+      get_commodity_groups
       get_ts
       gwp_factors
       make_output_path
       model_periods
+      node_glb
+      nodes_world_agg
       remove_ts
+      select_allow_empty
+      select_expand
       share_curtailment
+      zeros_like
 
    The following functions, defined elsewhere,
    are exposed through :mod:`.operator`
@@ -186,6 +218,8 @@ Operators
    .. autosummary::
 
       message_ix_models.util.add_par_data
+      message_ix_models.util.merge_data
+      message_ix_models.util.nodes_ex_world
 
    Other operators or genno-compatible functions are provided by:
 
@@ -219,6 +253,7 @@ Utilities
 
    .. autosummary::
 
+      IAMCConversion
       add_replacements
       collapse
       collapse_gwp_info

doc/pkg-data/codelists.rst

@@ -18,9 +18,18 @@ These codes have the following annotations:
 ``units`` (mandatory)
    Units typically associated with this commodity.
 ``iea-eweb-flow`` (optional)
-   List of ``FLOW`` codes from the IEA :ref:`tools-iea-web` associated with this MESSAGEix-GLOBIOM commodity.
+   List of ``FLOW`` codes from the IEA :ref:`tools-iea-web`
+   associated with this MESSAGEix-GLOBIOM commodity.
 ``iea-eweb-product`` (optional)
-   List of ``PRODUCT`` codes from the IEA :ref:`tools-iea-web` associated with this MESSAGEix-GLOBIOM commodity.
+   List of ``PRODUCT`` codes from the IEA :ref:`tools-iea-web`
+   associated with this MESSAGEix-GLOBIOM commodity.
+``report`` (optional)
+   English name for IAMC-structured reporting output.
+   See :func:`.add_replacements`.
+``report-only`` (optional)
+   :class:`bool`.
+   If :any:`True`,
+   the code is ignored by :func:`.bare.get_spec`.
 
 .. literalinclude:: ../../message_ix_models/data/commodity.yaml
    :language: yaml

doc/whatsnew.rst

@@ -11,6 +11,11 @@ Next release
   - New method :meth:`~.report.Config.iter_callbacks`.
 
 - New function :func:`.tools.iamc.compare` (:pull:`178`).
+- New module :mod:`.report.extraction` for reporting of resource extraction (:pull:`451`).
+- New reporting operators (:pull:`451`):
+  :func:`.get_commodity_groups`,
+  :func:`.node_glb`, and
+  :func:`.zeros_like`.
 - Expand :doc:`api/report/index` documentation (:pull:`178`)
   to cover features implemented/not implemented by :mod:`genno`-based reporting.
   Module globals :data:`.NOT_IMPLEMENTED_MEASURE` and :data:`.NOT_IMPLEMENTED_IAMC`

message_ix_models/report/__init__.py

@@ -29,6 +29,7 @@
     "NOT_IMPLEMENTED_IAMC",
     "NOT_IMPLEMENTED_MEASURE",
     "Config",
+    "defaults",
     "prepare_reporter",
     "register",
     "report",
@@ -327,6 +328,9 @@ def prepare_reporter(
 ) -> tuple[Reporter, "KeyLike | None"]:
     """Return a :class:`.Reporter` and `key` prepared to report a :class:`.Scenario`.
 
+    Every function returned by :attr:`.Config.iter_callbacks` is called, in order, to
+    allow each to populate the `reporter` with additional tasks.
+
     Parameters
     ----------
     context : .Context
@@ -342,8 +346,8 @@ def prepare_reporter(
     Returns
     -------
     .Reporter
-        Reporter prepared with MESSAGEix-GLOBIOM calculations; if `reporter` is given,
-        this is a reference to the same object.
+        Reporter prepared with tasks for reporting a MESSAGEix-GLOBIOM scenario; if
+        `reporter` is given, this is a reference to the same object.
 
         If :attr:`.cli_output` is given, a task with the key "cli-output" is added that
         writes the :attr:`.Config.key` to that path.
@@ -430,6 +434,16 @@ def prepare_reporter(
 
 
 def defaults(rep: Reporter, context: Context) -> None:
+    """Prepare default contents and configuration of `rep` for MESSAGEix-GLOBIOM.
+
+    This includes:
+
+    - Populate :data:`.key.coords` and :data:`.key.groups`.
+    - Add a :func:`genno.operator.concat` task with no arguments at
+      :data:`.key.all_iamc`.
+    - Call :func:`.add_replacements` for members of the :ref:`commodity-yaml` and
+      :ref:`technology-yaml` code lists
+    """
     from message_ix_models.model.structure import get_codes
 
     from . import key as k

message_ix_models/report/config.py

@@ -28,6 +28,7 @@ def _default_callbacks() -> list[Callback]:
 
     from . import defaults, extraction
 
+    # NB When updating this list, also update the docstring of Config.callback
     return [defaults, extraction.callback, plot.callback]
 
 
@@ -45,13 +46,17 @@ class Config(ConfigHelper):
     #: Shorthand to set :py:`legacy["use"]` on a new instance.
     _legacy: InitVar[bool | None] = False
 
-    #: List of callbacks for preparing the :class:`.Reporter`.
+    #: List of callback functions for preparing the :class:`.Reporter`.
     #:
     #: Each registered function is called by :func:`~.report.prepare_reporter` in order
     #: to add or modify the task graph. Specific model variants and projects can
-    #: register a callback to extend the reporting graph.
+    #: register a callback to extend the reporting graph. The default list is:
     #:
-    #: Callback functions **must** take two arguments: the Computer/Reporter, and a
+    #: 1. :func:`.report.defaults`
+    #: 2. :func:`.report.extraction.callback`
+    #: 3. :func:`.report.plot.callback`
+    #:
+    #: A callback function **must** take two arguments: the Computer/Reporter, and a
     #: :class:`.Context`:
     #:
     #: .. code-block:: python
@@ -63,7 +68,7 @@ class Config(ConfigHelper):
     #:         # Modify `rep` by calling its methods ...
     #:         pass
     #:
-    #:     # Register this callback on an existing Context instance
+    #:     # Register this callback on an existing Context/report.Config instance
     #:     context.report.register(cb)
     #:
     #: See also :attr:`modules`.
@@ -78,7 +83,7 @@ class Config(ConfigHelper):
     #: Key for the Quantity or computation to report.
     key: "KeyLike | None" = None
 
-    #: Names of modules with reporting callbacks. See also :attr:`callbacks` and
+    #: Names of modules with reporting callbacks. See also :attr:`callback` and
     #: :meth:`iter_callbacks`.
     modules: list[str] = field(default_factory=list)
 
@@ -106,8 +111,8 @@ def iter_callbacks(self) -> Generator[Callback, None, None]:
         """Iterate over callback functions.
 
         1. All module names in :attr:`modules` are passed to :meth:`register`, such that
-           their callback functions are appended to :attr:`callbacks`.
-        2. The :attr:`callbacks` are yielded iteratively.
+           their callback functions are appended to :attr:`callback`.
+        2. The :attr:`callback` are yielded iteratively.
         """
         for name in self.modules:
             self.register(name)

message_ix_models/report/extraction.py

@@ -1,3 +1,5 @@
+"""Report resource extraction."""
+
 from typing import TYPE_CHECKING
 
 from genno import Keys

message_ix_models/report/key.py

@@ -2,20 +2,28 @@
 
 from genno import Key, Keys
 
+#: Gross domestic product.
 GDP = Key("GDP", "ny")
 
-# NB genno ≤ 1.27.1 is sensitive to the order
+#: Commodity price.
+#:
+#:  .. note:: genno ≤ 1.27.1 is sensitive to the dimension order; more recent versions
+#:     are not.
 PRICE_COMMODITY = Key("PRICE_COMMODITY", "nclyh")
 
 #: All IAMC-structured data.
 all_iamc = Key("all", (), "iamc")
 
-#: Identifiers for coordinates.
+#: Identifiers for coordinates, including:
+#:
+#: - :py:`.n_glb`: the output of :func:`.node_glb`.
 coords = Keys(
     n_glb="n::glb",
 )
 
-#: Identifiers for grouping/aggregation mappings.
+#: Identifiers for grouping/aggregation mappings, including:
+#:
+#: - :py:`.c`: the output of :func:`.get_commodity_groups`.
 groups = Keys(
     c="c::groups",
 )

message_ix_models/report/operator.py

@@ -165,9 +165,19 @@ def filter_ts(df: pd.DataFrame, expr: re.Pattern, *, column="variable") -> pd.Da
 
 @cache
 def get_commodity_groups() -> dict[Literal["c"], dict[str, list[str]]]:
-    """Return groups of commodities for reporting of extraction.
-
-    The structure is retrieved from :ref:`commodity-yaml` using :func:`.leaf_ids`.
+    """Return groups of commodities for aggregation.
+
+    The structure is retrieved from :ref:`commodity-yaml` using :func:`.leaf_ids`. The
+    group IDs include only those commodities from the code list with children. The
+    group members include only 'leaf' codes via :func:`.leaf_ids`; any intermediate
+    codes that have children are expanded to the list of those children.
+
+    Returns
+    -------
+    dict
+        with one top-level key, 'c', and at the second level mapping from group IDs to
+        their components. This is suitable for use as the :py:`groups` argument to
+        :func:`genno.operator.aggregate`.
     """
     cl = get_codelist("commodity")
     return {"c": {c.id: leaf_ids(c) for c in filter(lambda c: len(c.child), cl)}}

message_ix_models/report/util.py

@@ -24,6 +24,8 @@
 #:
 #: - Applied to whole strings along each dimension.
 #: - These columns have :meth:`str.title` applied before these replacements.
+#:
+#: See also :func:`add_replacements`.
 REPLACE_DIMS: dict[str, dict[str, str]] = {
     "c": {
         # in land_out, for CH4 emissions from GLOBIOM
@@ -36,14 +38,15 @@
     "t": dict(),
 }
 
-#: Replacements used in :meth:`collapse` after the 'variable' column is assembled.
-#: These are applied using :meth:`pandas.DataFrame.replace` with ``regex=True``; see
-#: the documentation of that method. For documentation of regular expressions, see
+#: Replacements used in :func:`collapse` after 'variable' labels are constructed. These
+#: are applied using :meth:`pandas.DataFrame.replace` with ``regex=True``; see the
+#: documentation of that method. For documentation of regular expressions, see
 #: https://docs.python.org/3/library/re.html and https://regex101.com.
 #:
-#: .. todo:: These may be particular or idiosyncratic to a single "template". The
-#:    strings used to collapse multiple conceptual dimensions into the IAMC "variable"
-#:    column are known to vary in poorly-documented ways across these templates.
+#: .. todo:: These may be particular or idiosyncratic to a single 'template'. The
+#:    strings used to collapse multiple conceptual dimensions into the IAMC 'variable'
+#:    dimension are known to vary across these templates, in ways that are sometimes not
+#:    documented.
 #:
 #:    This setting is currently applied universally. To improve, specify a different
 #:    mapping with the replacements needed for each individual template, and load the
@@ -212,18 +215,19 @@ def collapse(df: pd.DataFrame, var=[]) -> pd.DataFrame:
     """Callback for the `collapse` argument to :meth:`~.Reporter.convert_pyam`.
 
     Replacements from :data:`REPLACE_DIMS` and :data:`REPLACE_VARS` are applied.
-    The dimensions listed in the `var` arguments are automatically dropped from the
-    returned :class:`pyam.IamDataFrame`. If ``var[0]`` contains the word "emissions",
-    then :meth:`collapse_gwp_info` is invoked.
+    The dimensions listed in the `var` argument are automatically dropped from the
+    returned :class:`pyam.IamDataFrame`. If :py:`var[0]` contains the word "emissions",
+    then :func:`collapse_gwp_info` is invoked.
 
     Adapted from :func:`genno.compat.pyam.collapse`.
 
     Parameters
     ----------
     var : list of str, optional
-        Strings or dimensions to concatenate to the 'Variable' column. The first of
-        these is usually a string value used to populate the column. These are joined
-        using the pipe ('|') character.
+        Strings or dimensions to concatenate to a 'variable' string. The first of these
+        usually a :class:`str` used to populate the column; others may be fixed strings
+        or the IDs of dimensions in the input data. The components are joined using the
+        pipe ('|') character.
 
     See also
     --------
@@ -314,7 +318,25 @@ def copy_ts(rep: Reporter, other: str, filters: dict | None) -> Key:
 
 
 def add_replacements(dim: str, codes: Iterable[Code]) -> None:
-    """Update :data:`REPLACE_DIMS` for dimension `dim` with values from `codes`."""
+    """Update :data:`REPLACE_DIMS` for dimension `dim` with values from `codes`.
+
+    For every code in `codes` that has an annotation with the ID ``report``, the code
+    ID is mapped to the value of the annotation. For example, the following in one of
+    the :doc:`/pkg-data/codelists`:
+
+    .. code-block:: yaml
+
+       foo:
+         report: fOO
+
+       bar:
+         report: Baz
+
+       qux: {}  # No "report" annotation → no mapping
+
+    …results in entries :py:`{"foo": "fOO", "bar": "Baz"}` added to :data:`REPLACE_DIMS`
+    and used by :func:`collapse`.
+    """
     for code in codes:
         try:
             label = str(code.get_annotation(id="report").text)