Skip to content

Commit 8b23710

Browse files
lewisjaredlewisjared
authored
Merge pull request #534 from Climate-REF/445-esmvaltool-dimensions
Add file dimensions to all ESMValTool diagnostics
2 parents 3b5c967 + 488353f commit 8b23710

37 files changed

Lines changed: 876 additions & 158 deletions

File tree

‎changelog/534.feature.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Added dimensions to files produced by ESMValTool diagnostics.

‎packages/climate-ref-core/src/climate_ref_core/diagnostics.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
from climate_ref_core.constraints import GroupConstraint
1414
from climate_ref_core.datasets import ExecutionDatasetCollection, FacetFilter, SourceDatasetType
1515
from climate_ref_core.metric_values import SeriesMetricValue
16-
from climate_ref_core.metric_values.typing import SeriesDefinition
16+
from climate_ref_core.metric_values.typing import FileDefinition, SeriesDefinition
1717
from climate_ref_core.pycmec.metric import CMECMetric
1818
from climate_ref_core.pycmec.output import CMECOutput
1919

@@ -528,6 +528,7 @@ class Diagnostic(AbstractDiagnostic):
528528
"""
529529

530530
series: Sequence[SeriesDefinition] = tuple()
531+
files: Sequence[FileDefinition] = tuple()
531532
test_data_spec: TestDataSpecification | None = None
532533

533534
def __init__(self) -> None:

‎packages/climate-ref-core/src/climate_ref_core/metric_values/typing.py‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,20 +9,26 @@
99
Value = float | int
1010

1111

12-
class SeriesDefinition(BaseModel):
12+
class FileDefinition(BaseModel):
1313
"""
14-
A definition of a 1-d array with an associated index and additional dimensions.
14+
A definition of an output file with associated additional dimensions.
1515
"""
1616

1717
file_pattern: str
1818
"""A glob pattern to match files that contain the series values."""
1919

20-
sel: dict[str, Any] | None = None
21-
"""A dictionary of selection criteria to apply with :meth:`xarray.Dataset.sel` after loading the file."""
22-
2320
dimensions: dict[str, str]
2421
"""Key, value pairs that identify the dimensions of the metric."""
2522

23+
24+
class SeriesDefinition(FileDefinition):
25+
"""
26+
A definition of a 1-d array with an associated index and additional dimensions.
27+
"""
28+
29+
sel: dict[str, Any] | None = None
30+
"""A dictionary of selection criteria to apply with :meth:`xarray.Dataset.sel` after loading the file."""
31+
2632
values_name: str
2733
"""The name of the variable in the file that contains the values of the series."""
2834

‎packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/base.py‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,15 @@ def build_execution_result(
234234
for filename in metadata:
235235
caption = metadata[filename].get("caption", "")
236236
relative_path = definition.as_relative_path(filename)
237+
for file_def in (*definition.diagnostic.files, *definition.diagnostic.series):
238+
if fnmatch.fnmatch(
239+
str(relative_path),
240+
f"executions/*/{file_def.file_pattern.format(**input_selectors)}",
241+
):
242+
dimensions = file_def.dimensions
243+
break
244+
else:
245+
dimensions = {}
237246
if relative_path.suffix in plot_suffixes:
238247
key = OutputCV.PLOTS.value
239248
else:
@@ -242,10 +251,15 @@ def build_execution_result(
242251
OutputCV.FILENAME.value: f"{relative_path}",
243252
OutputCV.LONG_NAME.value: caption,
244253
OutputCV.DESCRIPTION.value: "",
254+
OutputCV.DIMENSIONS.value: dimensions,
245255
}
246256
series.extend(
247257
self._extract_series_from_file(
248-
definition, filename, relative_path, caption=caption, input_selectors=input_selectors
258+
definition,
259+
filename,
260+
relative_path,
261+
caption=caption,
262+
input_selectors=input_selectors,
249263
)
250264
)
251265

‎packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/climate_at_global_warming_levels.py‎

Lines changed: 64 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,21 @@
1-
import pandas
1+
from pathlib import Path
2+
3+
import pandas as pd
24

35
from climate_ref_core.constraints import (
46
AddSupplementaryDataset,
57
PartialDateTime,
68
RequireFacets,
79
RequireTimerange,
810
)
9-
from climate_ref_core.datasets import FacetFilter, SourceDatasetType
11+
from climate_ref_core.datasets import ExecutionDatasetCollection, FacetFilter, SourceDatasetType
1012
from climate_ref_core.diagnostics import DataRequirement
13+
from climate_ref_core.metric_values.typing import FileDefinition
14+
from climate_ref_core.pycmec.metric import CMECMetric, MetricCV
15+
from climate_ref_core.pycmec.output import CMECOutput
1116
from climate_ref_esmvaltool.diagnostics.base import ESMValToolDiagnostic
1217
from climate_ref_esmvaltool.recipe import dataframe_to_recipe
13-
from climate_ref_esmvaltool.types import Recipe
18+
from climate_ref_esmvaltool.types import MetricBundleArgs, OutputBundleArgs, Recipe
1419

1520

1621
class ClimateAtGlobalWarmingLevels(ESMValToolDiagnostic):
@@ -78,12 +83,32 @@ class ClimateAtGlobalWarmingLevels(ESMValToolDiagnostic):
7883
),
7984
),
8085
)
81-
facets = ()
86+
facets = ("experiment_id", "global warming level", "metric")
87+
88+
files = tuple(
89+
FileDefinition(
90+
file_pattern=f"plots/gwl_mean_plots_{var_name}/plot_gwl_stats/*.png",
91+
dimensions={
92+
"statistic": "mean",
93+
"variable_id": var_name,
94+
},
95+
)
96+
for var_name in variables
97+
) + tuple(
98+
FileDefinition(
99+
file_pattern=f"work/gwl_mean_plots_{var_name}/plot_gwl_stats/*.nc",
100+
dimensions={
101+
"statistic": "mean",
102+
"variable_id": var_name,
103+
},
104+
)
105+
for var_name in variables
106+
)
82107

83108
@staticmethod
84109
def update_recipe(
85110
recipe: Recipe,
86-
input_files: dict[SourceDatasetType, pandas.DataFrame],
111+
input_files: dict[SourceDatasetType, pd.DataFrame],
87112
) -> None:
88113
"""Update the recipe."""
89114
# Set up the datasets
@@ -124,3 +149,37 @@ def update_recipe(
124149
"preprocessor": "multi_model_gwl_stats",
125150
"timerange": "2000/2100",
126151
}
152+
153+
@staticmethod
154+
def format_result(
155+
result_dir: Path,
156+
execution_dataset: ExecutionDatasetCollection,
157+
metric_args: MetricBundleArgs,
158+
output_args: OutputBundleArgs,
159+
) -> tuple[CMECMetric, CMECOutput]:
160+
"""Format the result."""
161+
metric_args[MetricCV.DIMENSIONS.value] = {
162+
"json_structure": [
163+
"global warming level",
164+
"metric",
165+
],
166+
"global warming level": {},
167+
"metric": {"exceedance_year": {}},
168+
}
169+
170+
df = pd.read_csv(
171+
result_dir
172+
/ "work"
173+
/ "calculate_gwl_exceedance_years"
174+
/ "gwl_exceedance_calculation"
175+
/ "GWL_exceedance_years.csv"
176+
)
177+
for row in df.itertuples(index=False):
178+
gwl = str(row.GWL)
179+
if gwl not in metric_args[MetricCV.DIMENSIONS.value]["global warming level"]:
180+
metric_args[MetricCV.DIMENSIONS.value]["global warming level"][gwl] = {}
181+
metric_args[MetricCV.RESULTS.value][gwl] = {
182+
"exceedance_year": int(str(row.Exceedance_Year)),
183+
}
184+
185+
return CMECMetric.model_validate(metric_args), CMECOutput.model_validate(output_args)

‎packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/climate_drivers_for_fire.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
)
99
from climate_ref_core.datasets import FacetFilter, SourceDatasetType
1010
from climate_ref_core.diagnostics import DataRequirement
11+
from climate_ref_core.metric_values.typing import FileDefinition
1112
from climate_ref_esmvaltool.diagnostics.base import ESMValToolDiagnostic
1213
from climate_ref_esmvaltool.recipe import dataframe_to_recipe
1314
from climate_ref_esmvaltool.types import Recipe
@@ -73,6 +74,20 @@ class ClimateDriversForFire(ESMValToolDiagnostic):
7374
),
7475
)
7576
facets = ()
77+
files = (
78+
FileDefinition(
79+
file_pattern="plots/fire_evaluation/fire_evaluation/burnt_fraction_*.png",
80+
dimensions={"statistic": "burnt fraction"},
81+
),
82+
FileDefinition(
83+
file_pattern="plots/fire_evaluation/fire_evaluation/fire_weather_control_*.png",
84+
dimensions={"statistic": "fire weather control"},
85+
),
86+
FileDefinition(
87+
file_pattern="plots/fire_evaluation/fire_evaluation/fuel_load_continuity_control_*.png",
88+
dimensions={"statistic": "fuel load continuity control"},
89+
),
90+
)
7691

7792
@staticmethod
7893
def update_recipe(

‎packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/cloud_radiative_effects.py‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
)
1010
from climate_ref_core.datasets import FacetFilter, SourceDatasetType
1111
from climate_ref_core.diagnostics import DataRequirement
12-
from climate_ref_core.metric_values.typing import SeriesDefinition
12+
from climate_ref_core.metric_values.typing import FileDefinition, SeriesDefinition
1313
from climate_ref_esmvaltool.diagnostics.base import ESMValToolDiagnostic
1414
from climate_ref_esmvaltool.recipe import dataframe_to_recipe
1515
from climate_ref_esmvaltool.types import Recipe
@@ -58,6 +58,29 @@ class CloudRadiativeEffects(ESMValToolDiagnostic):
5858
)
5959

6060
facets = ()
61+
files = (
62+
tuple(
63+
FileDefinition(
64+
file_pattern=f"plots/plot_profiles/plot/variable_vs_lat_{var_name}_*.png",
65+
dimensions={"variable_id": var_name, "statistic": "zonal mean"},
66+
)
67+
for var_name in ["lwcre", "swcre"]
68+
)
69+
+ tuple(
70+
FileDefinition(
71+
file_pattern=f"plots/plot_maps/plot/map_{var_name}_*.png",
72+
dimensions={"variable_id": var_name, "statistic": "climatology map"},
73+
)
74+
for var_name in ["lwcre", "swcre"]
75+
)
76+
+ tuple(
77+
FileDefinition(
78+
file_pattern=f"work/plot_maps/plot/map_{var_name}_*.nc",
79+
dimensions={"variable_id": var_name, "statistic": "climatology map"},
80+
)
81+
for var_name in ["lwcre", "swcre"]
82+
)
83+
)
6184
series = tuple(
6285
SeriesDefinition(
6386
file_pattern=f"plot_profiles/plot/variable_vs_lat_{var_name}_*.nc",

‎packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/cloud_scatterplots.py‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
)
1111
from climate_ref_core.datasets import FacetFilter, SourceDatasetType
1212
from climate_ref_core.diagnostics import DataRequirement
13+
from climate_ref_core.metric_values.typing import FileDefinition
1314
from climate_ref_esmvaltool.diagnostics.base import ESMValToolDiagnostic
1415
from climate_ref_esmvaltool.recipe import dataframe_to_recipe
1516
from climate_ref_esmvaltool.types import Recipe
@@ -79,6 +80,16 @@ class CloudScatterplotCltSwcre(ESMValToolDiagnostic):
7980
facets = ()
8081
data_requirements = get_cmip6_data_requirements(("clt", "rsut", "rsutcs"))
8182
update_recipe = partial(update_recipe, var_x="clt", var_y="swcre")
83+
files = (
84+
FileDefinition(
85+
file_pattern="plots/plot_joint_clt_swcre_model/plot/png/*.png",
86+
dimensions={"statistic": "joint histogram of clt vs swcre"},
87+
),
88+
FileDefinition(
89+
file_pattern="work/plot_joint_clt_swcre_model/plot/*.nc",
90+
dimensions={"statistic": "joint histogram of clt vs swcre"},
91+
),
92+
)
8293

8394

8495
class CloudScatterplotClwviPr(ESMValToolDiagnostic):
@@ -92,6 +103,16 @@ class CloudScatterplotClwviPr(ESMValToolDiagnostic):
92103
facets = ()
93104
data_requirements = get_cmip6_data_requirements(("clwvi", "pr"))
94105
update_recipe = partial(update_recipe, var_x="clwvi", var_y="pr")
106+
files = (
107+
FileDefinition(
108+
file_pattern="plots/plot_joint_clwvi_pr_model/plot/png/*.png",
109+
dimensions={"statistic": "joint histogram of clwvi vs pr"},
110+
),
111+
FileDefinition(
112+
file_pattern="work/plot_joint_clwvi_pr_model/plot/*.nc",
113+
dimensions={"statistic": "joint histogram of clwvi vs pr"},
114+
),
115+
)
95116

96117

97118
class CloudScatterplotCliviLwcre(ESMValToolDiagnostic):
@@ -105,6 +126,16 @@ class CloudScatterplotCliviLwcre(ESMValToolDiagnostic):
105126
facets = ()
106127
data_requirements = get_cmip6_data_requirements(("clivi", "rlut", "rlutcs"))
107128
update_recipe = partial(update_recipe, var_x="clivi", var_y="lwcre")
129+
files = (
130+
FileDefinition(
131+
file_pattern="plots/plot_joint_clivi_lwcre_model/plot/png/*.png",
132+
dimensions={"statistic": "joint histogram of clivi vs lwcre"},
133+
),
134+
FileDefinition(
135+
file_pattern="work/plot_joint_clivi_lwcre_model/plot/*.nc",
136+
dimensions={"statistic": "joint histogram of clivi vs lwcre"},
137+
),
138+
)
108139

109140

110141
class CloudScatterplotCliTa(ESMValToolDiagnostic):
@@ -118,6 +149,16 @@ class CloudScatterplotCliTa(ESMValToolDiagnostic):
118149
facets = ()
119150
data_requirements = get_cmip6_data_requirements(("cli", "ta"))
120151
update_recipe = partial(update_recipe, var_x="cli", var_y="ta")
152+
files = (
153+
FileDefinition(
154+
file_pattern="plots/plot_joint_cli_ta_model/plot/png/*.png",
155+
dimensions={"statistic": "joint histogram of cli vs ta"},
156+
),
157+
FileDefinition(
158+
file_pattern="work/plot_joint_cli_ta_model/plot/*.nc",
159+
dimensions={"statistic": "joint histogram of cli vs ta"},
160+
),
161+
)
121162

122163

123164
class CloudScatterplotsReference(ESMValToolDiagnostic):
@@ -129,6 +170,40 @@ class CloudScatterplotsReference(ESMValToolDiagnostic):
129170
slug = "cloud-scatterplots-reference"
130171
base_recipe = "ref/recipe_ref_scatterplot.yml"
131172
facets = ()
173+
files = (
174+
FileDefinition(
175+
file_pattern="plots/plot_joint_cli_ta_ref/plot/png/*.png",
176+
dimensions={"statistic": "joint histogram of cli vs ta"},
177+
),
178+
FileDefinition(
179+
file_pattern="plots/plot_joint_clivi_lwcre_ref/plot/png/*.png",
180+
dimensions={"statistic": "joint histogram of clivi vs lwcre"},
181+
),
182+
FileDefinition(
183+
file_pattern="plots/plot_joint_clt_swcre_ref/plot/png/*.png",
184+
dimensions={"statistic": "joint histogram of clt vs swcre"},
185+
),
186+
FileDefinition(
187+
file_pattern="plots/plot_joint_clwvi_pr_ref/plot/png/*.png",
188+
dimensions={"statistic": "joint histogram of clwvi vs pr"},
189+
),
190+
FileDefinition(
191+
file_pattern="work/plot_joint_cli_ta_ref/plot/*.nc",
192+
dimensions={"statistic": "joint histogram of cli vs ta"},
193+
),
194+
FileDefinition(
195+
file_pattern="work/plot_joint_clivi_lwcre_ref/plot/*.nc",
196+
dimensions={"statistic": "joint histogram of clivi vs lwcre"},
197+
),
198+
FileDefinition(
199+
file_pattern="work/plot_joint_clt_swcre_ref/plot/*.nc",
200+
dimensions={"statistic": "joint histogram of clt vs swcre"},
201+
),
202+
FileDefinition(
203+
file_pattern="work/plot_joint_clwvi_pr_ref/plot/*.nc",
204+
dimensions={"statistic": "joint histogram of clwvi vs pr"},
205+
),
206+
)
132207
data_requirements = (
133208
DataRequirement(
134209
source_type=SourceDatasetType.obs4MIPs,

‎packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/ecs.py‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
)
1313
from climate_ref_core.datasets import ExecutionDatasetCollection, FacetFilter, SourceDatasetType
1414
from climate_ref_core.diagnostics import DataRequirement
15-
from climate_ref_core.metric_values.typing import SeriesDefinition
15+
from climate_ref_core.metric_values.typing import FileDefinition, SeriesDefinition
1616
from climate_ref_core.pycmec.metric import CMECMetric, MetricCV
1717
from climate_ref_core.pycmec.output import CMECOutput
1818
from climate_ref_esmvaltool.diagnostics.base import ESMValToolDiagnostic, fillvalues_to_nan
@@ -74,6 +74,20 @@ class EquilibriumClimateSensitivity(ESMValToolDiagnostic):
7474
attributes=[],
7575
),
7676
)
77+
files = (
78+
FileDefinition(
79+
file_pattern="plots/ecs/calculate/*.png",
80+
dimensions={"statistic": "global annual mean anomaly of rtnt vs tas"},
81+
),
82+
FileDefinition(
83+
file_pattern="work/ecs/calculate/ecs.nc",
84+
dimensions={"metric": "ecs"},
85+
),
86+
FileDefinition(
87+
file_pattern="work/ecs/calculate/lambda.nc",
88+
dimensions={"metric": "lambda"},
89+
),
90+
)
7791

7892
@staticmethod
7993
def update_recipe(

0 commit comments

Comments
 (0)