Skip to content

Commit 98694c3

Browse files
juliasloan25juliasloan25
authored
Merge pull request #706 from CliMA/js/table
add coupler summary table
2 parents cb5ce32 + c96101a commit 98694c3

File tree

58 files changed

+1027
-1166
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

58 files changed

+1027
-1166
lines changed

‎.buildkite/benchmarks/README.md‎

Lines changed: 47 additions & 0 deletions

‎.buildkite/benchmarks/pipeline.yml‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
agents:
2+
queue: clima
3+
slurm_time: 24:00:00
4+
modules: common
5+
6+
env:
7+
JULIA_NVTX_CALLBACKS: gc
8+
OPENBLAS_NUM_THREADS: 1
9+
OMPI_MCA_opal_warn_on_missing_libcuda: 0
10+
SLURM_KILL_BAD_EXIT: 1
11+
SLURM_GRES_FLAGS: "allow-task-sharing"
12+
BENCHMARK_CONFIG_PATH: "config/benchmark_configs"
13+
14+
steps:
15+
- label: "init :GPU:"
16+
key: "init_gpu_env"
17+
command:
18+
- echo "--- Instantiate experiments/AMIP"
19+
- julia --project=experiments/AMIP -e 'using Pkg; Pkg.instantiate(;verbose=true)'
20+
- julia --project=experiments/AMIP -e 'using Pkg; Pkg.precompile()'
21+
- julia --project=experiments/AMIP -e 'using Pkg; Pkg.status()'
22+
23+
- echo "--- Instantiate test env"
24+
- "julia --project=test/ -e 'using Pkg; Pkg.develop(path=\".\")'"
25+
- "julia --project=test/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
26+
- "julia --project=test/ -e 'using Pkg; Pkg.precompile()'"
27+
- "julia --project=test/ -e 'using Pkg; Pkg.status()'"
28+
29+
- echo "--- Download artifacts"
30+
- "julia --project=artifacts -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
31+
- "julia --project=artifacts -e 'using Pkg; Pkg.precompile()'"
32+
- "julia --project=artifacts -e 'using Pkg; Pkg.status()'"
33+
- "julia --project=artifacts artifacts/download_artifacts.jl"
34+
35+
agents:
36+
slurm_gpus: 1
37+
slurm_cpus_per_task: 8
38+
env:
39+
JULIA_NUM_PRECOMPILE_TASKS: 8
40+
JULIA_MAX_NUM_PRECOMPILE_FILES: 50
41+
42+
- wait
43+
44+
- group: "CPU benchmarks"
45+
steps:
46+
- label: "CPU ClimaAtmos with diagnostic EDMF"
47+
key: "climaatmos_diagedmf"
48+
command: "srun julia --color=yes --project=test/ test/component_model_tests/climaatmos_standalone/atmos_driver.jl --config_file $BENCHMARK_CONFIG_PATH/climaatmos_diagedmf.yml"
49+
artifact_paths: "experiments/AMIP/output/climaatmos/climaatmos_diagedmf_artifacts/*"
50+
env:
51+
BUILD_HISTORY_HANDLE: ""
52+
CLIMACOMMS_DEVICE: "CPU"
53+
agents:
54+
slurm_ntasks_per_node: 64
55+
slurm_nodes: 1
56+
slurm_mem_per_cpu: 4GB
57+
58+
- label: "CPU AMIP with diagnostic EDMF"
59+
key: "amip_diagedmf"
60+
command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $BENCHMARK_CONFIG_PATH/amip_diagedmf.yml"
61+
artifact_paths: "experiments/AMIP/output/amip/amip_diagedmf_artifacts/*"
62+
env:
63+
BUILD_HISTORY_HANDLE: ""
64+
CLIMACOMMS_DEVICE: "CPU"
65+
agents:
66+
slurm_ntasks_per_node: 64
67+
slurm_nodes: 1
68+
slurm_mem_per_cpu: 4GB
69+
70+
- group: "GPU benchmarks"
71+
steps:
72+
- label: "GPU ClimaAtmos with diagnostic EDMF"
73+
key: "gpu_climaatmos_diagedmf"
74+
command: "srun julia --threads=3 --color=yes --project=test/ test/component_model_tests/climaatmos_standalone/atmos_driver.jl --config_file $BENCHMARK_CONFIG_PATH/gpu_climaatmos_diagedmf.yml"
75+
artifact_paths: "experiments/AMIP/output/climaatmos/gpu_climaatmos_diagedmf_artifacts/*"
76+
agents:
77+
slurm_gpus_per_task: 1
78+
slurm_cpus_per_task: 4
79+
slurm_ntasks: 4
80+
slurm_mem: 16GB
81+
82+
- label: "GPU AMIP with diagnostic EDMF"
83+
key: "gpu_amip_diagedmf"
84+
command: "srun julia --threads=3 --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $BENCHMARK_CONFIG_PATH/gpu_amip_diagedmf.yml"
85+
artifact_paths: "experiments/AMIP/output/amip/gpu_amip_diagedmf_artifacts/*"
86+
agents:
87+
slurm_gpus_per_task: 1
88+
slurm_cpus_per_task: 4
89+
slurm_ntasks: 4
90+
slurm_mem: 16GB
91+
92+
- group: "Generate output table"
93+
steps:
94+
- label: "Compare AMIP/Atmos-only with diagnostic EDMF"
95+
key: "compare_amip_climaatmos_amip_diagedmf"
96+
command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/user_io/benchmarks.jl --cpu_run_name_coupled amip_diagedmf --cpu_run_name_atmos climaatmos_diagedmf --gpu_run_name_coupled gpu_amip_diagedmf --gpu_run_name_atmos gpu_climaatmos_diagedmf --mode_name amip --build_id $BUILDKITE_BUILD_NUMBER"
97+
artifact_paths: "experiments/AMIP/output/compare_amip_climaatmos_amip_diagedmf/*"
98+
depends_on:
99+
- "climaatmos_diagedmf"
100+
- "amip_diagedmf"
101+
- "gpu_climaatmos_diagedmf"
102+
- "gpu_amip_diagedmf"
103+
104+
- label: ":envelope: Slack report: CPU/GPU AMIP/Atmos-only table"
105+
depends_on:
106+
- "compare_amip_climaatmos_amip_diagedmf"
107+
command:
108+
- slack-upload -c "#coupler-report" -f experiments/AMIP/output/compare_amip_climaatmos_amip_diagedmf/table.txt -m txt -n compare_amip_climaatmos_amip_diagedmf_table -x "Coupler CPU/GPU Comparison Table"

‎.buildkite/longruns/pipeline.yml‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ env:
1111
JULIA_MAX_NUM_PRECOMPILE_FILES: 100
1212
GKSwstype: 100
1313
SLURM_KILL_BAD_EXIT: 1
14-
1514
CONFIG_PATH: "config/longrun_configs"
1615

1716
timeout_in_minutes: 1440
@@ -300,11 +299,11 @@ steps:
300299

301300
# DYAMOND AMIP: 1 day (convection resolving)
302301
- label: "GPU AMIP SUPERFINE: dyamond_target"
303-
key: "gpu_dyamond_target"
302+
key: "gpu_longrun_amip_dyamond"
304303
command:
305304
- echo "--- Run simulation"
306-
- "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_dyamond_target.yml"
307-
artifact_paths: "experiments/AMIP/output/amip/gpu_dyamond_target_artifacts/*"
305+
- "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_longrun_amip_dyamond.yml"
306+
artifact_paths: "experiments/AMIP/output/amip/gpu_longrun_amip_dyamond_artifacts/*"
308307
agents:
309308
queue: clima
310309
slurm_mem: 20GB

‎.buildkite/pipeline.yml‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,8 @@ env:
1313
GKSwstype: 100
1414
SLURM_KILL_BAD_EXIT: 1
1515

16-
CONFIG_PATH: "config/model_configs"
16+
CONFIG_PATH: "config/ci_configs"
1717
PERF_CONFIG_PATH: "config/perf_configs"
18-
MPI_CONFIG_PATH: "config/mpi_configs"
1918

2019
timeout_in_minutes: 240
2120

@@ -81,7 +80,7 @@ steps:
8180
steps:
8281
- label: "MPI Regridder unit tests"
8382
key: "regridder_mpi_tests"
84-
command: "srun julia --color=yes --project=test/ test/mpi_tests/regridder_mpi_tests.jl --config_file $MPI_CONFIG_PATH/regridder_mpi.yml"
83+
command: "srun julia --color=yes --project=test/ test/mpi_tests/regridder_mpi_tests.jl --config_file $CONFIG_PATH/regridder_mpi.yml"
8584
timeout_in_minutes: 20
8685
env:
8786
CLIMACORE_DISTRIBUTED: "MPI"

‎Project.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ authors = ["CliMA Contributors <[email protected]>"]
44
version = "0.0.1"
55

66
[deps]
7+
CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
78
ClimaComms = "3a4d1b5c-c61d-41fd-a00a-5873ba7a1b0d"
89
ClimaCore = "d414da3d-4745-48bb-8d80-42e94e092884"
910
ClimaCoreTempestRemap = "d934ef94-cdd4-4710-83d6-720549644b70"
@@ -23,6 +24,7 @@ Thermodynamics = "b60c26fb-14c3-4610-9d3e-2d17fe7ff00c"
2324
ClimaComms = "0.5.6"
2425
ClimaCore = "0.13"
2526
ClimaCoreTempestRemap = "0.3"
27+
CUDA = "5"
2628
Dates = "1"
2729
DocStringExtensions = "0.8, 0.9"
2830
JLD2 = "0.4"

‎config/benchmark_configs/amip_diagedmf.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
FLOAT_TYPE: "Float32"
2+
anim: false
3+
atmos_config_file: "config/benchmark_configs/climaatmos_diagedmf.yml"
4+
atmos_config_repo: "ClimaCoupler"
5+
dt_cpl: 120
6+
dt_save_state_to_disk: "Inf"
7+
dt_save_to_sol: "Inf"
8+
energy_check: false
9+
job_id: "amip_diagedmf"
10+
land_albedo_type: "map_temporal"
11+
mode_name: "amip"
12+
mono_surface: false
13+
monthly_checkpoint: false
14+
run_name: "amip_diagedmf"
15+
start_date: "19790301"
16+
t_end: "12hours"
17+
turb_flux_partition: "CombinedStateFluxes"
18+
use_coupler_diagnostics: false

‎config/benchmark_configs/climaatmos_diagedmf.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
FLOAT_TYPE: "Float32"
2+
approximate_linear_solve_iters: 2
3+
dt: 120secs
4+
dt_cloud_fraction: 1hours
5+
dt_rad: 1hours
6+
dt_save_state_to_disk: "Inf"
7+
dt_save_to_sol: "Inf"
8+
dz_bottom: 30.0
9+
dz_top: 3000.0
10+
edmfx_detr_model: "Generalized"
11+
edmfx_entr_model: "Generalized"
12+
edmfx_nh_pressure: true
13+
edmfx_sgs_diffusive_flux: true
14+
edmfx_sgs_mass_flux: true
15+
edmfx_upwinding: first_order
16+
h_elem: 30
17+
idealized_insolation: false
18+
implicit_diffusion: true
19+
job_id: "climaatmos_diagedmf"
20+
moist: equil
21+
output_default_diagnostics: false
22+
precip_model: 0M
23+
prognostic_tke: true
24+
rad: allskywithclear
25+
surface_setup: DefaultMoninObukhov
26+
t_end: 12hours
27+
toml: [toml/diagnostic_edmfx_box.toml]
28+
turbconv: diagnostic_edmfx
29+
z_elem: 63
30+
z_max: 55000.0

‎config/benchmark_configs/gpu_amip_diagedmf.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
FLOAT_TYPE: "Float32"
2+
anim: false
3+
atmos_config_file: "config/benchmark_configs/gpu_climaatmos_diagedmf.yml"
4+
atmos_config_repo: "ClimaCoupler"
5+
dt_cpl: 120
6+
dt_save_state_to_disk: "Inf"
7+
dt_save_to_sol: "Inf"
8+
energy_check: false
9+
job_id: "gpu_amip_diagedmf"
10+
land_albedo_type: "map_temporal"
11+
mode_name: "amip"
12+
mono_surface: false
13+
monthly_checkpoint: false
14+
run_name: "gpu_amip_diagedmf"
15+
start_date: "19790301"
16+
t_end: "12hours"
17+
turb_flux_partition: "CombinedStateFluxes"
18+
use_coupler_diagnostics: false

‎config/benchmark_configs/gpu_climaatmos_diagedmf.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
FLOAT_TYPE: "Float32"
2+
approximate_linear_solve_iters: 2
3+
dt: 120secs
4+
dt_cloud_fraction: 1hours
5+
dt_rad: 1hours
6+
dt_save_state_to_disk: "Inf"
7+
dt_save_to_sol: "Inf"
8+
dz_bottom: 30.0
9+
dz_top: 3000.0
10+
edmfx_detr_model: "Generalized"
11+
edmfx_entr_model: "Generalized"
12+
edmfx_nh_pressure: true
13+
edmfx_sgs_diffusive_flux: true
14+
edmfx_sgs_mass_flux: true
15+
edmfx_upwinding: first_order
16+
h_elem: 30
17+
idealized_insolation: false
18+
implicit_diffusion: true
19+
job_id: "gpu_climaatmos_diagedmf"
20+
moist: equil
21+
output_default_diagnostics: false
22+
precip_model: 0M
23+
prognostic_tke: true
24+
rad: allskywithclear
25+
surface_setup: DefaultMoninObukhov
26+
t_end: 12hours
27+
toml: [toml/diagnostic_edmfx_box.toml]
28+
turbconv: diagnostic_edmfx
29+
z_elem: 63
30+
z_max: 55000.0

config/model_configs/coarse_mpi_n2.yml renamed to config/ci_configs/coarse_mpi_n2.yml

File renamed without changes.

0 commit comments

Comments
 (0)