Merge pull request #772 from CliMA/js/common-data

fix CommonDataModel bug

Author: juliasloan25

artifacts/artifact_funcs.jl

@@ -61,23 +61,24 @@ function pr_obs_data_path()
 end
 
 """
-    artifact_data(datapath_full, name)
+    artifact_data(datapath_full, filename)
 
 Returns input dataset at datapath_full
 """
-function artifact_data(datapath_full, name)
-    datafile_truncated = joinpath(datapath_full, string(name, ".nc"))
+function artifact_data(datapath_full, filename)
+    datafile_truncated = joinpath(datapath_full, string(lowercase(filename), ".nc"))
     return datafile_truncated
 end
 
 """
-    artifact_data(datapath_full, name, datapath_trunc, date0, time_start, time_end, comms_ctx)
+    artifact_data(datapath_full, filename, varname, datapath_trunc, date0, t_start, t_end, comms_ctx)
 
-Truncates given data set, and constructs a new dataset containing only the dates needed and stores it in datapath_trunc
+Truncates given data set, and constructs a new dataset containing only
+the dates needed and stores it in datapath_trunc
 """
-function artifact_data(datapath_full, name, datapath_trunc, date0, time_start, time_end, comms_ctx)
-    datafile = joinpath(datapath_full, string(name, ".nc"))
+function artifact_data(datapath_full, filename, varname, datapath_trunc, date0, t_start, t_end, comms_ctx)
+    datafile = joinpath(datapath_full, string(lowercase(filename), ".nc"))
     datafile_truncated =
-        Regridder.truncate_dataset(datafile, name, datapath_trunc, date0, time_start, time_end, comms_ctx)
+        Regridder.truncate_dataset(datafile, filename, varname, datapath_trunc, date0, t_start, t_end, comms_ctx)
     return datafile_truncated
 end

experiments/AMIP/Manifest.toml

@@ -478,9 +478,9 @@ version = "0.12.11"
 
 [[deps.CommonDataModel]]
 deps = ["CFTime", "DataStructures", "Dates", "Preferences", "Printf", "Statistics"]
-git-tree-sha1 = "d7d7b58e149f19c322840a50d1bc20e8c23addb4"
+git-tree-sha1 = "d6fb5bf939a2753c74984b11434ea25d6c397a58"
 uuid = "1fbeeb36-5f17-413c-809b-666fb144f157"
-version = "0.3.5"
+version = "0.3.6"
 
 [[deps.CommonSolve]]
 git-tree-sha1 = "0eee5eb66b1cf62cd6ad1b460238e60e4b09400c"

experiments/AMIP/coupler_driver.jl

@@ -163,9 +163,9 @@ original sources.
 =#
 
 include(joinpath(pkgdir(ClimaCoupler), "artifacts", "artifact_funcs.jl"))
-sst_data = artifact_data(sst_dataset_path(), "sst", dir_paths.regrid, date0, t_start, t_end, comms_ctx)
-sic_data = artifact_data(sic_dataset_path(), "sic", dir_paths.regrid, date0, t_start, t_end, comms_ctx)
-co2_data = artifact_data(co2_dataset_path(), "mauna_loa_co2", dir_paths.regrid, date0, t_start, t_end, comms_ctx)
+sst_data = artifact_data(sst_dataset_path(), "sst", "SST", dir_paths.regrid, date0, t_start, t_end, comms_ctx)
+sic_data = artifact_data(sic_dataset_path(), "sic", "SEAICE", dir_paths.regrid, date0, t_start, t_end, comms_ctx)
+co2_data = artifact_data(co2_dataset_path(), "mauna_loa_co2", "co2", dir_paths.regrid, date0, t_start, t_end, comms_ctx)
 land_mask_data = artifact_data(mask_dataset_path(), "seamask")
 
 #=

experiments/ClimaCore/Manifest.toml

@@ -2,7 +2,7 @@
 
 julia_version = "1.10.0"
 manifest_format = "2.0"
-project_hash = "01a9ebd3f7699789bd6ed48bc9c7518a6ff5227e"
+project_hash = "89b3a4b3cb7163830203310d5b8b73e1b6be2a9d"
 
 [[deps.ADTypes]]
 git-tree-sha1 = "016833eb52ba2d6bea9fcb50ca295980e728ee24"
@@ -154,12 +154,6 @@ git-tree-sha1 = "389ad5c84de1ae7cf0e28e381131c98ea87d54fc"
 uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82"
 version = "0.5.0"
 
-[[deps.CFTime]]
-deps = ["Dates", "Printf"]
-git-tree-sha1 = "5afb5c5ba2688ca43a9ad2e5a91cbb93921ccfa1"
-uuid = "179af706-886a-5703-950a-314cd64e0468"
-version = "0.1.3"
-
 [[deps.CPUSummary]]
 deps = ["CpuId", "IfElse", "PrecompileTools", "Static"]
 git-tree-sha1 = "601f7e7b3d36f18790e2caf83a882d88e9b71ff1"
@@ -279,12 +273,6 @@ git-tree-sha1 = "fc08e5930ee9a4e03f84bfb5211cb54e7769758a"
 uuid = "5ae59095-9a9b-59fe-a467-6f913c188581"
 version = "0.12.10"
 
-[[deps.CommonDataModel]]
-deps = ["CFTime", "DataStructures", "Dates", "Preferences", "Printf", "Statistics"]
-git-tree-sha1 = "d7d7b58e149f19c322840a50d1bc20e8c23addb4"
-uuid = "1fbeeb36-5f17-413c-809b-666fb144f157"
-version = "0.3.5"
-
 [[deps.CommonSolve]]
 git-tree-sha1 = "0eee5eb66b1cf62cd6ad1b460238e60e4b09400c"
 uuid = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2"

experiments/ClimaCore/Project.toml

@@ -2,7 +2,6 @@
 ClimaCore = "d414da3d-4745-48bb-8d80-42e94e092884"
 ClimaParams = "5c42b081-d73a-476f-9059-fd94b934656c"
 ClimaTimeSteppers = "595c0a79-7f3d-439a-bc5a-b232dc3bde79"
-CommonDataModel = "1fbeeb36-5f17-413c-809b-666fb144f157"
 FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
 IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953"
 JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
@@ -19,7 +18,6 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 [compat]
 ClimaParams = "0.10"
 ClimaTimeSteppers = "0.7"
-CommonDataModel = "=0.3.5"
 FileIO = "1.16"
 IntervalSets = "0.6, 0.7"
 LinearAlgebra = "1.0"

src/Regridder.jl

@@ -638,33 +638,62 @@ function cgll2latlonz(field; DIR = "cgll2latlonz_dir", nlat = 360, nlon = 720, c
 end
 
 """
-    truncate_dataset(datafile, name, datapath_trunc, date0, time_start, time_end, comms_ctx)
+    truncate_dataset(datafile, filename, varname, datapath_trunc, date0, t_start, t_end, comms_ctx)
 
-Truncates given data set, and constructs a new dataset containing only the dates that are used in the simulation
+Truncates given data set, and constructs a new dataset containing only
+the dates that are used in the simulation
 """
 function truncate_dataset(
     datafile,
-    name,
+    filename,
+    varname,
     datapath_trunc,
     date0,
-    time_start,
-    time_end,
+    t_start,
+    t_end,
     comms_ctx::ClimaComms.AbstractCommsContext,
 )
-    date_start = date0 + Dates.Second(time_start)
-    date_end = date0 + Dates.Second(time_start + time_end)
+    date_start = date0 + Dates.Second(t_start)
+    date_end = date0 + Dates.Second(t_start + t_end)
 
-    file_name = replace(string(name, "_truncated_data_", string(date_start), string(date_end), ".nc"), r":" => "")
-    datafile_truncated = joinpath(datapath_trunc, file_name)
+    filename_truncated = replace(
+        string(lowercase(filename), "_truncated_data_", string(date_start), string(date_end), ".nc"),
+        r":" => "",
+    )
+    datafile_truncated = joinpath(datapath_trunc, filename_truncated)
 
     if ClimaComms.iamroot(comms_ctx)
         ds = NCDatasets.NCDataset(datafile, "r")
         dates = ds["time"][:]
 
+        # Find the bounding indices of the dates we need
         (start_id, end_id) = find_idx_bounding_dates(dates, date_start, date_end)
 
+        var_truncated = NCDatasets.nomissing(NCDatasets.view(ds, time = start_id:end_id)[varname])
+
+        # Create new dataset to fill with truncated data
         ds_truncated = NCDatasets.NCDataset(datafile_truncated, "c")
-        ds_truncated = NCDatasets.write(ds_truncated, NCDatasets.view(ds, time = start_id:end_id))
+
+        # Keep all dimensions of original dataset (except for time, which we truncate)
+        ds_dim_names = NCDatasets.dimnames(ds[varname])
+        for dim_name in ds_dim_names
+            dim_name != "time" && NCDatasets.defDim(ds_truncated, dim_name, ds.dim[dim_name])
+        end
+        dates_truncated = dates[start_id:end_id]
+        NCDatasets.defDim(ds_truncated, "time", length(dates_truncated))
+        ds_truncated.attrib["title"] = ds.attrib["title"] * " (dates truncated)"
+
+        # Define dimension variables
+        for dim_name in ds_dim_names
+            if dim_name == "time"
+                var = NCDatasets.defVar(ds_truncated, dim_name, dates_truncated, (dim_name,))
+            else
+                var = NCDatasets.defVar(ds_truncated, dim_name, ds[dim_name][:], (dim_name,))
+            end
+        end
+
+        # Create variable of interest in new dataset, and fill with input dataset values
+        var = NCDatasets.defVar(ds_truncated, varname, var_truncated, ds_dim_names)
 
         close(ds)
         close(ds_truncated)

test/Project.toml

@@ -13,7 +13,6 @@ ClimaCoupler = "4ade58fe-a8da-486c-bd89-46df092ec0c7"
 ClimaLand = "08f4d4ce-cf43-44bb-ad95-9d2d5f413532"
 ClimaParams = "5c42b081-d73a-476f-9059-fd94b934656c"
 ClimaTimeSteppers = "595c0a79-7f3d-439a-bc5a-b232dc3bde79"
-CommonDataModel = "1fbeeb36-5f17-413c-809b-666fb144f157"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 GeoMakie = "db073c08-6b98-4ee5-b6a4-5efafb3259c6"
@@ -43,7 +42,6 @@ ArtifactWrappers = "0.2"
 ClimaAtmos = "0.23"
 ClimaLand = "0.11"
 ClimaParams = "0.10"
-CommonDataModel = "=0.3.5"
 Dates = "1"
 Downloads = "1"
 IntervalSets = "0.5, 0.6, 0.7"

test/regridder_tests.jl

@@ -312,40 +312,42 @@ for FT in (Float32, Float64)
         end
     end
 end
-# test dataset truncation 
+
+# test dataset truncation
 @testset "test dataset truncation" begin
-    # Get the original dataset set up 
+    # Get the original dataset set up
     include(joinpath(pkgdir(ClimaCoupler), "artifacts", "artifact_funcs.jl"))
     sst_data_all = joinpath(sst_dataset_path(), "sst.nc")
     ds = NCDatasets.NCDataset(sst_data_all, "r")
     dates = ds["time"][:]
     first_date = dates[1]
     last_date = last(dates)
 
-    # set up comms_ctx 
+    # set up comms_ctx
     device = ClimaComms.device()
-    comms_ctx = ClimaComms.context(device)
-    ClimaComms.init(comms_ctx)
+    comms_ctx_device = ClimaComms.context(device)
+    ClimaComms.init(comms_ctx_device)
 
-    # make path for truncated datasets 
+    # make path for truncated datasets
     COUPLER_OUTPUT_DIR = joinpath("experiments", "AMIP", "output", "tests")
     mkpath(COUPLER_OUTPUT_DIR)
 
     REGRID_DIR = joinpath(COUPLER_OUTPUT_DIR, "regrid_tmp", "")
     mkpath(REGRID_DIR)
 
-    # values for the truncations 
-    time_start = 0.0
-    time_end = 1.728e6
+    # values for the truncations
+    t_start = 0.0
+    t_end = 1.728e6
     date0test = ["18690101", "18700101", "19790228", "20220301", "20230101"]
     for date in date0test
         date0 = Dates.DateTime(date, Dates.dateformat"yyyymmdd")
-        sst_data = Regridder.truncate_dataset(sst_data_all, "test", REGRID_DIR, date0, time_start, time_end, comms_ctx)
+        sst_data =
+            Regridder.truncate_dataset(sst_data_all, "sst", "SST", REGRID_DIR, date0, t_start, t_end, comms_ctx_device)
         ds_truncated = NCDatasets.NCDataset(sst_data, "r")
         new_dates = ds_truncated["time"][:]
 
-        date_start = date0 + Dates.Second(time_start)
-        date_end = date0 + Dates.Second(time_start + time_end)
+        date_start = date0 + Dates.Second(t_start)
+        date_end = date0 + Dates.Second(t_start + t_end)
 
         # start date is before the first date of datafile
         if date_start < first_date
@@ -359,10 +361,10 @@ end
             @test new_dates[2] >= date_start
         end
 
-        # end date is before the first date of datafile 
+        # end date is before the first date of datafile
         if date_end < first_date
             @test last(new_dates) == first_date
-            # end date is after the last date of datafile 
+            # end date is after the last date of datafile
         elseif date_end > last_date
             @test last(new_dates) == last_date
             # end date is within the bounds of datafile
@@ -371,7 +373,7 @@ end
             @test new_dates[length(new_dates) - 1] <= date_end
         end
 
-        # check that truncation is indexing correctly 
+        # check that truncation is indexing correctly
         all_data = ds["SST"][:, :, :]
         new_data = ds_truncated["SST"][:, :, :]
         (start_id, end_id) = Regridder.find_idx_bounding_dates(dates, date_start, date_end)