From 83a8cd117652e68f665490265583c0aa71020128 Mon Sep 17 00:00:00 2001 From: Kristen Thyng Date: Fri, 25 Oct 2024 14:58:15 -0700 Subject: [PATCH 1/3] added CIOFS Fresh added CIOFS fresh as model open and it makes kerchunk file on the fly --- .../models/opendrift/opendrift.py | 17 +++++++++++++---- .../models/opendrift/utils.py | 4 ++-- particle_tracking_manager/the_manager.py | 1 + 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/particle_tracking_manager/models/opendrift/opendrift.py b/particle_tracking_manager/models/opendrift/opendrift.py index 1873997..5d8cabc 100644 --- a/particle_tracking_manager/models/opendrift/opendrift.py +++ b/particle_tracking_manager/models/opendrift/opendrift.py @@ -682,12 +682,17 @@ def run_add_reader( end = ( f"{self.end_time.year}_{str(self.end_time.dayofyear).zfill(4)}" ) - loc_local = make_ciofs_kerchunk(start=start, end=end) - # loc_local = make_ciofs_kerchunk(start="2005_0052", end="2005_0068") - # loc_local = "/mnt/vault/ciofs/HINDCAST/ciofs_kerchunk_2005.parq" - # loc_local = "/mnt/vault/ciofs/HINDCAST/ciofs_kerchunk.parq" + loc_local = make_ciofs_kerchunk(start=start, end=end, name="ciofs") loc_remote = "http://xpublish-ciofs.srv.axds.co/datasets/ciofs_hindcast/zarr/" + elif self.ocean_model == "CIOFSFRESH": + start = f"{self.start_time.year}_{str(self.start_time.dayofyear - 1).zfill(4)}" + end = ( + f"{self.end_time.year}_{str(self.end_time.dayofyear).zfill(4)}" + ) + loc_local = make_ciofs_kerchunk(start=start, end=end, name="ciofs_fresh") + loc_remote = None + elif self.ocean_model == "CIOFSOP": standard_name_mapping.update( @@ -727,6 +732,10 @@ def run_add_reader( # otherwise remote else: if ".nc" in loc_remote: + + if self.ocean_model == "CIOFSFRESH": + raise NotImplementedError + ds = xr.open_dataset( loc_remote, chunks={}, diff --git a/particle_tracking_manager/models/opendrift/utils.py b/particle_tracking_manager/models/opendrift/utils.py index 7c4443a..51bfbeb 100644 --- a/particle_tracking_manager/models/opendrift/utils.py +++ b/particle_tracking_manager/models/opendrift/utils.py @@ -10,7 +10,7 @@ from kerchunk.combine import MultiZarrToZarr -def make_ciofs_kerchunk(start, end): +def make_ciofs_kerchunk(start, end, name): """_summary_ Parameters @@ -25,7 +25,7 @@ def make_ciofs_kerchunk(start, end): _description_ """ - output_dir_single_files = "/home/kristen/projects/kerchunk_setup/ciofs" + output_dir_single_files = f"/home/kristen/projects/kerchunk_setup/{name}" fs2 = fsspec.filesystem("") # local file system to save final jsons to diff --git a/particle_tracking_manager/the_manager.py b/particle_tracking_manager/the_manager.py index 4dc6577..9a2d645 100644 --- a/particle_tracking_manager/the_manager.py +++ b/particle_tracking_manager/the_manager.py @@ -17,6 +17,7 @@ _KNOWN_MODELS = [ "NWGOA", "CIOFS", + "CIOFSFRESH", "CIOFSOP", ] From a4d584a245cdd6098b8c3768d70804c549b0448c Mon Sep 17 00:00:00 2001 From: Kristen Thyng Date: Fri, 25 Oct 2024 15:00:58 -0700 Subject: [PATCH 2/3] updated whats new --- docs/whats_new.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/whats_new.md b/docs/whats_new.md index 0039602..1856c94 100644 --- a/docs/whats_new.md +++ b/docs/whats_new.md @@ -1,5 +1,9 @@ # What's New +## v0.9.1 (October 25, 2024) + +* Added local model option of CIOFS Fresh for which kerchunk files also can be generated on the fly + ## v0.9.0 (July 26, 2024) * Added utilities to generate kerchunk files on the fly for the time period of the simulation length for CIOFS and NWGOA. This has majorly sped up CIOFS simulations and modestly sped up NWGOA simulations. From a2b3454f3b02bbcaf89606409652186f52a41f6e Mon Sep 17 00:00:00 2001 From: Kristen Thyng Date: Fri, 25 Oct 2024 15:02:47 -0700 Subject: [PATCH 3/3] lint --- particle_tracking_manager/models/opendrift/opendrift.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/particle_tracking_manager/models/opendrift/opendrift.py b/particle_tracking_manager/models/opendrift/opendrift.py index 5d8cabc..4c0b32a 100644 --- a/particle_tracking_manager/models/opendrift/opendrift.py +++ b/particle_tracking_manager/models/opendrift/opendrift.py @@ -690,7 +690,9 @@ def run_add_reader( end = ( f"{self.end_time.year}_{str(self.end_time.dayofyear).zfill(4)}" ) - loc_local = make_ciofs_kerchunk(start=start, end=end, name="ciofs_fresh") + loc_local = make_ciofs_kerchunk( + start=start, end=end, name="ciofs_fresh" + ) loc_remote = None elif self.ocean_model == "CIOFSOP":