From 0be5515684a9f9403ee684e50eca09e90955ce44 Mon Sep 17 00:00:00 2001 From: justin-richling Date: Wed, 15 May 2024 16:36:26 -0600 Subject: [PATCH 01/16] Add functionality for CAM-CHEM variables Add new argument in variable defaults file for constituents of variables that have changed in CAM-CHEM runs Implement check in derivation of time series files to first check if chem/aerosol constituents are in history file then check if this variable is in the list of CAM-CHEM --- lib/adf_diag.py | 100 +++++++++++++++++++++++------ lib/adf_variable_defaults.yaml | 16 +++-- lib/website_templates/adf_diag.css | 2 +- 3 files changed, 93 insertions(+), 25 deletions(-) diff --git a/lib/adf_diag.py b/lib/adf_diag.py index 9c57c1f36..ab5d1ca13 100644 --- a/lib/adf_diag.py +++ b/lib/adf_diag.py @@ -1,3 +1,4 @@ + """ Location of the "AdfDiag" object, which is used to store all relevant data and @@ -546,21 +547,75 @@ def call_ncrcat(cmd): diag_var_list += ["T"] #End aerosol calcs + #Initialize dictionary for derived var with needed list of constituents + constit_dict = {} for var in diag_var_list: + #Check if current variable is a derived quantity if var not in hist_file_var_list: vres = res.get(var, {}) + + #Initialiaze list for constituents + #NOTE: This is if the variable is NOT derivable but need + # an empty list as a check later + constit_list = [] + + #intialize boolean to check if variable is derivable + derive = False # assume it can't be derived and update if it can + + #Try and build variable from 'derivable_from' if "derivable_from" in vres: + derive = True constit_list = vres["derivable_from"] - for constit in constit_list: - if constit not in diag_var_list: - diag_var_list.append(constit) - vars_to_derive.append(var) - continue + #Check if variable is potentially part of a CAM-CHEM run + if (any(item not in hist_file_ds.data_vars for item in constit_list)) and (var in res["cam_chem_list"]): + #Set check to look for CAM-CHEM constituents list in variable defaults + get_cam_chem_constits = True + else: + get_cam_chem_constits = False + #End if + + #If this is a CAM-CHEM run, update constit_list + if get_cam_chem_constits: + print(f"Looks like this a CAM-CHEM run, checking constits for '{var}'") + if "derivable_from_cam_chem" in vres: + constit_list = vres['derivable_from_cam_chem'] + else: + derive = False + errmsg = f"\n Missing 'derivable_from_cam_chem' config argument for {var}." + errmsg += "\n\tPlease remove variable from ADF run or set appropriate" + errmsg += " argument in variable defaults yaml file." + print(errmsg) + #End if + #End if + + #Now check if this variable can be derived + if derive: + for constit in constit_list: + if constit not in diag_var_list: + diag_var_list.append(constit) + #Add variable to list to derive + vars_to_derive.append(var) + #Add constituent list to variable key in dictionary + constit_dict[var] = constit_list + continue + #End if + else: + errmsg = f"\n Missing 'derivable_from' config argument for {var}." + errmsg += "\n\tPlease remove variable from ADF run or set appropriate" + errmsg += " argument in variable defaults yaml file." + print(errmsg) + #continue + #End if 'derivable_from' + + #Lastly, raise error if the variable is not a derived quanitity but is also not + #in the history file(s) + if (not derive) and (not constit_list): msg = f"WARNING: {var} is not in the file {hist_files[0]}." msg += " No time series will be generated." print(msg) continue + #End if # Check if variable has a "lev" dimension according to first file: has_lev = bool("lev" in hist_file_ds[var].dims) @@ -646,7 +701,8 @@ def call_ncrcat(cmd): if vars_to_derive: self.derive_variables( - res=res, vars_to_derive=vars_to_derive, ts_dir=ts_dir[case_idx] + res=res, vars_to_derive=vars_to_derive, ts_dir=ts_dir[case_idx], + constit_dict=constit_dict ) # End with @@ -1020,7 +1076,7 @@ def setup_run_cvdp(self): ######### - def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, overwrite=None): + def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, constit_dict=None, overwrite=None): """ Derive variables acccording to steps given here. Since derivations will depend on the variable, each variable to derive will need its own set of steps below. @@ -1032,28 +1088,32 @@ def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, overwrite """ + #Loop through derived variables for var in vars_to_derive: print(f"\t - deriving time series for {var}") - #Check whether there are parts to derive from and if there is an associated equation - vres = res.get(var, {}) - if "derivable_from" in vres: - constit_list = vres['derivable_from'] - else: - print("WARNING: No constituents listed in defaults config file, moving on") - continue + #Grab list of constituents for this variable + constit_list = constit_dict[var] - #Grab all required time series files for derived var + #Grab all required time series files for derived variable constit_files = [] for constit in constit_list: - if glob.glob(os.path.join(ts_dir, f"*.{constit}.*.nc")): + #Check if the constituent file is present, if so add it to list + if glob.glob(os.path.join(ts_dir, f"*.{constit}.*.nc")): constit_files.append(glob.glob(os.path.join(ts_dir, f"*.{constit}.*"))[0]) - #Check if all the constituent files were found + #Check if all the necessary constituent files were found if len(constit_files) != len(constit_list): ermsg = f"Not all constituent files present; {var} cannot be calculated." ermsg += f" Please remove {var} from diag_var_list or find the relevant CAM files." print(ermsg) + #Add what's missing to debug log + dmsg = "create time series:" + dmsg += f"\n \t needed constituents for derivation of {var}:\n\t\t- {constit_list}\n" + dmsg += f" \t found constituent file(s) in {Path(constit_files[0]).parent}:\n" + dmsg += f"\t\t- {[Path(file).parts[-1] for file in constit_files if Path(file).is_file()]}" + self.debug_log(dmsg) + else: #Open a new dataset with all the constituent files/variables ds = xr.open_mfdataset(constit_files) @@ -1084,7 +1144,8 @@ def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, overwrite der_val.name = var ds[var] = der_val - #Aerosol Calculations - used for zonal plots + #Aerosol Calculations + #---------------------------------------------------------------------------------- #These will be multiplied by rho (density of dry air) ds_pmid_done = False ds_t_done = False @@ -1114,6 +1175,7 @@ def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, overwrite #Sulfate conversion factor if var == "SO4": ds[var] = ds[var]*(96./115.) + #---------------------------------------------------------------------------------- #Drop all constituents from final saved dataset #These are not necessary because they have their own time series files @@ -1157,4 +1219,4 @@ def my_formatwarning(msg, *args, **kwargs): return xr.open_dataset(fils[0]) #End if #End def -######## \ No newline at end of file +######## diff --git a/lib/adf_variable_defaults.yaml b/lib/adf_variable_defaults.yaml index b8ecaa1ee..1197b515f 100644 --- a/lib/adf_variable_defaults.yaml +++ b/lib/adf_variable_defaults.yaml @@ -78,6 +78,12 @@ default_ptypes: ["Tables","LatLon","LatLon_Vector","Zonal","Meridional", #Dry Air Gas Constant: Rgas: 287.04 #[J/K/Kg]=8.314/0.028965 +#+++++++++++++ +# CAM CHEM Variables +#+++++++++++++ +#List of variables for CAM-CHEM runs that have different constituents than regular CAM runs +cam_chem_list: ["SOA","SO4"] + #+++++++++++++ # Category: Microphysics #+++++++++++++ @@ -231,7 +237,8 @@ SO4: colorbar: label : '$\mu$g/m3' category: "Aerosols" - derivable_from: ["so4_a1", "so4_a2", "so4_a3", "so4_a5"] + derivable_from_cam_chem: ["so4_a1", "so4_a2", "so4_a3", "so4_a5"] + derivable_from: ["so4_a1", "so4_a2", "so4_a3"] SOA: colormap: "RdBu_r" @@ -244,6 +251,7 @@ SOA: label : '$\mu$g/m3' category: "Aerosols" derivable_from: ["soa_a1", "soa_a2"] + derivable_from_cam_chem: ["soa1_a1", "soa2_a1", "soa3_a1", "soa4_a1", "soa5_a1", "soa1_a2", "soa2_a2", "soa3_a2", "soa4_a2", "soa5_a2"] DUST: colormap: "RdBu_r" @@ -277,8 +285,6 @@ SeaSalt: category: "Aerosols" derivable_from: ["ncl_a1", "ncl_a2", "ncl_a3"] - - #+++++++++++++++++ # Category: Budget #+++++++++++++++++ @@ -374,7 +380,6 @@ H2SO4: N2O: category: "Composition" - #+++++++++++++++++ # Category: Clouds #+++++++++++++++++ @@ -700,7 +705,6 @@ OCNFRAC: LANDFRAC: category: "Surface variables" - #+++++++++++++++++ # Category: State #+++++++++++++++++ @@ -816,6 +820,7 @@ QRS: #+++++++++++++++++ # Category: TOA energy flux #+++++++++++++++++ + RESTOM: colormap: "RdBu_r" contour_levels_range: [-100, 100, 5] @@ -1108,6 +1113,7 @@ OMEGAT: #++++++++++++++ # Category: TEM #++++++++++++++ + uzm: ylim: [1e3,1] units: m s-1 diff --git a/lib/website_templates/adf_diag.css b/lib/website_templates/adf_diag.css index 291f227aa..a2e146dc7 100644 --- a/lib/website_templates/adf_diag.css +++ b/lib/website_templates/adf_diag.css @@ -261,7 +261,7 @@ table.dataframe thead th{ display: grid; column-gap: 50px; row-gap: 50px; - grid-template-columns: repeat(4, auto); + grid-template-columns: repeat(3, auto); background-color: #e4eef0; padding: 85px; } From ed32ce89c2f45af601641bcc58f29fcac66a25f2 Mon Sep 17 00:00:00 2001 From: justin-richling Date: Wed, 15 May 2024 16:37:41 -0600 Subject: [PATCH 02/16] Update adf_diag.py --- lib/adf_diag.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/adf_diag.py b/lib/adf_diag.py index ab5d1ca13..58aa526b8 100644 --- a/lib/adf_diag.py +++ b/lib/adf_diag.py @@ -576,7 +576,7 @@ def call_ncrcat(cmd): #If this is a CAM-CHEM run, update constit_list if get_cam_chem_constits: - print(f"Looks like this a CAM-CHEM run, checking constits for '{var}'") + print(f"Looks like this a CAM-CHEM run, checking constituents for '{var}'") if "derivable_from_cam_chem" in vres: constit_list = vres['derivable_from_cam_chem'] else: @@ -605,7 +605,6 @@ def call_ncrcat(cmd): errmsg += "\n\tPlease remove variable from ADF run or set appropriate" errmsg += " argument in variable defaults yaml file." print(errmsg) - #continue #End if 'derivable_from' #Lastly, raise error if the variable is not a derived quanitity but is also not From a4a0773543a32a386dd966b8c020f4fcc1913128 Mon Sep 17 00:00:00 2001 From: justin-richling Date: Wed, 15 May 2024 16:54:25 -0600 Subject: [PATCH 03/16] github fixes --- lib/adf_diag.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/lib/adf_diag.py b/lib/adf_diag.py index 58aa526b8..82939e3f5 100644 --- a/lib/adf_diag.py +++ b/lib/adf_diag.py @@ -556,9 +556,9 @@ def call_ncrcat(cmd): #Initialiaze list for constituents #NOTE: This is if the variable is NOT derivable but need - # an empty list as a check later + # an empty list as a check later constit_list = [] - + #intialize boolean to check if variable is derivable derive = False # assume it can't be derived and update if it can @@ -576,14 +576,16 @@ def call_ncrcat(cmd): #If this is a CAM-CHEM run, update constit_list if get_cam_chem_constits: - print(f"Looks like this a CAM-CHEM run, checking constituents for '{var}'") + print(f"Looks like this a CAM-CHEM run,") + print(f" checking constituents for '{var}'") if "derivable_from_cam_chem" in vres: constit_list = vres['derivable_from_cam_chem'] else: derive = False - errmsg = f"\n Missing 'derivable_from_cam_chem' config argument for {var}." - errmsg += "\n\tPlease remove variable from ADF run or set appropriate" - errmsg += " argument in variable defaults yaml file." + errmsg = f"\n Missing 'derivable_from_cam_chem' " + errmsg += f"config argument for {var}." + errmsg += "\n\tPlease remove variable from ADF run or set" + errmsg += " appropriate argument in variable defaults yaml file." print(errmsg) #End if #End if @@ -1075,7 +1077,8 @@ def setup_run_cvdp(self): ######### - def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, constit_dict=None, overwrite=None): + def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, + constit_dict=None, overwrite=None): """ Derive variables acccording to steps given here. Since derivations will depend on the variable, each variable to derive will need its own set of steps below. @@ -1098,7 +1101,7 @@ def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, constit_d constit_files = [] for constit in constit_list: #Check if the constituent file is present, if so add it to list - if glob.glob(os.path.join(ts_dir, f"*.{constit}.*.nc")): + if glob.glob(os.path.join(ts_dir, f"*.{constit}.*.nc")): constit_files.append(glob.glob(os.path.join(ts_dir, f"*.{constit}.*"))[0]) #Check if all the necessary constituent files were found @@ -1110,13 +1113,13 @@ def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, constit_d dmsg = "create time series:" dmsg += f"\n \t needed constituents for derivation of {var}:\n\t\t- {constit_list}\n" dmsg += f" \t found constituent file(s) in {Path(constit_files[0]).parent}:\n" - dmsg += f"\t\t- {[Path(file).parts[-1] for file in constit_files if Path(file).is_file()]}" + dmsg += f"\t\t- {[Path(f).parts[-1] for f in constit_files if Path(f).is_file()]}" self.debug_log(dmsg) else: #Open a new dataset with all the constituent files/variables ds = xr.open_mfdataset(constit_files) - + # create new file name for derived variable derived_file = constit_files[0].replace(constit_list[0], var) @@ -1186,7 +1189,8 @@ def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, constit_d #Helper Function(s) def _load_dataset(fils): """ - This method exists to get an xarray Dataset from input file information that can be passed into the plotting methods. + This method exists to get an xarray Dataset from input file information that + can be passed into the plotting methods. Parameters ---------- From d343198a2251ec22ffad42bd6bd17752a34890e7 Mon Sep 17 00:00:00 2001 From: justin-richling Date: Wed, 15 May 2024 17:03:46 -0600 Subject: [PATCH 04/16] Update adf_diag.py --- lib/adf_diag.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/lib/adf_diag.py b/lib/adf_diag.py index 82939e3f5..62a4a51a6 100644 --- a/lib/adf_diag.py +++ b/lib/adf_diag.py @@ -567,16 +567,17 @@ def call_ncrcat(cmd): derive = True constit_list = vres["derivable_from"] #Check if variable is potentially part of a CAM-CHEM run - if (any(item not in hist_file_ds.data_vars for item in constit_list)) and (var in res["cam_chem_list"]): - #Set check to look for CAM-CHEM constituents list in variable defaults - get_cam_chem_constits = True + if any(item not in hist_file_ds.data_vars for item in constit_list): + if var in res["cam_chem_list"]: + #Set check to look for CAM-CHEM constituents in variable defaults + get_cam_chem_constits = True else: get_cam_chem_constits = False #End if #If this is a CAM-CHEM run, update constit_list if get_cam_chem_constits: - print(f"Looks like this a CAM-CHEM run,") + print("Looks like this a CAM-CHEM run,") print(f" checking constituents for '{var}'") if "derivable_from_cam_chem" in vres: constit_list = vres['derivable_from_cam_chem'] @@ -1101,7 +1102,7 @@ def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, constit_files = [] for constit in constit_list: #Check if the constituent file is present, if so add it to list - if glob.glob(os.path.join(ts_dir, f"*.{constit}.*.nc")): + if glob.glob(os.path.join(ts_dir, f"*.{constit}.*.nc")): constit_files.append(glob.glob(os.path.join(ts_dir, f"*.{constit}.*"))[0]) #Check if all the necessary constituent files were found @@ -1111,8 +1112,8 @@ def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, print(ermsg) #Add what's missing to debug log dmsg = "create time series:" - dmsg += f"\n \t needed constituents for derivation of {var}:\n\t\t- {constit_list}\n" - dmsg += f" \t found constituent file(s) in {Path(constit_files[0]).parent}:\n" + dmsg += f"\n\tneeded constituents for derivation of {var}:\n\t\t- {constit_list}\n" + dmsg += f"\tfound constituent file(s) in {Path(constit_files[0]).parent}:\n" dmsg += f"\t\t- {[Path(f).parts[-1] for f in constit_files if Path(f).is_file()]}" self.debug_log(dmsg) @@ -1158,16 +1159,18 @@ def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, ds_pmid = _load_dataset(glob.glob(os.path.join(ts_dir, "*.PMID.*"))[0]) ds_pmid_done = True if not ds_pmid: - errmsg = f"Missing necessary files for dry air density (rho) calculation.\n" - errmsg += "Please make sure 'PMID' is in the CAM run for aerosol calculations" + errmsg = "Missing necessary files for dry air density (rho) " + errmsg += "calculation.\nPlease make sure 'PMID' is in the CAM " + errmsg += "run for aerosol calculations" print(errmsg) continue if not ds_t_done: ds_t = _load_dataset(glob.glob(os.path.join(ts_dir, "*.T.*"))[0]) ds_t_done = True if not ds_t: - errmsg = f"Missing necessary files for dry air density (rho) calculation.\n" - errmsg += "Please make sure 'T' is in the CAM run for aerosol calculations" + errmsg = "Missing necessary files for dry air density (rho) " + errmsg += "calculation.\nPlease make sure 'T' is in the CAM " + errmsg += "run for aerosol calculations" print(errmsg) continue @@ -1216,7 +1219,7 @@ def my_formatwarning(msg, *args, **kwargs): if len(fils) == 0: warnings.warn("Input file list is empty.") return None - elif len(fils) > 1: + if len(fils) > 1: return xr.open_mfdataset(fils, combine='by_coords') else: return xr.open_dataset(fils[0]) From 723c1c035b670e5bab505b7d2557506a808608b9 Mon Sep 17 00:00:00 2001 From: justin-richling Date: Thu, 16 May 2024 15:35:11 -0600 Subject: [PATCH 05/16] Update adf_diag.py --- lib/adf_diag.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/lib/adf_diag.py b/lib/adf_diag.py index 62a4a51a6..4ee4dc564 100644 --- a/lib/adf_diag.py +++ b/lib/adf_diag.py @@ -561,6 +561,8 @@ def call_ncrcat(cmd): #intialize boolean to check if variable is derivable derive = False # assume it can't be derived and update if it can + #intialize boolean for CAM-CHEM variable + get_cam_chem_constits = False #Try and build variable from 'derivable_from' if "derivable_from" in vres: @@ -571,8 +573,6 @@ def call_ncrcat(cmd): if var in res["cam_chem_list"]: #Set check to look for CAM-CHEM constituents in variable defaults get_cam_chem_constits = True - else: - get_cam_chem_constits = False #End if #If this is a CAM-CHEM run, update constit_list @@ -583,7 +583,7 @@ def call_ncrcat(cmd): constit_list = vres['derivable_from_cam_chem'] else: derive = False - errmsg = f"\n Missing 'derivable_from_cam_chem' " + errmsg = "\n Missing 'derivable_from_cam_chem' " errmsg += f"config argument for {var}." errmsg += "\n\tPlease remove variable from ADF run or set" errmsg += " appropriate argument in variable defaults yaml file." @@ -957,6 +957,12 @@ def setup_run_cvdp(self): ) # End if + #intialize objects that might not be declared later + case_name_baseline = None + baseline_ts_loc = None + syears_baseline = None + eyears_baseline = None + # check to see if there is a CAM baseline case. If there is, read in relevant information. if not self.get_basic_info("compare_obs"): case_name_baseline = self.get_baseline_info("cam_case_name") @@ -1129,9 +1135,9 @@ def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, if overwrite: Path(derived_file).unlink() else: - print( - f"[{__name__}] Warning: '{var}' file was found and overwrite is False. Will use existing file." - ) + msg = f"[{__name__}] Warning: '{var}' file was found " + msg += "and overwrite is False. Will use existing file." + print(msg) continue #NOTE: this will need to be changed when derived equations are more complex! - JR @@ -1142,7 +1148,7 @@ def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, der_val = 0 for v in constit_list: der_val += ds[v] - + #Set derived variable name and add to dataset der_val.name = var ds[var] = der_val @@ -1153,7 +1159,7 @@ def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, ds_pmid_done = False ds_t_done = False if var in res["aerosol_zonal_list"]: - + #Only calculate once for all aerosol vars if not ds_pmid_done: ds_pmid = _load_dataset(glob.glob(os.path.join(ts_dir, "*.PMID.*"))[0]) From 2114a65ecfc992d22813c9cafb0c2b0f1258be51 Mon Sep 17 00:00:00 2001 From: justin-richling Date: Thu, 13 Jun 2024 15:53:29 -0600 Subject: [PATCH 06/16] Clean up derived variables Change logic for checking for CAM-CHEM derived variables and also add time stamp to ADF log file name --- lib/adf_base.py | 6 +- lib/adf_diag.py | 154 ++++++++++++++++++++++++++++-------------------- 2 files changed, 95 insertions(+), 65 deletions(-) diff --git a/lib/adf_base.py b/lib/adf_base.py index b6d8fbea7..e259a5dee 100644 --- a/lib/adf_base.py +++ b/lib/adf_base.py @@ -49,7 +49,11 @@ def __init__(self, debug = False): # Create debug log, if requested: if debug: - logging.basicConfig(filename="ADF_debug.log", level=logging.DEBUG) + from datetime import datetime + # Get the current date and time + current_timestamp = datetime.now() + ext = f'{str(current_timestamp).replace(" ","-")}' + logging.basicConfig(filename=f"ADF_debug_{ext}.log", level=logging.DEBUG) self.__debug_log = logging.getLogger("ADF") else: self.__debug_log = None diff --git a/lib/adf_diag.py b/lib/adf_diag.py index 4ee4dc564..e855e87ff 100644 --- a/lib/adf_diag.py +++ b/lib/adf_diag.py @@ -539,84 +539,103 @@ def call_ncrcat(cmd): # Aerosol Calcs #-------------- #Always make sure PMID is made if aerosols are desired in config file + # Since there's no requirement for `aerosol_zonal_list` to be included, allow it to be absent: + azl = res.get("aerosol_zonal_list", []) if "PMID" not in diag_var_list: - if any(item in res["aerosol_zonal_list"] for item in diag_var_list): + if any(item in azl for item in diag_var_list): diag_var_list += ["PMID"] if "T" not in diag_var_list: - if any(item in res["aerosol_zonal_list"] for item in diag_var_list): + if any(item in azl for item in diag_var_list): diag_var_list += ["T"] #End aerosol calcs - #Initialize dictionary for derived var with needed list of constituents + #Initialize dictionary for derived variable with needed list of constituents constit_dict = {} + for var in diag_var_list: + # Notify user of new time series file: + print(f"\t - time series for {var}") + + # Set error messages for printing/debugging + # Derived variable, but missing constituent list + constit_errmsg = f"create time series for {case_name}:" + constit_errmsg += f"\n Can't create time series for {var}. \n\tThis variable" + constit_errmsg += " is flagged for derivation, but is missing list of constiuents." + constit_errmsg += "\n\tPlease add list of constituents to 'derivable_from' " + constit_errmsg += f"for {var} in variable defaults yaml file." + #Check if current variable is a derived quantity if var not in hist_file_var_list: vres = res.get(var, {}) #Initialiaze list for constituents - #NOTE: This is if the variable is NOT derivable but need + #NOTE: This is if the variable is NOT derivable but needs # an empty list as a check later constit_list = [] #intialize boolean to check if variable is derivable derive = False # assume it can't be derived and update if it can - #intialize boolean for CAM-CHEM variable - get_cam_chem_constits = False - - #Try and build variable from 'derivable_from' - if "derivable_from" in vres: - derive = True - constit_list = vres["derivable_from"] - #Check if variable is potentially part of a CAM-CHEM run - if any(item not in hist_file_ds.data_vars for item in constit_list): - if var in res["cam_chem_list"]: - #Set check to look for CAM-CHEM constituents in variable defaults - get_cam_chem_constits = True - #End if - #If this is a CAM-CHEM run, update constit_list - if get_cam_chem_constits: - print("Looks like this a CAM-CHEM run,") - print(f" checking constituents for '{var}'") - if "derivable_from_cam_chem" in vres: - constit_list = vres['derivable_from_cam_chem'] - else: - derive = False - errmsg = "\n Missing 'derivable_from_cam_chem' " - errmsg += f"config argument for {var}." - errmsg += "\n\tPlease remove variable from ADF run or set" - errmsg += " appropriate argument in variable defaults yaml file." - print(errmsg) - #End if + #intialize boolean for regular CAM variable constituents + try_cam_constits = True + + #Check first if variable is potentially part of a CAM-CHEM run + if "derivable_from_cam_chem" in vres: + constit_list = vres["derivable_from_cam_chem"] + if constit_list: + if all(item in hist_file_ds.data_vars for item in constit_list): + #Set check to look for regular CAM constituents in variable defaults + try_cam_constits = False + derive = True + msg = f"create time series for {case_name}:" + msg += "\n\tLooks like this a CAM-CHEM run, " + msg += f"checking constituents for '{var}'" + self.debug_log(msg) + else: + self.debug_log(constit_errmsg) #End if + #End if - #Now check if this variable can be derived - if derive: - for constit in constit_list: - if constit not in diag_var_list: - diag_var_list.append(constit) - #Add variable to list to derive - vars_to_derive.append(var) - #Add constituent list to variable key in dictionary - constit_dict[var] = constit_list - continue + #If not CAM-CHEM, check regular CAM runs + if try_cam_constits: + if "derivable_from" in vres: + derive = True + constit_list = vres["derivable_from"] + else: + # Missing variable or missing derivable_from argument + derive_from_errmsg = f"create time series for {case_name}:" + derive_from_errmsg += f"\n Can't create time series for {var}." + derive_from_errmsg += "\n\tEither the variable is missing from CAM output or it " + derive_from_errmsg += "is a derived quantity and is missing the 'derivable_from' " + derive_from_errmsg += "config argument.\n\tPlease add variable to CAM run " + derive_from_errmsg += "or set appropriate argument in variable defaults yaml file." + self.debug_log(derive_from_errmsg) #End if - - else: - errmsg = f"\n Missing 'derivable_from' config argument for {var}." - errmsg += "\n\tPlease remove variable from ADF run or set appropriate" - errmsg += " argument in variable defaults yaml file." - print(errmsg) - #End if 'derivable_from' - + #End if + + #Check if this variable can be derived + if (derive) and (constit_list): + for constit in constit_list: + if constit not in diag_var_list: + diag_var_list.append(constit) + #Add variable to list to derive + vars_to_derive.append(var) + #Add constituent list to variable key in dictionary + constit_dict[var] = constit_list + continue + #Log if this variable can be derived but is missing list of constituents + elif (derive) and (not constit_list): + self.debug_log(constit_errmsg) + continue #Lastly, raise error if the variable is not a derived quanitity but is also not #in the history file(s) - if (not derive) and (not constit_list): - msg = f"WARNING: {var} is not in the file {hist_files[0]}." - msg += " No time series will be generated." + else: + msg = f"WARNING: {var} is not in the file {hist_files[0]} " + msg += "nor can it be derived.\n" + msg += "\t ** No time series will be generated." print(msg) continue + #End if #End if # Check if variable has a "lev" dimension according to first file: @@ -640,9 +659,6 @@ def call_ncrcat(cmd): # If not, then simply skip this variable: continue - # Notify user of new time series file: - print(f"\t - time series for {var}") - # Variable list starts with just the variable ncrcat_var_list = f"{var}" @@ -1113,15 +1129,22 @@ def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, #Check if all the necessary constituent files were found if len(constit_files) != len(constit_list): - ermsg = f"Not all constituent files present; {var} cannot be calculated." - ermsg += f" Please remove {var} from diag_var_list or find the relevant CAM files." + ermsg = f"\t ** Not all constituent files present; {var} cannot be calculated." + ermsg += f" Please remove {var} from 'diag_var_list' or find the " + ermsg += "relevant CAM files.\n" print(ermsg) - #Add what's missing to debug log - dmsg = "create time series:" - dmsg += f"\n\tneeded constituents for derivation of {var}:\n\t\t- {constit_list}\n" - dmsg += f"\tfound constituent file(s) in {Path(constit_files[0]).parent}:\n" - dmsg += f"\t\t- {[Path(f).parts[-1] for f in constit_files if Path(f).is_file()]}" - self.debug_log(dmsg) + if constit_files: + #Add what's missing to debug log + dmsg = "create time series:" + dmsg += f"\n\tneeded constituents for derivation of {var}:\n\t\t- {constit_list}\n" + dmsg += f"\tfound constituent file(s) in {Path(constit_files[0]).parent}:\n" + dmsg += f"\t\t- {[Path(f).parts[-1] for f in constit_files if Path(f).is_file()]}" + self.debug_log(dmsg) + else: + dmsg = "create time series:" + dmsg += f"\n\tneeded constituents for derivation of {var}:\n\t\t- {constit_list}\n" + dmsg += f"\tNo constituent(s) found in history files" + self.debug_log(dmsg) else: #Open a new dataset with all the constituent files/variables @@ -1158,7 +1181,10 @@ def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, #These will be multiplied by rho (density of dry air) ds_pmid_done = False ds_t_done = False - if var in res["aerosol_zonal_list"]: + + # User-defined defaults might not include aerosol zonal list + azl = res.get("aerosol_zonal_list", []) + if var in azl: #Only calculate once for all aerosol vars if not ds_pmid_done: From ce47e2dfa54e57e2e24b6e10aaee141b6118a605 Mon Sep 17 00:00:00 2001 From: justin-richling Date: Thu, 13 Jun 2024 16:03:14 -0600 Subject: [PATCH 07/16] Update adf_diag.py --- lib/adf_diag.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/lib/adf_diag.py b/lib/adf_diag.py index e855e87ff..0fa8380cd 100644 --- a/lib/adf_diag.py +++ b/lib/adf_diag.py @@ -539,7 +539,7 @@ def call_ncrcat(cmd): # Aerosol Calcs #-------------- #Always make sure PMID is made if aerosols are desired in config file - # Since there's no requirement for `aerosol_zonal_list` to be included, allow it to be absent: + # Since there's no requirement for `aerosol_zonal_list`, allow it to be absent: azl = res.get("aerosol_zonal_list", []) if "PMID" not in diag_var_list: if any(item in azl for item in diag_var_list): @@ -603,13 +603,14 @@ def call_ncrcat(cmd): constit_list = vres["derivable_from"] else: # Missing variable or missing derivable_from argument - derive_from_errmsg = f"create time series for {case_name}:" - derive_from_errmsg += f"\n Can't create time series for {var}." - derive_from_errmsg += "\n\tEither the variable is missing from CAM output or it " - derive_from_errmsg += "is a derived quantity and is missing the 'derivable_from' " - derive_from_errmsg += "config argument.\n\tPlease add variable to CAM run " - derive_from_errmsg += "or set appropriate argument in variable defaults yaml file." - self.debug_log(derive_from_errmsg) + der_from_msg = f"create time series for {case_name}:" + der_from_msg += f"\n Can't create time series for {var}.\n\tEither " + der_from_msg += "the variable is missing from CAM output or it is a " + der_from_msg += "derived quantity and is missing the 'derivable_from' " + der_from_msg += "config argument.\n\tPlease add variable to CAM run " + der_from_msg += "or set appropriate argument in variable " + der_from_msg += "defaults yaml file." + self.debug_log(der_from_msg) #End if #End if @@ -1136,13 +1137,15 @@ def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, if constit_files: #Add what's missing to debug log dmsg = "create time series:" - dmsg += f"\n\tneeded constituents for derivation of {var}:\n\t\t- {constit_list}\n" - dmsg += f"\tfound constituent file(s) in {Path(constit_files[0]).parent}:\n" - dmsg += f"\t\t- {[Path(f).parts[-1] for f in constit_files if Path(f).is_file()]}" + dmsg += f"\n\tneeded constituents for derivation of " + dmsg += f"{var}:\n\t\t- {constit_list}\n\tfound constituent file(s) in " + dmsg += f"{Path(constit_files[0]).parent}:\n\t\t" + dmsg += f"- {[Path(f).parts[-1] for f in constit_files if Path(f).is_file()]}" self.debug_log(dmsg) else: dmsg = "create time series:" - dmsg += f"\n\tneeded constituents for derivation of {var}:\n\t\t- {constit_list}\n" + dmsg += f"\n\tneeded constituents for derivation of " + dmsg += f"{var}:\n\t\t- {constit_list}\n" dmsg += f"\tNo constituent(s) found in history files" self.debug_log(dmsg) From 0305224cd91f2dd39809a2ac7772c9a66dbcb7cd Mon Sep 17 00:00:00 2001 From: justin-richling Date: Thu, 20 Jun 2024 12:49:15 -0600 Subject: [PATCH 08/16] Update ADF debug log file name --- lib/adf_base.py | 3 ++- lib/test/unit_tests/test_adf_base.py | 21 +++++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/lib/adf_base.py b/lib/adf_base.py index e259a5dee..0c806e9e8 100644 --- a/lib/adf_base.py +++ b/lib/adf_base.py @@ -53,7 +53,8 @@ def __init__(self, debug = False): # Get the current date and time current_timestamp = datetime.now() ext = f'{str(current_timestamp).replace(" ","-")}' - logging.basicConfig(filename=f"ADF_debug_{ext}.log", level=logging.DEBUG) + debug_fname = f"ADF_debug_{ext}.log" + logging.basicConfig(filename=debug_fname, level=logging.DEBUG) self.__debug_log = logging.getLogger("ADF") else: self.__debug_log = None diff --git a/lib/test/unit_tests/test_adf_base.py b/lib/test/unit_tests/test_adf_base.py index b757cbd29..c4e9ca8cb 100644 --- a/lib/test/unit_tests/test_adf_base.py +++ b/lib/test/unit_tests/test_adf_base.py @@ -22,6 +22,7 @@ #Import AdfBase class from adf_base import AdfBase +debug_fname = AdfBase.debug_fname from adf_base import AdfError #++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -45,8 +46,8 @@ def tearDown(self): """ #Remove log file if it exists: - if os.path.exists("ADF_debug.log"): - os.remove("ADF_debug.log") + if os.path.exists(debug_fname): + os.remove(debug_fname) #Close all log streams: @@ -80,8 +81,8 @@ def test_AdfBase_debug_create(self): #Assert that new object is of the "AdfBase" class: self.assertIsInstance(adf_test, AdfBase) - #Assert that "ADF_debug.log" file exists in local directory: - self.assertTrue(os.path.exists("ADF_debug.log")) + #Assert that ADF debug log file exists in local directory: + self.assertTrue(os.path.exists(debug_fname)) def test_AdfBase_bad_debug(self): @@ -117,8 +118,10 @@ def test_AdfBase_debug_nothing(self): #Call "debug_log" method: adf_test.debug_log("test") + print("YAHOOO:",AdfBase.debug_fname) + #Check that no log file exists: - self.assertFalse(os.path.exists("ADF_debug.log")) + self.assertFalse(os.path.exists(debug_fname)) def test_AdfBase_debug_write(self): @@ -134,14 +137,16 @@ def test_AdfBase_debug_write(self): #Call "debug_log" method: adf_test.debug_log("test") + print("YAHOOO:",AdfBase.debug_fname) + #Check that debug log exists: - self.assertTrue(os.path.exists("ADF_debug.log")) + self.assertTrue(os.path.exists(debug_fname)) #If debug log exists, then open file: - if os.path.exists("ADF_debug.log"): + if os.path.exists(debug_fname): #Open log file: - with open("ADF_debug.log") as logfil: + with open(debug_fname) as logfil: #Extract file contents: log_text = logfil.read() From fa6971ec5dd345c921c097887a14bf3f3c2d9ad6 Mon Sep 17 00:00:00 2001 From: justin-richling Date: Thu, 20 Jun 2024 13:21:11 -0600 Subject: [PATCH 09/16] Update debug log files --- lib/adf_base.py | 9 +++++++++ lib/test/unit_tests/test_adf_base.py | 22 +++++++++++++++------- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/lib/adf_base.py b/lib/adf_base.py index 0c806e9e8..160dd3efe 100644 --- a/lib/adf_base.py +++ b/lib/adf_base.py @@ -54,13 +54,22 @@ def __init__(self, debug = False): current_timestamp = datetime.now() ext = f'{str(current_timestamp).replace(" ","-")}' debug_fname = f"ADF_debug_{ext}.log" + self.__debug_fname = debug_fname logging.basicConfig(filename=debug_fname, level=logging.DEBUG) self.__debug_log = logging.getLogger("ADF") else: self.__debug_log = None + + ######### + # Create property needed to return the number of test cases (num_cases) to user: + @property + def debug_fname(self): + """Return the "debug_fname" string to the user.""" + return self.__debug_fname + def debug_log(self, msg: str): """ diff --git a/lib/test/unit_tests/test_adf_base.py b/lib/test/unit_tests/test_adf_base.py index c4e9ca8cb..324bb31b1 100644 --- a/lib/test/unit_tests/test_adf_base.py +++ b/lib/test/unit_tests/test_adf_base.py @@ -12,6 +12,7 @@ import os import os.path import logging +import glob #Set relevant path variables: _CURRDIR = os.path.abspath(os.path.dirname(__file__)) @@ -22,7 +23,6 @@ #Import AdfBase class from adf_base import AdfBase -debug_fname = AdfBase.debug_fname from adf_base import AdfError #++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -45,9 +45,12 @@ def tearDown(self): Remove log files (if they exist). """ - #Remove log file if it exists: - if os.path.exists(debug_fname): - os.remove(debug_fname) + debug_list = glob.glob("ADF_debug*") + + for dfile in debug_list: + #Remove log file if it exists: + if os.path.exists(dfile): + os.remove(dfile) #Close all log streams: @@ -78,6 +81,9 @@ def test_AdfBase_debug_create(self): #Create AdfBase object with debug setting: adf_test = AdfBase(debug=True) + #Grab debug log name + debug_fname = adf_test.debug_fname + #Assert that new object is of the "AdfBase" class: self.assertIsInstance(adf_test, AdfBase) @@ -118,7 +124,8 @@ def test_AdfBase_debug_nothing(self): #Call "debug_log" method: adf_test.debug_log("test") - print("YAHOOO:",AdfBase.debug_fname) + #Grab debug log name + debug_fname = adf_test.debug_fname #Check that no log file exists: self.assertFalse(os.path.exists(debug_fname)) @@ -134,11 +141,12 @@ def test_AdfBase_debug_write(self): #Create AdfBase object with debug setting: adf_test = AdfBase(debug=True) + #Grab debug log name + debug_fname = adf_test.debug_fname + #Call "debug_log" method: adf_test.debug_log("test") - print("YAHOOO:",AdfBase.debug_fname) - #Check that debug log exists: self.assertTrue(os.path.exists(debug_fname)) From 998d78d3f7c6da7c1beec7c710ab5980f22b6ff2 Mon Sep 17 00:00:00 2001 From: justin-richling Date: Thu, 20 Jun 2024 13:27:02 -0600 Subject: [PATCH 10/16] update debug errors --- lib/adf_base.py | 2 ++ lib/test/unit_tests/test_adf_base.py | 10 ++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/adf_base.py b/lib/adf_base.py index 160dd3efe..0121ba53e 100644 --- a/lib/adf_base.py +++ b/lib/adf_base.py @@ -47,6 +47,8 @@ def __init__(self, debug = False): if not isinstance(debug, bool): raise TypeError("'debug' must be a boolean type (True or False)") + self.__debug_fname = '' + # Create debug log, if requested: if debug: from datetime import datetime diff --git a/lib/test/unit_tests/test_adf_base.py b/lib/test/unit_tests/test_adf_base.py index 324bb31b1..2c693fed1 100644 --- a/lib/test/unit_tests/test_adf_base.py +++ b/lib/test/unit_tests/test_adf_base.py @@ -47,10 +47,10 @@ def tearDown(self): debug_list = glob.glob("ADF_debug*") - for dfile in debug_list: - #Remove log file if it exists: - if os.path.exists(dfile): - os.remove(dfile) + #for dfile in debug_list: + # #Remove log file if it exists: + # if os.path.exists(dfile): + # os.remove(dfile) #Close all log streams: @@ -147,6 +147,8 @@ def test_AdfBase_debug_write(self): #Call "debug_log" method: adf_test.debug_log("test") + print(debug_fname) + #Check that debug log exists: self.assertTrue(os.path.exists(debug_fname)) From 1b4ab1f3c282cc4e02de777d2e84e58052958dc6 Mon Sep 17 00:00:00 2001 From: justin-richling Date: Thu, 20 Jun 2024 13:30:24 -0600 Subject: [PATCH 11/16] Update adf_base.py --- lib/adf_base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/adf_base.py b/lib/adf_base.py index 0121ba53e..b028f21bd 100644 --- a/lib/adf_base.py +++ b/lib/adf_base.py @@ -54,7 +54,9 @@ def __init__(self, debug = False): from datetime import datetime # Get the current date and time current_timestamp = datetime.now() - ext = f'{str(current_timestamp).replace(" ","-")}' + # Format the datetime object to a string without microseconds + dt_str = current_timestamp.strftime('%Y-%m-%d %H:%M:%S') + ext = f'{str(dt_str).replace(" ","-")}' debug_fname = f"ADF_debug_{ext}.log" self.__debug_fname = debug_fname logging.basicConfig(filename=debug_fname, level=logging.DEBUG) From 6b99cbdbf8916c4d7c3df709542e23a3fb593b1a Mon Sep 17 00:00:00 2001 From: justin-richling Date: Thu, 20 Jun 2024 13:40:36 -0600 Subject: [PATCH 12/16] Update adf_variable_defaults.yaml --- lib/adf_variable_defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/adf_variable_defaults.yaml b/lib/adf_variable_defaults.yaml index 1197b515f..44ae838ea 100644 --- a/lib/adf_variable_defaults.yaml +++ b/lib/adf_variable_defaults.yaml @@ -237,8 +237,8 @@ SO4: colorbar: label : '$\mu$g/m3' category: "Aerosols" - derivable_from_cam_chem: ["so4_a1", "so4_a2", "so4_a3", "so4_a5"] derivable_from: ["so4_a1", "so4_a2", "so4_a3"] + derivable_from_cam_chem: ["so4_a1", "so4_a2", "so4_a3", "so4_a5"] SOA: colormap: "RdBu_r" From 780e9a8a921df98a28e591e5bf59648daf047514 Mon Sep 17 00:00:00 2001 From: justin-richling Date: Thu, 20 Jun 2024 14:10:53 -0600 Subject: [PATCH 13/16] Update adf_info.py --- lib/adf_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/adf_info.py b/lib/adf_info.py index 42fe47fc3..a507ec5dc 100644 --- a/lib/adf_info.py +++ b/lib/adf_info.py @@ -109,7 +109,7 @@ def __init__(self, config_file, debug=False): hist_str = self.get_basic_info('hist_str') #If hist_str is not present, then default to 'cam.h0': if not hist_str: - hist_str = 'cam.h0' + hist_str = 'cam.h0a' #End if #Initialize ADF variable list: From 72979d8fd43076612d75adb6f03ebd281aac25b3 Mon Sep 17 00:00:00 2001 From: justin-richling Date: Mon, 24 Jun 2024 08:41:55 -0600 Subject: [PATCH 14/16] Fix GitHub comments --- lib/adf_base.py | 4 ++-- lib/adf_diag.py | 10 +++++----- lib/adf_variable_defaults.yaml | 2 +- lib/test/unit_tests/test_adf_base.py | 12 +++++------- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/lib/adf_base.py b/lib/adf_base.py index b028f21bd..64618f2c0 100644 --- a/lib/adf_base.py +++ b/lib/adf_base.py @@ -17,6 +17,7 @@ #++++++++++++++++++++++++++++++ import logging +from datetime import datetime #+++++++++++++++++++++++++ # ADF Error-handling class @@ -51,7 +52,6 @@ def __init__(self, debug = False): # Create debug log, if requested: if debug: - from datetime import datetime # Get the current date and time current_timestamp = datetime.now() # Format the datetime object to a string without microseconds @@ -68,7 +68,7 @@ def __init__(self, debug = False): ######### - # Create property needed to return the number of test cases (num_cases) to user: + # Create property needed to return the name of the debug log file (debug_fname) to user: @property def debug_fname(self): """Return the "debug_fname" string to the user.""" diff --git a/lib/adf_diag.py b/lib/adf_diag.py index 0fa8380cd..a9bc24c45 100644 --- a/lib/adf_diag.py +++ b/lib/adf_diag.py @@ -720,8 +720,8 @@ def call_ncrcat(cmd): if vars_to_derive: self.derive_variables( - res=res, vars_to_derive=vars_to_derive, ts_dir=ts_dir[case_idx], - constit_dict=constit_dict + res=res, hist_str=hist_str, vars_to_derive=vars_to_derive, + ts_dir=ts_dir[case_idx], constit_dict=constit_dict ) # End with @@ -1101,7 +1101,7 @@ def setup_run_cvdp(self): ######### - def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, + def derive_variables(self, res=None, hist_str=None, vars_to_derive=None, ts_dir=None, constit_dict=None, overwrite=None): """ Derive variables acccording to steps given here. Since derivations will depend on the @@ -1125,8 +1125,8 @@ def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, constit_files = [] for constit in constit_list: #Check if the constituent file is present, if so add it to list - if glob.glob(os.path.join(ts_dir, f"*.{constit}.*.nc")): - constit_files.append(glob.glob(os.path.join(ts_dir, f"*.{constit}.*"))[0]) + if glob.glob(os.path.join(ts_dir, f"*{hist_str}*.{constit}.*.nc")): + constit_files.append(glob.glob(os.path.join(ts_dir, f"*{hist_str}*.{constit}.*"))[0]) #Check if all the necessary constituent files were found if len(constit_files) != len(constit_list): diff --git a/lib/adf_variable_defaults.yaml b/lib/adf_variable_defaults.yaml index 44ae838ea..ae18d2310 100644 --- a/lib/adf_variable_defaults.yaml +++ b/lib/adf_variable_defaults.yaml @@ -79,7 +79,7 @@ default_ptypes: ["Tables","LatLon","LatLon_Vector","Zonal","Meridional", Rgas: 287.04 #[J/K/Kg]=8.314/0.028965 #+++++++++++++ -# CAM CHEM Variables +# CAM-CHEM Variables #+++++++++++++ #List of variables for CAM-CHEM runs that have different constituents than regular CAM runs cam_chem_list: ["SOA","SO4"] diff --git a/lib/test/unit_tests/test_adf_base.py b/lib/test/unit_tests/test_adf_base.py index 2c693fed1..6e13f6ea1 100644 --- a/lib/test/unit_tests/test_adf_base.py +++ b/lib/test/unit_tests/test_adf_base.py @@ -45,12 +45,12 @@ def tearDown(self): Remove log files (if they exist). """ - debug_list = glob.glob("ADF_debug*") + debug_list = glob.glob("ADF_debug*.log") - #for dfile in debug_list: - # #Remove log file if it exists: - # if os.path.exists(dfile): - # os.remove(dfile) + for dfile in debug_list: + #Remove log file if it exists: + if os.path.exists(dfile): + os.remove(dfile) #Close all log streams: @@ -147,8 +147,6 @@ def test_AdfBase_debug_write(self): #Call "debug_log" method: adf_test.debug_log("test") - print(debug_fname) - #Check that debug log exists: self.assertTrue(os.path.exists(debug_fname)) From 5fa4ef1ee915b6754bebdbf245bca7465c7d55ab Mon Sep 17 00:00:00 2001 From: justin-richling Date: Mon, 24 Jun 2024 09:18:01 -0600 Subject: [PATCH 15/16] Add check for if `hist_str` is None --- lib/adf_diag.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/adf_diag.py b/lib/adf_diag.py index a9bc24c45..3a61a27a0 100644 --- a/lib/adf_diag.py +++ b/lib/adf_diag.py @@ -1125,8 +1125,13 @@ def derive_variables(self, res=None, hist_str=None, vars_to_derive=None, ts_dir= constit_files = [] for constit in constit_list: #Check if the constituent file is present, if so add it to list - if glob.glob(os.path.join(ts_dir, f"*{hist_str}*.{constit}.*.nc")): - constit_files.append(glob.glob(os.path.join(ts_dir, f"*{hist_str}*.{constit}.*"))[0]) + if hist_str: + const_glob_str = f"*{hist_str}*.{constit}.*.nc" + else: + const_glob_str = f"*.{constit}.*.nc" + #end if + if glob.glob(os.path.join(ts_dir, const_glob_str)): + constit_files.append(glob.glob(os.path.join(ts_dir, const_glob_str ))[0]) #Check if all the necessary constituent files were found if len(constit_files) != len(constit_list): From 9450a4e2ba3bfca9c6083520ae229ceebd4d0e9d Mon Sep 17 00:00:00 2001 From: justin-richling Date: Mon, 24 Jun 2024 09:18:06 -0600 Subject: [PATCH 16/16] Update adf_info.py --- lib/adf_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/adf_info.py b/lib/adf_info.py index a507ec5dc..e7297a03b 100644 --- a/lib/adf_info.py +++ b/lib/adf_info.py @@ -107,7 +107,7 @@ def __init__(self, config_file, debug=False): #Read hist_str (component.hist_num) from the yaml file, or set to default hist_str = self.get_basic_info('hist_str') - #If hist_str is not present, then default to 'cam.h0': + #If hist_str is not present, then default to 'cam.h0a': if not hist_str: hist_str = 'cam.h0a' #End if