diff --git a/lib/adf_base.py b/lib/adf_base.py index b6d8fbea7..64618f2c0 100644 --- a/lib/adf_base.py +++ b/lib/adf_base.py @@ -17,6 +17,7 @@ #++++++++++++++++++++++++++++++ import logging +from datetime import datetime #+++++++++++++++++++++++++ # ADF Error-handling class @@ -47,15 +48,32 @@ def __init__(self, debug = False): if not isinstance(debug, bool): raise TypeError("'debug' must be a boolean type (True or False)") + self.__debug_fname = '' + # Create debug log, if requested: if debug: - logging.basicConfig(filename="ADF_debug.log", level=logging.DEBUG) + # Get the current date and time + current_timestamp = datetime.now() + # Format the datetime object to a string without microseconds + dt_str = current_timestamp.strftime('%Y-%m-%d %H:%M:%S') + ext = f'{str(dt_str).replace(" ","-")}' + debug_fname = f"ADF_debug_{ext}.log" + self.__debug_fname = debug_fname + logging.basicConfig(filename=debug_fname, level=logging.DEBUG) self.__debug_log = logging.getLogger("ADF") else: self.__debug_log = None + + ######### + # Create property needed to return the name of the debug log file (debug_fname) to user: + @property + def debug_fname(self): + """Return the "debug_fname" string to the user.""" + return self.__debug_fname + def debug_log(self, msg: str): """ diff --git a/lib/adf_diag.py b/lib/adf_diag.py index 9c57c1f36..3a61a27a0 100644 --- a/lib/adf_diag.py +++ b/lib/adf_diag.py @@ -1,3 +1,4 @@ + """ Location of the "AdfDiag" object, which is used to store all relevant data and @@ -538,29 +539,105 @@ def call_ncrcat(cmd): # Aerosol Calcs #-------------- #Always make sure PMID is made if aerosols are desired in config file + # Since there's no requirement for `aerosol_zonal_list`, allow it to be absent: + azl = res.get("aerosol_zonal_list", []) if "PMID" not in diag_var_list: - if any(item in res["aerosol_zonal_list"] for item in diag_var_list): + if any(item in azl for item in diag_var_list): diag_var_list += ["PMID"] if "T" not in diag_var_list: - if any(item in res["aerosol_zonal_list"] for item in diag_var_list): + if any(item in azl for item in diag_var_list): diag_var_list += ["T"] #End aerosol calcs + #Initialize dictionary for derived variable with needed list of constituents + constit_dict = {} + for var in diag_var_list: + # Notify user of new time series file: + print(f"\t - time series for {var}") + + # Set error messages for printing/debugging + # Derived variable, but missing constituent list + constit_errmsg = f"create time series for {case_name}:" + constit_errmsg += f"\n Can't create time series for {var}. \n\tThis variable" + constit_errmsg += " is flagged for derivation, but is missing list of constiuents." + constit_errmsg += "\n\tPlease add list of constituents to 'derivable_from' " + constit_errmsg += f"for {var} in variable defaults yaml file." + + #Check if current variable is a derived quantity if var not in hist_file_var_list: vres = res.get(var, {}) - if "derivable_from" in vres: - constit_list = vres["derivable_from"] + + #Initialiaze list for constituents + #NOTE: This is if the variable is NOT derivable but needs + # an empty list as a check later + constit_list = [] + + #intialize boolean to check if variable is derivable + derive = False # assume it can't be derived and update if it can + + #intialize boolean for regular CAM variable constituents + try_cam_constits = True + + #Check first if variable is potentially part of a CAM-CHEM run + if "derivable_from_cam_chem" in vres: + constit_list = vres["derivable_from_cam_chem"] + if constit_list: + if all(item in hist_file_ds.data_vars for item in constit_list): + #Set check to look for regular CAM constituents in variable defaults + try_cam_constits = False + derive = True + msg = f"create time series for {case_name}:" + msg += "\n\tLooks like this a CAM-CHEM run, " + msg += f"checking constituents for '{var}'" + self.debug_log(msg) + else: + self.debug_log(constit_errmsg) + #End if + #End if + + #If not CAM-CHEM, check regular CAM runs + if try_cam_constits: + if "derivable_from" in vres: + derive = True + constit_list = vres["derivable_from"] + else: + # Missing variable or missing derivable_from argument + der_from_msg = f"create time series for {case_name}:" + der_from_msg += f"\n Can't create time series for {var}.\n\tEither " + der_from_msg += "the variable is missing from CAM output or it is a " + der_from_msg += "derived quantity and is missing the 'derivable_from' " + der_from_msg += "config argument.\n\tPlease add variable to CAM run " + der_from_msg += "or set appropriate argument in variable " + der_from_msg += "defaults yaml file." + self.debug_log(der_from_msg) + #End if + #End if + + #Check if this variable can be derived + if (derive) and (constit_list): for constit in constit_list: if constit not in diag_var_list: diag_var_list.append(constit) + #Add variable to list to derive vars_to_derive.append(var) + #Add constituent list to variable key in dictionary + constit_dict[var] = constit_list continue + #Log if this variable can be derived but is missing list of constituents + elif (derive) and (not constit_list): + self.debug_log(constit_errmsg) + continue + #Lastly, raise error if the variable is not a derived quanitity but is also not + #in the history file(s) else: - msg = f"WARNING: {var} is not in the file {hist_files[0]}." - msg += " No time series will be generated." + msg = f"WARNING: {var} is not in the file {hist_files[0]} " + msg += "nor can it be derived.\n" + msg += "\t ** No time series will be generated." print(msg) continue + #End if + #End if # Check if variable has a "lev" dimension according to first file: has_lev = bool("lev" in hist_file_ds[var].dims) @@ -583,9 +660,6 @@ def call_ncrcat(cmd): # If not, then simply skip this variable: continue - # Notify user of new time series file: - print(f"\t - time series for {var}") - # Variable list starts with just the variable ncrcat_var_list = f"{var}" @@ -646,7 +720,8 @@ def call_ncrcat(cmd): if vars_to_derive: self.derive_variables( - res=res, vars_to_derive=vars_to_derive, ts_dir=ts_dir[case_idx] + res=res, hist_str=hist_str, vars_to_derive=vars_to_derive, + ts_dir=ts_dir[case_idx], constit_dict=constit_dict ) # End with @@ -899,6 +974,12 @@ def setup_run_cvdp(self): ) # End if + #intialize objects that might not be declared later + case_name_baseline = None + baseline_ts_loc = None + syears_baseline = None + eyears_baseline = None + # check to see if there is a CAM baseline case. If there is, read in relevant information. if not self.get_basic_info("compare_obs"): case_name_baseline = self.get_baseline_info("cam_case_name") @@ -1020,7 +1101,8 @@ def setup_run_cvdp(self): ######### - def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, overwrite=None): + def derive_variables(self, res=None, hist_str=None, vars_to_derive=None, ts_dir=None, + constit_dict=None, overwrite=None): """ Derive variables acccording to steps given here. Since derivations will depend on the variable, each variable to derive will need its own set of steps below. @@ -1032,32 +1114,50 @@ def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, overwrite """ + #Loop through derived variables for var in vars_to_derive: print(f"\t - deriving time series for {var}") - #Check whether there are parts to derive from and if there is an associated equation - vres = res.get(var, {}) - if "derivable_from" in vres: - constit_list = vres['derivable_from'] - else: - print("WARNING: No constituents listed in defaults config file, moving on") - continue + #Grab list of constituents for this variable + constit_list = constit_dict[var] - #Grab all required time series files for derived var + #Grab all required time series files for derived variable constit_files = [] for constit in constit_list: - if glob.glob(os.path.join(ts_dir, f"*.{constit}.*.nc")): - constit_files.append(glob.glob(os.path.join(ts_dir, f"*.{constit}.*"))[0]) + #Check if the constituent file is present, if so add it to list + if hist_str: + const_glob_str = f"*{hist_str}*.{constit}.*.nc" + else: + const_glob_str = f"*.{constit}.*.nc" + #end if + if glob.glob(os.path.join(ts_dir, const_glob_str)): + constit_files.append(glob.glob(os.path.join(ts_dir, const_glob_str ))[0]) - #Check if all the constituent files were found + #Check if all the necessary constituent files were found if len(constit_files) != len(constit_list): - ermsg = f"Not all constituent files present; {var} cannot be calculated." - ermsg += f" Please remove {var} from diag_var_list or find the relevant CAM files." + ermsg = f"\t ** Not all constituent files present; {var} cannot be calculated." + ermsg += f" Please remove {var} from 'diag_var_list' or find the " + ermsg += "relevant CAM files.\n" print(ermsg) + if constit_files: + #Add what's missing to debug log + dmsg = "create time series:" + dmsg += f"\n\tneeded constituents for derivation of " + dmsg += f"{var}:\n\t\t- {constit_list}\n\tfound constituent file(s) in " + dmsg += f"{Path(constit_files[0]).parent}:\n\t\t" + dmsg += f"- {[Path(f).parts[-1] for f in constit_files if Path(f).is_file()]}" + self.debug_log(dmsg) + else: + dmsg = "create time series:" + dmsg += f"\n\tneeded constituents for derivation of " + dmsg += f"{var}:\n\t\t- {constit_list}\n" + dmsg += f"\tNo constituent(s) found in history files" + self.debug_log(dmsg) + else: #Open a new dataset with all the constituent files/variables ds = xr.open_mfdataset(constit_files) - + # create new file name for derived variable derived_file = constit_files[0].replace(constit_list[0], var) @@ -1066,9 +1166,9 @@ def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, overwrite if overwrite: Path(derived_file).unlink() else: - print( - f"[{__name__}] Warning: '{var}' file was found and overwrite is False. Will use existing file." - ) + msg = f"[{__name__}] Warning: '{var}' file was found " + msg += "and overwrite is False. Will use existing file." + print(msg) continue #NOTE: this will need to be changed when derived equations are more complex! - JR @@ -1079,32 +1179,38 @@ def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, overwrite der_val = 0 for v in constit_list: der_val += ds[v] - + #Set derived variable name and add to dataset der_val.name = var ds[var] = der_val - #Aerosol Calculations - used for zonal plots + #Aerosol Calculations + #---------------------------------------------------------------------------------- #These will be multiplied by rho (density of dry air) ds_pmid_done = False ds_t_done = False - if var in res["aerosol_zonal_list"]: - + + # User-defined defaults might not include aerosol zonal list + azl = res.get("aerosol_zonal_list", []) + if var in azl: + #Only calculate once for all aerosol vars if not ds_pmid_done: ds_pmid = _load_dataset(glob.glob(os.path.join(ts_dir, "*.PMID.*"))[0]) ds_pmid_done = True if not ds_pmid: - errmsg = f"Missing necessary files for dry air density (rho) calculation.\n" - errmsg += "Please make sure 'PMID' is in the CAM run for aerosol calculations" + errmsg = "Missing necessary files for dry air density (rho) " + errmsg += "calculation.\nPlease make sure 'PMID' is in the CAM " + errmsg += "run for aerosol calculations" print(errmsg) continue if not ds_t_done: ds_t = _load_dataset(glob.glob(os.path.join(ts_dir, "*.T.*"))[0]) ds_t_done = True if not ds_t: - errmsg = f"Missing necessary files for dry air density (rho) calculation.\n" - errmsg += "Please make sure 'T' is in the CAM run for aerosol calculations" + errmsg = "Missing necessary files for dry air density (rho) " + errmsg += "calculation.\nPlease make sure 'T' is in the CAM " + errmsg += "run for aerosol calculations" print(errmsg) continue @@ -1114,6 +1220,7 @@ def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, overwrite #Sulfate conversion factor if var == "SO4": ds[var] = ds[var]*(96./115.) + #---------------------------------------------------------------------------------- #Drop all constituents from final saved dataset #These are not necessary because they have their own time series files @@ -1125,7 +1232,8 @@ def derive_variables(self, res=None, vars_to_derive=None, ts_dir=None, overwrite #Helper Function(s) def _load_dataset(fils): """ - This method exists to get an xarray Dataset from input file information that can be passed into the plotting methods. + This method exists to get an xarray Dataset from input file information that + can be passed into the plotting methods. Parameters ---------- @@ -1151,10 +1259,10 @@ def my_formatwarning(msg, *args, **kwargs): if len(fils) == 0: warnings.warn("Input file list is empty.") return None - elif len(fils) > 1: + if len(fils) > 1: return xr.open_mfdataset(fils, combine='by_coords') else: return xr.open_dataset(fils[0]) #End if #End def -######## \ No newline at end of file +######## diff --git a/lib/adf_info.py b/lib/adf_info.py index cdde4d6f3..442bdea9d 100644 --- a/lib/adf_info.py +++ b/lib/adf_info.py @@ -107,9 +107,9 @@ def __init__(self, config_file, debug=False): #Read hist_str (component.hist_num) from the yaml file, or set to default hist_str = self.get_basic_info('hist_str') - #If hist_str is not present, then default to 'cam.h0': + #If hist_str is not present, then default to 'cam.h0a': if not hist_str: - hist_str = 'cam.h0' + hist_str = 'cam.h0a' #End if self.__hist_str = hist_str diff --git a/lib/adf_variable_defaults.yaml b/lib/adf_variable_defaults.yaml index b8ecaa1ee..ae18d2310 100644 --- a/lib/adf_variable_defaults.yaml +++ b/lib/adf_variable_defaults.yaml @@ -78,6 +78,12 @@ default_ptypes: ["Tables","LatLon","LatLon_Vector","Zonal","Meridional", #Dry Air Gas Constant: Rgas: 287.04 #[J/K/Kg]=8.314/0.028965 +#+++++++++++++ +# CAM-CHEM Variables +#+++++++++++++ +#List of variables for CAM-CHEM runs that have different constituents than regular CAM runs +cam_chem_list: ["SOA","SO4"] + #+++++++++++++ # Category: Microphysics #+++++++++++++ @@ -231,7 +237,8 @@ SO4: colorbar: label : '$\mu$g/m3' category: "Aerosols" - derivable_from: ["so4_a1", "so4_a2", "so4_a3", "so4_a5"] + derivable_from: ["so4_a1", "so4_a2", "so4_a3"] + derivable_from_cam_chem: ["so4_a1", "so4_a2", "so4_a3", "so4_a5"] SOA: colormap: "RdBu_r" @@ -244,6 +251,7 @@ SOA: label : '$\mu$g/m3' category: "Aerosols" derivable_from: ["soa_a1", "soa_a2"] + derivable_from_cam_chem: ["soa1_a1", "soa2_a1", "soa3_a1", "soa4_a1", "soa5_a1", "soa1_a2", "soa2_a2", "soa3_a2", "soa4_a2", "soa5_a2"] DUST: colormap: "RdBu_r" @@ -277,8 +285,6 @@ SeaSalt: category: "Aerosols" derivable_from: ["ncl_a1", "ncl_a2", "ncl_a3"] - - #+++++++++++++++++ # Category: Budget #+++++++++++++++++ @@ -374,7 +380,6 @@ H2SO4: N2O: category: "Composition" - #+++++++++++++++++ # Category: Clouds #+++++++++++++++++ @@ -700,7 +705,6 @@ OCNFRAC: LANDFRAC: category: "Surface variables" - #+++++++++++++++++ # Category: State #+++++++++++++++++ @@ -816,6 +820,7 @@ QRS: #+++++++++++++++++ # Category: TOA energy flux #+++++++++++++++++ + RESTOM: colormap: "RdBu_r" contour_levels_range: [-100, 100, 5] @@ -1108,6 +1113,7 @@ OMEGAT: #++++++++++++++ # Category: TEM #++++++++++++++ + uzm: ylim: [1e3,1] units: m s-1 diff --git a/lib/test/unit_tests/test_adf_base.py b/lib/test/unit_tests/test_adf_base.py index b757cbd29..6e13f6ea1 100644 --- a/lib/test/unit_tests/test_adf_base.py +++ b/lib/test/unit_tests/test_adf_base.py @@ -12,6 +12,7 @@ import os import os.path import logging +import glob #Set relevant path variables: _CURRDIR = os.path.abspath(os.path.dirname(__file__)) @@ -44,9 +45,12 @@ def tearDown(self): Remove log files (if they exist). """ - #Remove log file if it exists: - if os.path.exists("ADF_debug.log"): - os.remove("ADF_debug.log") + debug_list = glob.glob("ADF_debug*.log") + + for dfile in debug_list: + #Remove log file if it exists: + if os.path.exists(dfile): + os.remove(dfile) #Close all log streams: @@ -77,11 +81,14 @@ def test_AdfBase_debug_create(self): #Create AdfBase object with debug setting: adf_test = AdfBase(debug=True) + #Grab debug log name + debug_fname = adf_test.debug_fname + #Assert that new object is of the "AdfBase" class: self.assertIsInstance(adf_test, AdfBase) - #Assert that "ADF_debug.log" file exists in local directory: - self.assertTrue(os.path.exists("ADF_debug.log")) + #Assert that ADF debug log file exists in local directory: + self.assertTrue(os.path.exists(debug_fname)) def test_AdfBase_bad_debug(self): @@ -117,8 +124,11 @@ def test_AdfBase_debug_nothing(self): #Call "debug_log" method: adf_test.debug_log("test") + #Grab debug log name + debug_fname = adf_test.debug_fname + #Check that no log file exists: - self.assertFalse(os.path.exists("ADF_debug.log")) + self.assertFalse(os.path.exists(debug_fname)) def test_AdfBase_debug_write(self): @@ -131,17 +141,20 @@ def test_AdfBase_debug_write(self): #Create AdfBase object with debug setting: adf_test = AdfBase(debug=True) + #Grab debug log name + debug_fname = adf_test.debug_fname + #Call "debug_log" method: adf_test.debug_log("test") #Check that debug log exists: - self.assertTrue(os.path.exists("ADF_debug.log")) + self.assertTrue(os.path.exists(debug_fname)) #If debug log exists, then open file: - if os.path.exists("ADF_debug.log"): + if os.path.exists(debug_fname): #Open log file: - with open("ADF_debug.log") as logfil: + with open(debug_fname) as logfil: #Extract file contents: log_text = logfil.read() diff --git a/lib/website_templates/adf_diag.css b/lib/website_templates/adf_diag.css index 291f227aa..a2e146dc7 100644 --- a/lib/website_templates/adf_diag.css +++ b/lib/website_templates/adf_diag.css @@ -261,7 +261,7 @@ table.dataframe thead th{ display: grid; column-gap: 50px; row-gap: 50px; - grid-template-columns: repeat(4, auto); + grid-template-columns: repeat(3, auto); background-color: #e4eef0; padding: 85px; }