From 4b3e80480242f984f018065a2d971947c1200dcb Mon Sep 17 00:00:00 2001 From: lasseke <57358428+lasseke@users.noreply.github.com> Date: Wed, 23 Feb 2022 11:23:23 +0100 Subject: [PATCH 1/6] Update pip requirements and setup.py, change package version to 0.0.1 --- .../NREC_VM_setup/privileged/requirements_pip.txt | 13 +++++++------ landsites_tools/__init__.py | 2 +- requirements.txt | 3 --- setup.py | 11 ++--------- 4 files changed, 10 insertions(+), 19 deletions(-) delete mode 100644 requirements.txt diff --git a/.machine/NREC_VM_setup/privileged/requirements_pip.txt b/.machine/NREC_VM_setup/privileged/requirements_pip.txt index d4ed15b6..435065ec 100644 --- a/.machine/NREC_VM_setup/privileged/requirements_pip.txt +++ b/.machine/NREC_VM_setup/privileged/requirements_pip.txt @@ -1,7 +1,7 @@ # From CTSM Dockerfile -rst2pdf -sphinx -sphinxcontrib-programoutput +rst2pdf==0.99 +Sphinx==4.4.0 +sphinxcontrib-programoutput==0.17 # Platform dependencies pandas==1.1.5 @@ -10,8 +10,9 @@ ipdb==0.13.8 jupyterlab==3.0.16 matplotlib==3.3.4 xarray==0.16.2 +dask[dataframe]==2021.3.0 netCDF4==1.5.6 cfunits==3.3.3 -pysqlite3 -jupyter -bokeh +pysqlite3==0.4.6 +jupyter==1.0.0 +bokeh==2.3.3 diff --git a/landsites_tools/__init__.py b/landsites_tools/__init__.py index cdf45c58..0fed3840 100644 --- a/landsites_tools/__init__.py +++ b/landsites_tools/__init__.py @@ -1,2 +1,2 @@ """LandsitesTools package.""" -__version__ = '2.0.0' +__version__ = '0.1.0' diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 231f36d3..00000000 --- a/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -numpy==1.19.5 -pandas==1.1.5 -geopandas==0.9.0 diff --git a/setup.py b/setup.py index 1815699f..9a10e8aa 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,9 @@ #!/usr/bin/env python3 """NorESM land sites platform installation script.""" from pathlib import Path -from setuptools import Command, find_packages, setup +from setuptools import find_packages, setup from landsites_tools import __version__ -sys.path.insert(0, os.path.dirname(__file__)) - setup( name='landsites_tools', version=__version__, @@ -18,13 +16,8 @@ download_url='https://github.com/NorESMhub/NorESM_LandSites_Platform', license='GNU General Public License v3.0', cmdclass={'bdist_wheel': None}, - packages=find_packages(include=['landsites_tools','landsites_tools.*']), + packages=find_packages(include=['landsites_tools', 'landsites_tools.*']), python_requires=">=3.6.8", - install_requires=[ - 'numpy>=1.19.5', - 'pandas>=1.1.5', - 'geopandas>=0.9.0', - ], classifiers=[ 'Development Status :: 1 - Planning', 'Environment :: Console', From 78c8f3048e7a66f89220f22e73d81b89dd1ac9df Mon Sep 17 00:00:00 2001 From: lasseke <57358428+lasseke@users.noreply.github.com> Date: Wed, 23 Feb 2022 11:25:16 +0100 Subject: [PATCH 2/6] Change default settings files to use one case, remove tests in notebook --- data/.nlp/default_settings.txt | 2 +- landsites_tools/settings.txt | 2 +- notebooks/plot_example.ipynb | 20 -------------------- 3 files changed, 2 insertions(+), 22 deletions(-) diff --git a/data/.nlp/default_settings.txt b/data/.nlp/default_settings.txt index 271175a1..19ee8c16 100644 --- a/data/.nlp/default_settings.txt +++ b/data/.nlp/default_settings.txt @@ -6,7 +6,7 @@ dir_output = data/output # output root folder, absolute or relative to projec [run] start_time = 2000-01-01 # yyyy-mm-dd hh:mm end_time = 2001-01-01 # yyyy-mm-dd -sites2run = ALP1, ALP2 # data/.dicts/sites.json +sites2run = ALP1 # data/.dicts/sites.json type_run = startup # startup, hybrid, branch, restart type_model = CLM-SP # CLM-SP, CLM-BGC, CLM-FATES, FATES-SP, FATES-nocomp, FATES-hydro initial_file = # initial conditions (empty: cold start) diff --git a/landsites_tools/settings.txt b/landsites_tools/settings.txt index 7eb3011f..a97fd3b9 100644 --- a/landsites_tools/settings.txt +++ b/landsites_tools/settings.txt @@ -6,7 +6,7 @@ dir_output = data/output # output root folder, absolute or relative to projec [run] start_time = 2000-01-01 # yyyy-mm-dd hh:mm end_time = 2001-01-01 # yyyy-mm-dd -sites2run = ALP1, ALP2 # data/.dicts/sites.json +sites2run = ALP1 # data/.dicts/sites.json type_run = startup # startup, hybrid, branch, restart type_model = CLM-SP # CLM-SP, CLM-BGC, CLM-FATES, FATES-SP, FATES-nocomp, FATES-hydro initial_file = # initial conditions (empty: cold start) diff --git a/notebooks/plot_example.ipynb b/notebooks/plot_example.ipynb index 34540d05..db5c1a71 100644 --- a/notebooks/plot_example.ipynb +++ b/notebooks/plot_example.ipynb @@ -72,26 +72,6 @@ "nc_data[\"time\"] = new.indexes[\"time\"].to_datetimeindex()" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "96b38c77-73f7-4681-a7f6-df9acb0555e2", - "metadata": {}, - "outputs": [], - "source": [ - "#test = pd.to_datetime(nc_data[\"time\"], infer_datetime_format=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c8e9f913-fffc-4a21-8ebc-c32498be17c8", - "metadata": {}, - "outputs": [], - "source": [ - "nc_data[var_name].units" - ] - }, { "cell_type": "markdown", "id": "462e4c9d-52f9-40a5-9cbd-36c388192c95", From a9402047847d3b052d3cdc1acc308a15486b580f Mon Sep 17 00:00:00 2001 From: lasseke <57358428+lasseke@users.noreply.github.com> Date: Mon, 28 Feb 2022 13:43:01 +0100 Subject: [PATCH 3/6] Update bokeh app to make cases, dev. stage --- bokeh_app/make_settings/helpers.py | 15 +- bokeh_app/make_settings/main.py | 344 ++++++++++++++---- .../make_settings/static/html/run_header.html | 26 ++ .../{header.html => settings_header.html} | 0 4 files changed, 304 insertions(+), 81 deletions(-) create mode 100644 bokeh_app/make_settings/static/html/run_header.html rename bokeh_app/make_settings/static/html/{header.html => settings_header.html} (100%) diff --git a/bokeh_app/make_settings/helpers.py b/bokeh_app/make_settings/helpers.py index 4bb938b4..7287adb5 100644 --- a/bokeh_app/make_settings/helpers.py +++ b/bokeh_app/make_settings/helpers.py @@ -4,10 +4,12 @@ from pathlib import Path from landsites_tools.utils.interface_settings import SettingsParser + def write_settings_file(file_name, sites2run, dir_cases, dir_clm_input, -dir_output, start_time, end_time, type_run, type_model, pft_indices): + dir_output, start_time, end_time, type_run, type_model, + pft_indices): """ - Create a new settings file with the inputs specified in the bokeh interface. + Create a new settings file with inputs specified in the bokeh interface. Arguments: file_name (string): name of the new settings file """ @@ -16,7 +18,7 @@ def write_settings_file(file_name, sites2run, dir_cases, dir_clm_input, default_settings_path = root_path / 'data/.nlp/' default_save_path = root_path / 'landsites_tools/custom_settings/' - ### Load settings as SettingsParser object + # Load settings as SettingsParser object interface_settings = SettingsParser( default_settings_path / "default_settings.txt" ) @@ -43,8 +45,9 @@ def write_settings_file(file_name, sites2run, dir_cases, dir_clm_input, # Fates settings interface_settings.set_parameter("included_pft_indices", pft_indices) - ### Write settings file - interface_settings.write_settings_file(new_file=True, - path=(default_save_path / f"{file_name}")) + # Write settings file + interface_settings. \ + write_settings_file(new_file=True, + path=(default_save_path / f"{file_name}")) return True diff --git a/bokeh_app/make_settings/main.py b/bokeh_app/make_settings/main.py index 5098ee1b..d05086ea 100644 --- a/bokeh_app/make_settings/main.py +++ b/bokeh_app/make_settings/main.py @@ -1,13 +1,16 @@ #!/usr/bin/python3 """ -Application to create a settings file for the NorESM land sites -platform via interactive widgets. +Application to create a settings file and build/run cases for the NorESM land +sites platform via interactive widgets. """ # General Python imports import json from pathlib import Path from datetime import datetime import geopandas as gpd +import subprocess +from subprocess import PIPE +import glob # Local imports import helpers @@ -16,58 +19,76 @@ # Bokeh imports from bokeh.io import curdoc from bokeh.layouts import column, row, grid -from bokeh.models import Div, CheckboxGroup, DatePicker, RadioButtonGroup, MultiChoice +from bokeh.models import Div, CheckboxGroup, DatePicker, RadioButtonGroup +from bokeh.models import MultiChoice, Panel, Tabs, FileInput, MultiSelect from bokeh.models.widgets import TextInput, Button, Paragraph -################################################################################ +############################################################################### """ File-specific helper functions """ -################################################################################ +############################################################################### -#### Handling input changes and validation +# Handling input changes and validation default_text = "You need to (re-)validate the input before creating the file." + def _input_valid(): create_button.disabled = False output.css_classes = ['valid'] + + def _input_invalid(): create_button.disabled = True output.css_classes = ['invalid'] + + def _input_changed(attr, new, old): output.text = default_text _input_invalid() + ### Paths to roots app_root_dir_path = Path(__file__).parent platform_dir_path = Path(__file__).parents[2] +############################################################################### +############################################################################### +############################################################################### +""" +Make settings tab +""" +############################################################################### +############################################################################### +############################################################################### + + ### Load constraint dictionary with open(platform_dir_path / 'data/.nlp/settings_constraints.json', 'r') \ -as constraints_file: + as constraints_file: constraints_dict = json.load(constraints_file) """ -Define site header as html file. +Set settings header as defined in html file. """ -with open(app_root_dir_path / 'static/html/header.html') as html_file: +with open(app_root_dir_path / 'static/html/settings_header.html') as html_file: html_text = html_file.read() header_div = Div(text=html_text, sizing_mode="stretch_width") -################################################################################ +############################################################################### """ Site selection """ -################################################################################ +############################################################################### # Read site geojson nlp_sites_gdf = gpd.read_file( -platform_dir_path / 'data/.nlp/site_info.geojson') + platform_dir_path / 'data/.nlp/site_info.geojson') site_description_div = Div( -text="

Pick sites

Select the sites you want to simulate "\ -+"(multi-selection possible):", -sizing_mode="stretch_width") + text="

Pick sites

Select the sites you want to simulate " + + "(multi-selection possible):", + sizing_mode="stretch_width") # Site names site_labels = [row["name"] for _, row in nlp_sites_gdf.iterrows()] sites_checkbox_group = CheckboxGroup(labels=site_labels, active=[]) @@ -75,42 +96,45 @@ def _input_changed(attr, new, old): # Reset checks if anything changes sites_checkbox_group.on_change('active', _input_changed) + def _check_sites(): return bool(sites_checkbox_group.active) + sites_section = column(site_description_div, sites_checkbox_group) -################################################################################ +############################################################################### """ Directory paths """ -################################################################################ +############################################################################### input_dir_paths_div = Div( -text="

Set directory paths

Skip this section to use the default paths "\ -+ "as indicated. Paths can be absolute or relative to the repository root.
", -sizing_mode="stretch_width") + text="

Set directory paths

Skip this section to use the default paths " + + "as indicated. Paths can be absolute or relative to the repository root.
", + sizing_mode="stretch_width") clm_in_div = Div( -text='''Enter the path for the CLM input: ''', -css_classes=['item-description'], sizing_mode='stretch_width') + text='''Enter the path for the CLM input: ''', + css_classes=['item-description'], sizing_mode='stretch_width') clm_input_dir_text_input = TextInput(value="data/input/clm") clm_input_dir_text_input.on_change('value', _input_changed) cases_div = Div( -text='''Enter the path for the case folders: ''', -css_classes=['item-description'], sizing_mode='stretch_width') + text='''Enter the path for the case folders: ''', + css_classes=['item-description'], sizing_mode='stretch_width') cases_dir_text_input = TextInput(value="data/cases") cases_dir_text_input.on_change('value', _input_changed) output_div = Div( -text='''Enter the path for the model output: ''', -css_classes=['item-description'], sizing_mode='stretch_width') + text='''Enter the path for the model output: ''', + css_classes=['item-description'], sizing_mode='stretch_width') output_dir_text_input = TextInput(value="data/output") output_dir_text_input.on_change('value', _input_changed) + def _check_paths(): for input in [clm_input_dir_text_input, cases_dir_text_input, - output_dir_text_input]: + output_dir_text_input]: try: if pth.is_valid_path(Path(input.value), type="dir"): continue @@ -120,31 +144,34 @@ def _check_paths(): except: return False return True + + # Layout path_input_section = column(input_dir_paths_div, -row(clm_in_div, clm_input_dir_text_input), -row(cases_div, cases_dir_text_input), -row(output_div, output_dir_text_input) -) + row(clm_in_div, clm_input_dir_text_input), + row(cases_div, cases_dir_text_input), + row(output_div, output_dir_text_input) + ) -################################################################################ +############################################################################### """ Simulation period """ -################################################################################ +############################################################################### period_div = Div( -text="

Simulation period

", -sizing_mode="stretch_width") + text="

Simulation period

", + sizing_mode="stretch_width") start_date_picker = DatePicker(title='Select simulation start date:', -value="2000-01-01", min_date="1900-01-01", max_date="2020-12-30") + value="2000-01-01", min_date="1900-01-01", max_date="2020-12-30") start_date_picker.on_change('value', _input_changed) end_date_picker = DatePicker(title='Select simulation end date:', -value="2001-01-01", min_date="1900-01-02", max_date="2020-12-31") + value="2001-01-01", min_date="1900-01-02", max_date="2020-12-31") end_date_picker.on_change('value', _input_changed) dates_section = column(period_div, row(start_date_picker, end_date_picker)) + def _check_dates(): global start_date, end_date start_date = datetime.strptime(start_date_picker.value, '%Y-%m-%d') @@ -153,74 +180,77 @@ def _check_dates(): # Is end date after start date? return ((end_date-start_date).days > 0) -################################################################################ + +############################################################################### """ CLM settings """ -################################################################################ +############################################################################### clm_div = Div(text="

CLM settings

", -sizing_mode="stretch_width") + sizing_mode="stretch_width") ### Radio buttons - run type type_run_div = Div(text='''Select the model run type.''', -css_classes=['item-description'], sizing_mode='stretch_width') + css_classes=['item-description'], sizing_mode='stretch_width') type_run_labels = constraints_dict['type_run']['valid_values'] type_run_radio = RadioButtonGroup(labels=type_run_labels, active=0) #show(runtype_radio_buttons) ### Radio buttons - run mode type_model_div = Div( -text='''Select the model run mode.''', -css_classes=['item-description'], sizing_mode='stretch_width') + text='''Select the model run mode.''', + css_classes=['item-description'], sizing_mode='stretch_width') type_model_labels = constraints_dict['type_model']['valid_values'] type_model_radio = RadioButtonGroup(labels=type_model_labels, active=0) ### Column for display clm_settings_section = column(clm_div, -type_run_div, type_run_radio, -type_model_div, type_model_radio) + type_run_div, type_run_radio, + type_model_div, type_model_radio) -################################################################################ +############################################################################### """ FATES settings """ -################################################################################ +############################################################################### fates_div = Div( -text="

FATES settings

OBS! Only used if model run mode includes FATES.", -sizing_mode="stretch_width") + text="

FATES settings

OBS! Only used if model run mode includes FATES.", + sizing_mode="stretch_width") ### PFTs to include -pft_div = Div(text='''Select all the ''' \ -+ '''''' \ -+ '''PFTs you want to include.''', -css_classes=['item-description'], sizing_mode='stretch_width') +pft_div = Div(text='''Select all the ''' + + '''''' + + '''PFTs you want to include.''', + css_classes=['item-description'], sizing_mode='stretch_width') pft_options = constraints_dict['pft_indices']['long_names'] pft_multi_choice = MultiChoice(value=pft_options, options=pft_options) fates_settings_section = column(fates_div, pft_div, pft_multi_choice) -################################################################################ +############################################################################### """ Button to create settings file """ -################################################################################ +############################################################################### create_button_div = Div( -text="

Create file

Enter a name for the new settings file:
", -sizing_mode="stretch_width") + text="

Create file

Enter a name for the new settings file:
", + sizing_mode="stretch_width") input = TextInput(value="file-name") check_input_button = Button(label="Check input", button_type='primary', -disabled=False) + disabled=False) create_button = Button(label="Create file!", button_type='primary', -disabled=True) + disabled=True) output = Paragraph(text=default_text, css_classes=["invalid"]) create_file_section = column(create_button_div, input, -row(check_input_button, create_button), output) + row(check_input_button, create_button), output) """ Updates when buttons are clicked """ + + def check_input(): """Button handler for checking the user input.""" @@ -234,13 +264,14 @@ def check_input(): pass elif not _check_paths(): output.text = "One of the specified paths does not exist. " \ - + "See console for details." + + "See console for details." _input_invalid() pass else: output.text = "Input checks passed. You can now create the file." _input_valid() + def update(): """Button handler for creating a new file.""" @@ -272,16 +303,16 @@ def update(): # Write settings file helpers.write_settings_file( - file_name=str(file_name), - sites2run=",".join(site_names), - dir_cases=str(clm_input_dir_text_input.value), - dir_clm_input=str(cases_dir_text_input.value), - dir_output=str(output_dir_text_input.value), - start_time=str(start_date), - end_time=str(end_date), - type_run=str(type_run_labels[type_run_radio.active]), - type_model=str(type_model_labels[type_model_radio.active]), - pft_indices=",".join([str(x) for x in pft_indices]) + file_name=str(file_name), + sites2run=",".join(site_names), + dir_cases=str(clm_input_dir_text_input.value), + dir_clm_input=str(cases_dir_text_input.value), + dir_output=str(output_dir_text_input.value), + start_time=str(start_date), + end_time=str(end_date), + type_run=str(type_run_labels[type_run_radio.active]), + type_model=str(type_model_labels[type_model_radio.active]), + pft_indices=",".join([str(x) for x in pft_indices]) ) output.text = f"Created '{file_name}'!" @@ -289,15 +320,166 @@ def update(): output.text = f"Error when creating {file_name}." raise + check_input_button.on_click(check_input) create_button.on_click(update) -################################################################################ +############################################################################### +############################################################################### +############################################################################### +""" +Build and run cases tab +""" +############################################################################### +############################################################################### +############################################################################### + +""" +Set settings header as defined in html file. +""" +with open(app_root_dir_path / 'static/html/run_header.html') as html_file: + run_html_text = html_file.read() + +run_header_div = Div(text=run_html_text, sizing_mode="stretch_width") + +############################################################################### +""" +Build cases +""" +############################################################################### + + +root_path = Path(__file__).parents[2] +settings_path = root_path / 'landsites_tools/custom_settings/' +script_path = root_path / 'landsites_tools/simulation/' + + +def get_settings_files(): + """Get all settings files in 'landsites_tools/custom_settings' in a list""" + + file_path_str_list = glob.glob(str(settings_path) + "/*.txt") + + file_paths_list = [Path(fname) for fname in file_path_str_list] + file_names_list = [p.name for p in file_paths_list] + + return(file_paths_list, file_names_list) + + +def retrieve_select_options(settings_file_names): + + make_cases_select_options = \ + [(str(idx), f_name) for idx, f_name in enumerate(settings_file_names)] + + return(make_cases_select_options) + + +make_cases_div = Div( + text="

Make case(s)

", + sizing_mode="stretch_width") + +# Settings file picker +make_cases_select_div = Div(text="Select one settings file.", + css_classes=['item-description'], + sizing_mode='stretch_width') + +# List for picking a file +settings_file_paths, settings_file_names = get_settings_files() +make_cases_select_options = retrieve_select_options(settings_file_names) + +make_cases_select = MultiSelect(value=[], + options=make_cases_select_options) + +# Make cases button +make_cases_button = Button(label="Make case(s)", button_type='primary', + disabled=False) +refresh_button = Button(label="Refresh files", button_type='default', + disabled=False) + +make_cases_output = Div(text="Click the button above to make new cases.", + css_classes=["item-description"], + style={'overflow': 'auto', + 'width': '100%', + 'height': '250px'} + ) + +""" +JS_scroll_bottom = CustomJS(args={'bokeh_div': make_cases_output}, code=''' +function updateScroll(){ + + var element = document.getElementsByClassName("build-log")[0].children[0]; + + element.scrollTop = element.scrollHeight; + console.log(element.scrollTop) +} +updateScroll(); + +''') + +make_cases_output.js_on_change("text", JS_scroll_bottom) + +#make_cases_output.js_on_event(X, ) +""" + + +def make_cases(): + + if len(make_cases_select.value) != 1: + make_cases_output.text = \ + "You must select one single settings file!" + return + + print(make_cases_select.value) + file_idx = int(make_cases_select.value[0]) + print(file_idx) + + make_cases_output.text = \ + f"Making cases for {settings_file_names[file_idx]}" \ + + "...
This step will take approx. 10 minutes per case. Check " \ + + "your local terminal for progress information." + + make_cases_settings_file = settings_file_paths[file_idx] + + curdoc().hold("combine") + + proc = subprocess.run( + f"python3 {script_path}/make_cases.py -f {make_cases_settings_file}", + shell=True, check=True, stdout=PIPE, stderr=PIPE) + + curdoc().unhold() + + make_cases_output.text = \ + f"Finished for {settings_file_names[file_idx]}!
" \ + + "You can find logging information below.

" \ + + f"{(proc.stdout).decode('utf-8')}" + + +def refresh_file_list(): + settings_file_paths, settings_file_names = get_settings_files() + make_cases_select_options = retrieve_select_options(settings_file_names) + + make_cases_select.options = make_cases_select_options + make_cases_select.value = [] + + +make_cases_button.on_click(make_cases) +refresh_button.on_click(refresh_file_list) + +make_cases_section = column(make_cases_div, + make_cases_select_div, + make_cases_select, + row(make_cases_button, refresh_button), + make_cases_output + ) + + +############################################################################### """ Specify site layout """ -################################################################################ -layout = grid([ +############################################################################### + +# Define layouts +layout_settings = grid([ [header_div], [sites_section], [path_input_section], @@ -308,5 +490,17 @@ def update(): ], sizing_mode=None) +layout_running = grid([ + [run_header_div], + [make_cases_section] + ], + sizing_mode=None) + +# Define tabs +tab_1 = Panel(child=layout_settings, title="Create settings") +tab_2 = Panel(child=layout_running, title="Build/run cases") + +page_tabs = Tabs(tabs=[tab_1, tab_2]) + # add the layout to curdoc -curdoc().add_root(layout) +curdoc().add_root(page_tabs) diff --git a/bokeh_app/make_settings/static/html/run_header.html b/bokeh_app/make_settings/static/html/run_header.html new file mode 100644 index 00000000..847efb78 --- /dev/null +++ b/bokeh_app/make_settings/static/html/run_header.html @@ -0,0 +1,26 @@ + + +

Build and run cases from existing settings files

+
+The NorESM landsites platform uses settings files to build and run customized +single-point NorESM model experiments for selected locations. In this section, +you can specify a previously created settings file to + +make cases, i.e., automatically download the input data and create and +adapt directories that contain the files required by the underlying model. In +addition, you can subsequently + +run cases, i.e., start the actual model simulation.

+ +Attention! There are two important constraints +you need to be aware of: + +
diff --git a/bokeh_app/make_settings/static/html/header.html b/bokeh_app/make_settings/static/html/settings_header.html similarity index 100% rename from bokeh_app/make_settings/static/html/header.html rename to bokeh_app/make_settings/static/html/settings_header.html From a639ec24489db9a0aed36703a2b0c86808f1fb98 Mon Sep 17 00:00:00 2001 From: lasseke <57358428+lasseke@users.noreply.github.com> Date: Mon, 28 Feb 2022 23:48:13 +0100 Subject: [PATCH 4/6] Finalized working draft of making/running cases in bokeh app --- bokeh_app/make_settings/main.py | 174 ++++++++++++++++++++++++-------- 1 file changed, 133 insertions(+), 41 deletions(-) diff --git a/bokeh_app/make_settings/main.py b/bokeh_app/make_settings/main.py index d05086ea..8de0f0d6 100644 --- a/bokeh_app/make_settings/main.py +++ b/bokeh_app/make_settings/main.py @@ -9,7 +9,7 @@ from datetime import datetime import geopandas as gpd import subprocess -from subprocess import PIPE +from subprocess import PIPE, STDOUT import glob # Local imports @@ -20,8 +20,9 @@ from bokeh.io import curdoc from bokeh.layouts import column, row, grid from bokeh.models import Div, CheckboxGroup, DatePicker, RadioButtonGroup -from bokeh.models import MultiChoice, Panel, Tabs, FileInput, MultiSelect +from bokeh.models import MultiChoice, Panel, Tabs, MultiSelect, CustomJS from bokeh.models.widgets import TextInput, Button, Paragraph +from bokeh.events import ButtonClick ############################################################################### """ @@ -344,7 +345,7 @@ def update(): ############################################################################### """ -Build cases +Make cases """ ############################################################################### @@ -357,20 +358,21 @@ def update(): def get_settings_files(): """Get all settings files in 'landsites_tools/custom_settings' in a list""" - file_path_str_list = glob.glob(str(settings_path) + "/*.txt") + global settings_file_paths + global settings_file_names - file_paths_list = [Path(fname) for fname in file_path_str_list] - file_names_list = [p.name for p in file_paths_list] + file_path_str_list = glob.glob(str(settings_path) + "/*.txt") - return(file_paths_list, file_names_list) + settings_file_paths = [Path(fname) for fname in file_path_str_list] + settings_file_names = [p.name for p in settings_file_paths] def retrieve_select_options(settings_file_names): - make_cases_select_options = \ + settings_files_select_options = \ [(str(idx), f_name) for idx, f_name in enumerate(settings_file_names)] - return(make_cases_select_options) + return(settings_files_select_options) make_cases_div = Div( @@ -383,11 +385,11 @@ def retrieve_select_options(settings_file_names): sizing_mode='stretch_width') # List for picking a file -settings_file_paths, settings_file_names = get_settings_files() -make_cases_select_options = retrieve_select_options(settings_file_names) +get_settings_files() +settings_files_select_options = retrieve_select_options(settings_file_names) make_cases_select = MultiSelect(value=[], - options=make_cases_select_options) + options=settings_files_select_options) # Make cases button make_cases_button = Button(label="Make case(s)", button_type='primary', @@ -395,30 +397,25 @@ def retrieve_select_options(settings_file_names): refresh_button = Button(label="Refresh files", button_type='default', disabled=False) -make_cases_output = Div(text="Click the button above to make new cases.", +make_cases_output = Div(text="Click the button above to create and build " + + "the cases specified in the chosen settings file.", css_classes=["item-description"], style={'overflow': 'auto', 'width': '100%', - 'height': '250px'} + 'height': '200px'} ) """ -JS_scroll_bottom = CustomJS(args={'bokeh_div': make_cases_output}, code=''' -function updateScroll(){ - - var element = document.getElementsByClassName("build-log")[0].children[0]; - - element.scrollTop = element.scrollHeight; - console.log(element.scrollTop) -} -updateScroll(); - +JS_executing_feedback = CustomJS(args={'bokeh_div': make_cases_output, + 'make_button': make_cases_button}, + code=''' +bokeh_div.text="Making cases...
This step will take approx. 10 minutes " + +"per case.
Logging information will be printed here when the process " + +"is finished."; +make_button.disabled=true; ''') - -make_cases_output.js_on_change("text", JS_scroll_bottom) - -#make_cases_output.js_on_event(X, ) """ +#make_cases_button.js_on_event(ButtonClick, JS_executing_feedback) def make_cases(): @@ -428,40 +425,50 @@ def make_cases(): "You must select one single settings file!" return - print(make_cases_select.value) file_idx = int(make_cases_select.value[0]) - print(file_idx) make_cases_output.text = \ f"Making cases for {settings_file_names[file_idx]}" \ + "...
This step will take approx. 10 minutes per case. Check " \ + "your local terminal for progress information." - make_cases_settings_file = settings_file_paths[file_idx] + make_cases_button.disabled = True + + +def make_cases_subprocess(): - curdoc().hold("combine") + file_idx = int(make_cases_select.value[0]) + + make_cases_settings_file = settings_file_paths[file_idx] proc = subprocess.run( f"python3 {script_path}/make_cases.py -f {make_cases_settings_file}", - shell=True, check=True, stdout=PIPE, stderr=PIPE) - - curdoc().unhold() + shell=True, check=True) # , stdout=PIPE, stderr=STDOUT) + # new_line = '\n' make_cases_output.text = \ f"Finished for {settings_file_names[file_idx]}!
" \ - + "You can find logging information below.

" \ - + f"{(proc.stdout).decode('utf-8')}" + + "You can find logging information in the terminal." + # + f"{(proc.stdout).decode('utf-8').replace(new_line, '
')}" + + make_cases_button.disabled = False def refresh_file_list(): - settings_file_paths, settings_file_names = get_settings_files() - make_cases_select_options = retrieve_select_options(settings_file_names) - make_cases_select.options = make_cases_select_options + get_settings_files() + settings_files_select_options = retrieve_select_options( + settings_file_names) + + make_cases_select.options = settings_files_select_options make_cases_select.value = [] + run_cases_select.options = settings_files_select_options + run_cases_select.value = [] + make_cases_button.on_click(make_cases) +make_cases_button.on_click(make_cases_subprocess) refresh_button.on_click(refresh_file_list) make_cases_section = column(make_cases_div, @@ -472,6 +479,90 @@ def refresh_file_list(): ) +############################################################################### +""" +Run cases +""" +############################################################################### + +run_cases_div = Div( + text="

Run case(s)

", + sizing_mode="stretch_width") + +# Settings file picker +run_cases_select_div = Div(text="Select one settings file.", + css_classes=['item-description'], + sizing_mode='stretch_width') + +# List for picking a file +run_cases_select = MultiSelect(value=[], + options=settings_files_select_options) + +# Make cases button +run_cases_button = Button(label="Run case(s)", button_type='primary', + disabled=False) +refresh_button_2 = Button(label="Refresh files", button_type='default', + disabled=False) + +run_cases_output = Div(text="Click the button above to run the cases " + + "specified in the chosen settings file.", + css_classes=["item-description"], + style={'overflow': 'auto', + 'width': '100%', + 'height': '200px'} + ) + + +def run_cases(): + + file_idx = int(run_cases_select.value[0]) + + run_cases_output.text = \ + f"Running cases in {settings_file_names[file_idx]}" \ + + "...

This step will take a while. The run time depends on " \ + + "the model settings, e.g., the simulation period or the computer " \ + + "resource allocation in docker-compose.yml. Expect roughly 15 min " \ + + "for each simulation year. Check your local terminal for " \ + + "additional progress information." + + run_cases_button.disabled = True + + +def run_cases_subprocess(): + + if len(run_cases_select.value) != 1: + run_cases_output.text = \ + "You must select one single settings file!" + return + + file_idx = int(run_cases_select.value[0]) + + run_cases_settings_file = settings_file_paths[file_idx] + + proc = subprocess.run( + f"python3 {script_path}/run_cases.py -f {run_cases_settings_file} -q", + shell=True, check=True) # , stdout=PIPE, stderr=STDOUT) + + # new_line = '\n' + run_cases_output.text = \ + f"Finished for {settings_file_names[file_idx]}!
" \ + + "You can find logging information in the terminal." + # + f"{(proc.stdout).decode('utf-8').replace(new_line, '
')}" + + run_cases_button.disabled = False + + +run_cases_button.on_click(run_cases) +run_cases_button.on_click(run_cases_subprocess) +refresh_button_2.on_click(refresh_file_list) + +run_cases_section = column(run_cases_div, + run_cases_select_div, + run_cases_select, + row(run_cases_button, refresh_button_2), + run_cases_output + ) + ############################################################################### """ Specify site layout @@ -492,7 +583,8 @@ def refresh_file_list(): layout_running = grid([ [run_header_div], - [make_cases_section] + [make_cases_section], + [run_cases_section] ], sizing_mode=None) From e897dc0ea7c9f199918bab330b85678eb3a8466b Mon Sep 17 00:00:00 2001 From: lasseke <57358428+lasseke@users.noreply.github.com> Date: Mon, 7 Mar 2022 11:50:59 +0100 Subject: [PATCH 5/6] Started restructuring platform default settings, dev. stage --- data/.nlp/settings_constraints.json | 13 ++- data/.nlp/site_info.geojson | 24 +++--- data/.nlp/sites.json | 86 ------------------- .../supported_default_settings/clm.json} | 0 .../constraint_files}/clm_history_vars.txt | 0 .../supported_default_settings/fates.json | 39 +++++++++ .../supported_default_settings/general.json | 76 ++++++++++++++++ 7 files changed, 133 insertions(+), 105 deletions(-) delete mode 100644 data/.nlp/sites.json rename data/{tst => .nlp/supported_default_settings/clm.json} (100%) rename data/.nlp/{ => supported_default_settings/constraint_files}/clm_history_vars.txt (100%) create mode 100644 data/.nlp/supported_default_settings/fates.json create mode 100644 data/.nlp/supported_default_settings/general.json diff --git a/data/.nlp/settings_constraints.json b/data/.nlp/settings_constraints.json index 1db22cde..a6c3404e 100644 --- a/data/.nlp/settings_constraints.json +++ b/data/.nlp/settings_constraints.json @@ -7,8 +7,7 @@ "type_model": { "type": "str", "valid_in_txt_file": 0, - "valid_values": ["CLM-SP", "CLM-BGC", "CLM-FATES", "FATES-SP", - "FATES-nocomp", "FATES-hydro"] + "valid_values": ["CLM_5.0_FATES", "CLM_5.0_SP", "CLM_5.0_BGC"] }, "frequency_output": { "type": "str", @@ -37,13 +36,13 @@ "long_names": [ "Broadleaf evergreen tropical tree", "Needleleaf evergreen extratropical tree", - "Needleleaf colddecid extratropical tree", + "Needleleaf cold-decidious extratropical tree", "Broadleaf evergreen extratropical tree", - "Broadleaf hydrodecid tropical tree", - "Broadleaf colddecid extratropical tree", + "Broadleaf hydro-decidious tropical tree", + "Broadleaf cold-decidious extratropical tree", "Broadleaf evergreen extratropical shrub", - "Broadleaf hydrodecid extratropical shrub", - "Broadleaf colddecid extratropical shrub", + "Broadleaf hydro-decidious extratropical shrub", + "Broadleaf cold-decidious extratropical shrub", "Arctic C3 grass", "Cool C3 grass", "C4 grass" diff --git a/data/.nlp/site_info.geojson b/data/.nlp/site_info.geojson index 98deac9c..c9c52887 100644 --- a/data/.nlp/site_info.geojson +++ b/data/.nlp/site_info.geojson @@ -10,7 +10,7 @@ "properties": { "name": "ALP1", "res": "1x1", - "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_ALP1.tar", + "default_input_url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_ALP1.tar", "group": "SeedClim" } }, @@ -23,7 +23,7 @@ "properties": { "name": "ALP2", "res": "1x1", - "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_ALP2.tar", + "default_input_url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_ALP2.tar", "group": "SeedClim" } }, @@ -36,7 +36,7 @@ "properties": { "name": "ALP3", "res": "1x1", - "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_ALP3.tar", + "default_input_url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_ALP3.tar", "group": "SeedClim" } }, @@ -49,7 +49,7 @@ "properties": { "name": "ALP4", "res": "1x1", - "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_ALP4.tar", + "default_input_url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_ALP4.tar", "group": "SeedClim" } }, @@ -62,7 +62,7 @@ "properties": { "name": "SUB1", "res": "1x1", - "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_SUB1.tar", + "default_input_url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_SUB1.tar", "group": "SeedClim" } }, @@ -75,7 +75,7 @@ "properties": { "name": "SUB2", "res": "1x1", - "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_SUB2.tar", + "default_input_url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_SUB2.tar", "group": "SeedClim" } }, @@ -88,7 +88,7 @@ "properties": { "name": "SUB3", "res": "1x1", - "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_SUB3.tar", + "default_input_url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_SUB3.tar", "group": "SeedClim" } }, @@ -101,7 +101,7 @@ "properties": { "name": "SUB4", "res": "1x1", - "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_SUB4.tar", + "default_input_url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_SUB4.tar", "group": "SeedClim" } }, @@ -114,7 +114,7 @@ "properties": { "name": "BOR1", "res": "1x1", - "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_BOR1.tar", + "default_input_url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_BOR1.tar", "group": "SeedClim" } }, @@ -127,7 +127,7 @@ "properties": { "name": "BOR2", "res": "1x1", - "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_BOR2.tar", + "default_input_url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_BOR2.tar", "group": "SeedClim" } }, @@ -140,7 +140,7 @@ "properties": { "name": "BOR3", "res": "1x1", - "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_BOR3.tar", + "default_input_url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_BOR3.tar", "group": "SeedClim" } }, @@ -153,7 +153,7 @@ "properties": { "name": "BOR4", "res": "1x1", - "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_BOR4.tar", + "default_input_url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_BOR4.tar", "group": "SeedClim" } } diff --git a/data/.nlp/sites.json b/data/.nlp/sites.json deleted file mode 100644 index 6d961e60..00000000 --- a/data/.nlp/sites.json +++ /dev/null @@ -1,86 +0,0 @@ -{ - "ALP1": { - "name":"ALP1", - "res":"1x1", - "lat":"61.0243", - "lon":"8.12343", - "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_ALP1.tar" - }, - "ALP2": { - "name":"ALP2", - "res":"1x1", - "lat":"60.8231", - "lon":"7.27596", - "url":"https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_ALP2.tar" - }, - "ALP3": { - "name":"ALP3", - "res":"1x1", - "lat":"60.8328", - "lon":"7.17561", - "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_ALP3.tar" - }, - "ALP4": { - "name":"ALP4", - "res":"1x1", - "lat":"60.9335", - "lon":"6.41504", - "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_ALP4.tar" - }, - "SUB1": { - "name":"SUB1", - "res":"1x1", - "lat":"60.8203", - "lon":"8.70466", - "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_SUB1.tar" - }, - "SUB2": { - "name":"SUB2", - "res":"1x1", - "lat":"60.8760", - "lon":"7.17666", - "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_SUB2.tar" - }, - "SUB3": { - "name":"SUB3", - "res":"1x1", - "lat":"61.0866", - "lon":"6.63028", - "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_SUB3.tar" - }, - "SUB4": { - "name":"SUB4", - "res":"1x1", - "lat":"60.5445", - "lon":"6.51468", - "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_SUB4.tar" - }, - "BOR1": { - "name":"BOR1", - "res":"1x1", - "lat":"61.0355", - "lon":"9.07876", - "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_BOR1.tar" - }, - "BOR2": { - "name":"BOR2", - "res":"1x1", - "lat":"60.8803", - "lon":"7.16982", - "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_BOR2.tar" - }, - "BOR3": { - "name":"BOR3", - "res":"1x1", - "lat":"60.6652", - "lon":"6.33738", - "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_BOR3.tar" - }, - "BOR4": { - "name":"BOR4", - "res":"1x1", - "lat":"60.6901", - "lon":"5.96487", - "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_BOR4.tar" - } -} diff --git a/data/tst b/data/.nlp/supported_default_settings/clm.json similarity index 100% rename from data/tst rename to data/.nlp/supported_default_settings/clm.json diff --git a/data/.nlp/clm_history_vars.txt b/data/.nlp/supported_default_settings/constraint_files/clm_history_vars.txt similarity index 100% rename from data/.nlp/clm_history_vars.txt rename to data/.nlp/supported_default_settings/constraint_files/clm_history_vars.txt diff --git a/data/.nlp/supported_default_settings/fates.json b/data/.nlp/supported_default_settings/fates.json new file mode 100644 index 00000000..ca9441eb --- /dev/null +++ b/data/.nlp/supported_default_settings/fates.json @@ -0,0 +1,39 @@ +{ + "pft_indices": { + "type": "list", + "valid_in_txt_file": 0, + "valid_values": [1,2,3,4,5,6,7,8,9,10,11,12], + "long_names": [ + "Broadleaf evergreen tropical tree", + "Needleleaf evergreen extratropical tree", + "Needleleaf cold-decidious extratropical tree", + "Broadleaf evergreen extratropical tree", + "Broadleaf hydro-decidious tropical tree", + "Broadleaf cold-decidious extratropical tree", + "Broadleaf evergreen extratropical shrub", + "Broadleaf hydro-decidious extratropical shrub", + "Broadleaf cold-decidious extratropical shrub", + "Arctic C3 grass", + "Cool C3 grass", + "C4 grass" + ] + }, + "stomatal_conductance_model": { + "type": "int", + "valid_in_txt_file": 0, + "valid_values": [1,2], + "long_names": [ + "Ball-Berry", + "Medlyn" + ] + }, + "organ_hydraulic_transfer_model": { + "type": "int", + "valid_in_txt_file": 0, + "valid_values": [1,2], + "long_names": [ + "Christofferson et al. 2016 (TFS)", + "Van Genuchten 1980" + ] + } +} diff --git a/data/.nlp/supported_default_settings/general.json b/data/.nlp/supported_default_settings/general.json new file mode 100644 index 00000000..7d15074a --- /dev/null +++ b/data/.nlp/supported_default_settings/general.json @@ -0,0 +1,76 @@ +{ + "paths": + { + "case_input_data_root": { + "ctsm_parameter" : "DIN_LOC_ROOT", + "xml_file": "env_run.xml", + "default": "$ENV{CESMDATAROOT}/inputdata", + "description": "Path to a case's input data root directory. The input + data file structure and naming should follow the standards specified in + https://svn-ccsm-inputdata.cgd.ucar.edu/trunk/inputdata/." + }, + "case_atm_forcing": { + "ctsm_parameter" : "DIN_LOC_ROOT_CLMFORC", + "xml_file": "env_run.xml", + "default": "$DIN_LOC_ROOT/atm/datm7/GSWP3v1", + "description": "Path to the directory containing a case's atmospheric + forcing NetCDF files, i.e, climate input data. The default is single-point + forcing derived from the GSWP3 data set." + }, + "case_output_dir": { + "ctsm_parameter" : "DOUT_S_ROOT", + "xml_file" : "env_run.xml", + "default" : "$ENV{HOME}/archive/$CASE", + "description": "Here." + }, + }, + "simulation": + { + "start_time" : { + "ctsm_parameter" : "RUN_STARTDATE", + "xml_file" : "env_run.xml", + "default" : "2000-01-01" + }, + "n_days" : { + "ctsm_parameter" : "STOP_N", + "xml_file" : "env_run.xml", + "default" : 365 + }, + "type_run": { + "ctsm_parameter" : "RUN_TYPE", + "xml_file" : "env_run.xml", + "default" : "startup" + }, + "compset": { + "CLM_5.0_FATES": + { + "long_name" : "2000_DATM%1PTGSWP3_CLM50%FATES_SICE_SOCN_MOSART_SGLC_SWAV", + "description": "CLM version 5.0, using the Functionally Assembled Ecosystem + Simulator (FATES) to model demographic vegetation dynamics." + }, + "CLM_5.0_SP": + { + "long_name" : "2000_DATM%1PTGSWP3_CLM50%SP_SICE_SOCN_MOSART_SGLC_SWAV", + "description": "CLM version 5.0, using MODIS satellite measurements + (SP = satellite phenology) to prescribe vegetation properties." + }, + "CLM_5.0_BGC": + { + "long_name" : "2000_DATM%GSWP3v1_CLM50%FATES_SICE_SOCN_MOSART_SGLC_SWAV", + "description": "CLM version 5.0, using Biogeochemistry (BGC) mode to + model carbon and nitrogen dynamics." + } + } + }, + "nlp_general": + { + "STOP_OPTION": { + "xml_file": "env_run.xml", + "value": "ndays" + }, + "STOP_DATE": { + "xml_file": "env_run.xml", + "value": -999 + } + } +} From 3f4d92df0c4217ad3c8ecccf6eff68a6f68bd99f Mon Sep 17 00:00:00 2001 From: lasseke <57358428+lasseke@users.noreply.github.com> Date: Mon, 7 Mar 2022 12:09:48 +0100 Subject: [PATCH 6/6] Revert adding parameter restructuring in wrong branch --- .../clm_history_vars.txt | 0 data/.nlp/settings_constraints.json | 13 +-- data/.nlp/site_info.geojson | 24 +++--- data/.nlp/sites.json | 86 +++++++++++++++++++ .../supported_default_settings/fates.json | 39 --------- .../supported_default_settings/general.json | 76 ---------------- .../clm.json => tst} | 0 7 files changed, 105 insertions(+), 133 deletions(-) rename data/.nlp/{supported_default_settings/constraint_files => }/clm_history_vars.txt (100%) create mode 100644 data/.nlp/sites.json delete mode 100644 data/.nlp/supported_default_settings/fates.json delete mode 100644 data/.nlp/supported_default_settings/general.json rename data/{.nlp/supported_default_settings/clm.json => tst} (100%) diff --git a/data/.nlp/supported_default_settings/constraint_files/clm_history_vars.txt b/data/.nlp/clm_history_vars.txt similarity index 100% rename from data/.nlp/supported_default_settings/constraint_files/clm_history_vars.txt rename to data/.nlp/clm_history_vars.txt diff --git a/data/.nlp/settings_constraints.json b/data/.nlp/settings_constraints.json index a6c3404e..1db22cde 100644 --- a/data/.nlp/settings_constraints.json +++ b/data/.nlp/settings_constraints.json @@ -7,7 +7,8 @@ "type_model": { "type": "str", "valid_in_txt_file": 0, - "valid_values": ["CLM_5.0_FATES", "CLM_5.0_SP", "CLM_5.0_BGC"] + "valid_values": ["CLM-SP", "CLM-BGC", "CLM-FATES", "FATES-SP", + "FATES-nocomp", "FATES-hydro"] }, "frequency_output": { "type": "str", @@ -36,13 +37,13 @@ "long_names": [ "Broadleaf evergreen tropical tree", "Needleleaf evergreen extratropical tree", - "Needleleaf cold-decidious extratropical tree", + "Needleleaf colddecid extratropical tree", "Broadleaf evergreen extratropical tree", - "Broadleaf hydro-decidious tropical tree", - "Broadleaf cold-decidious extratropical tree", + "Broadleaf hydrodecid tropical tree", + "Broadleaf colddecid extratropical tree", "Broadleaf evergreen extratropical shrub", - "Broadleaf hydro-decidious extratropical shrub", - "Broadleaf cold-decidious extratropical shrub", + "Broadleaf hydrodecid extratropical shrub", + "Broadleaf colddecid extratropical shrub", "Arctic C3 grass", "Cool C3 grass", "C4 grass" diff --git a/data/.nlp/site_info.geojson b/data/.nlp/site_info.geojson index c9c52887..98deac9c 100644 --- a/data/.nlp/site_info.geojson +++ b/data/.nlp/site_info.geojson @@ -10,7 +10,7 @@ "properties": { "name": "ALP1", "res": "1x1", - "default_input_url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_ALP1.tar", + "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_ALP1.tar", "group": "SeedClim" } }, @@ -23,7 +23,7 @@ "properties": { "name": "ALP2", "res": "1x1", - "default_input_url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_ALP2.tar", + "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_ALP2.tar", "group": "SeedClim" } }, @@ -36,7 +36,7 @@ "properties": { "name": "ALP3", "res": "1x1", - "default_input_url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_ALP3.tar", + "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_ALP3.tar", "group": "SeedClim" } }, @@ -49,7 +49,7 @@ "properties": { "name": "ALP4", "res": "1x1", - "default_input_url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_ALP4.tar", + "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_ALP4.tar", "group": "SeedClim" } }, @@ -62,7 +62,7 @@ "properties": { "name": "SUB1", "res": "1x1", - "default_input_url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_SUB1.tar", + "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_SUB1.tar", "group": "SeedClim" } }, @@ -75,7 +75,7 @@ "properties": { "name": "SUB2", "res": "1x1", - "default_input_url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_SUB2.tar", + "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_SUB2.tar", "group": "SeedClim" } }, @@ -88,7 +88,7 @@ "properties": { "name": "SUB3", "res": "1x1", - "default_input_url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_SUB3.tar", + "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_SUB3.tar", "group": "SeedClim" } }, @@ -101,7 +101,7 @@ "properties": { "name": "SUB4", "res": "1x1", - "default_input_url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_SUB4.tar", + "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_SUB4.tar", "group": "SeedClim" } }, @@ -114,7 +114,7 @@ "properties": { "name": "BOR1", "res": "1x1", - "default_input_url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_BOR1.tar", + "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_BOR1.tar", "group": "SeedClim" } }, @@ -127,7 +127,7 @@ "properties": { "name": "BOR2", "res": "1x1", - "default_input_url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_BOR2.tar", + "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_BOR2.tar", "group": "SeedClim" } }, @@ -140,7 +140,7 @@ "properties": { "name": "BOR3", "res": "1x1", - "default_input_url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_BOR3.tar", + "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_BOR3.tar", "group": "SeedClim" } }, @@ -153,7 +153,7 @@ "properties": { "name": "BOR4", "res": "1x1", - "default_input_url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_BOR4.tar", + "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_BOR4.tar", "group": "SeedClim" } } diff --git a/data/.nlp/sites.json b/data/.nlp/sites.json new file mode 100644 index 00000000..6d961e60 --- /dev/null +++ b/data/.nlp/sites.json @@ -0,0 +1,86 @@ +{ + "ALP1": { + "name":"ALP1", + "res":"1x1", + "lat":"61.0243", + "lon":"8.12343", + "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_ALP1.tar" + }, + "ALP2": { + "name":"ALP2", + "res":"1x1", + "lat":"60.8231", + "lon":"7.27596", + "url":"https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_ALP2.tar" + }, + "ALP3": { + "name":"ALP3", + "res":"1x1", + "lat":"60.8328", + "lon":"7.17561", + "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_ALP3.tar" + }, + "ALP4": { + "name":"ALP4", + "res":"1x1", + "lat":"60.9335", + "lon":"6.41504", + "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_ALP4.tar" + }, + "SUB1": { + "name":"SUB1", + "res":"1x1", + "lat":"60.8203", + "lon":"8.70466", + "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_SUB1.tar" + }, + "SUB2": { + "name":"SUB2", + "res":"1x1", + "lat":"60.8760", + "lon":"7.17666", + "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_SUB2.tar" + }, + "SUB3": { + "name":"SUB3", + "res":"1x1", + "lat":"61.0866", + "lon":"6.63028", + "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_SUB3.tar" + }, + "SUB4": { + "name":"SUB4", + "res":"1x1", + "lat":"60.5445", + "lon":"6.51468", + "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_SUB4.tar" + }, + "BOR1": { + "name":"BOR1", + "res":"1x1", + "lat":"61.0355", + "lon":"9.07876", + "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_BOR1.tar" + }, + "BOR2": { + "name":"BOR2", + "res":"1x1", + "lat":"60.8803", + "lon":"7.16982", + "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_BOR2.tar" + }, + "BOR3": { + "name":"BOR3", + "res":"1x1", + "lat":"60.6652", + "lon":"6.33738", + "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_BOR3.tar" + }, + "BOR4": { + "name":"BOR4", + "res":"1x1", + "lat":"60.6901", + "lon":"5.96487", + "url": "https://ns2806k.webs.sigma2.no/EMERALD/EMERALD_platform/inputdata_fates_platform/inputdata_version2.0.0_BOR4.tar" + } +} diff --git a/data/.nlp/supported_default_settings/fates.json b/data/.nlp/supported_default_settings/fates.json deleted file mode 100644 index ca9441eb..00000000 --- a/data/.nlp/supported_default_settings/fates.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "pft_indices": { - "type": "list", - "valid_in_txt_file": 0, - "valid_values": [1,2,3,4,5,6,7,8,9,10,11,12], - "long_names": [ - "Broadleaf evergreen tropical tree", - "Needleleaf evergreen extratropical tree", - "Needleleaf cold-decidious extratropical tree", - "Broadleaf evergreen extratropical tree", - "Broadleaf hydro-decidious tropical tree", - "Broadleaf cold-decidious extratropical tree", - "Broadleaf evergreen extratropical shrub", - "Broadleaf hydro-decidious extratropical shrub", - "Broadleaf cold-decidious extratropical shrub", - "Arctic C3 grass", - "Cool C3 grass", - "C4 grass" - ] - }, - "stomatal_conductance_model": { - "type": "int", - "valid_in_txt_file": 0, - "valid_values": [1,2], - "long_names": [ - "Ball-Berry", - "Medlyn" - ] - }, - "organ_hydraulic_transfer_model": { - "type": "int", - "valid_in_txt_file": 0, - "valid_values": [1,2], - "long_names": [ - "Christofferson et al. 2016 (TFS)", - "Van Genuchten 1980" - ] - } -} diff --git a/data/.nlp/supported_default_settings/general.json b/data/.nlp/supported_default_settings/general.json deleted file mode 100644 index 7d15074a..00000000 --- a/data/.nlp/supported_default_settings/general.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "paths": - { - "case_input_data_root": { - "ctsm_parameter" : "DIN_LOC_ROOT", - "xml_file": "env_run.xml", - "default": "$ENV{CESMDATAROOT}/inputdata", - "description": "Path to a case's input data root directory. The input - data file structure and naming should follow the standards specified in - https://svn-ccsm-inputdata.cgd.ucar.edu/trunk/inputdata/." - }, - "case_atm_forcing": { - "ctsm_parameter" : "DIN_LOC_ROOT_CLMFORC", - "xml_file": "env_run.xml", - "default": "$DIN_LOC_ROOT/atm/datm7/GSWP3v1", - "description": "Path to the directory containing a case's atmospheric - forcing NetCDF files, i.e, climate input data. The default is single-point - forcing derived from the GSWP3 data set." - }, - "case_output_dir": { - "ctsm_parameter" : "DOUT_S_ROOT", - "xml_file" : "env_run.xml", - "default" : "$ENV{HOME}/archive/$CASE", - "description": "Here." - }, - }, - "simulation": - { - "start_time" : { - "ctsm_parameter" : "RUN_STARTDATE", - "xml_file" : "env_run.xml", - "default" : "2000-01-01" - }, - "n_days" : { - "ctsm_parameter" : "STOP_N", - "xml_file" : "env_run.xml", - "default" : 365 - }, - "type_run": { - "ctsm_parameter" : "RUN_TYPE", - "xml_file" : "env_run.xml", - "default" : "startup" - }, - "compset": { - "CLM_5.0_FATES": - { - "long_name" : "2000_DATM%1PTGSWP3_CLM50%FATES_SICE_SOCN_MOSART_SGLC_SWAV", - "description": "CLM version 5.0, using the Functionally Assembled Ecosystem - Simulator (FATES) to model demographic vegetation dynamics." - }, - "CLM_5.0_SP": - { - "long_name" : "2000_DATM%1PTGSWP3_CLM50%SP_SICE_SOCN_MOSART_SGLC_SWAV", - "description": "CLM version 5.0, using MODIS satellite measurements - (SP = satellite phenology) to prescribe vegetation properties." - }, - "CLM_5.0_BGC": - { - "long_name" : "2000_DATM%GSWP3v1_CLM50%FATES_SICE_SOCN_MOSART_SGLC_SWAV", - "description": "CLM version 5.0, using Biogeochemistry (BGC) mode to - model carbon and nitrogen dynamics." - } - } - }, - "nlp_general": - { - "STOP_OPTION": { - "xml_file": "env_run.xml", - "value": "ndays" - }, - "STOP_DATE": { - "xml_file": "env_run.xml", - "value": -999 - } - } -} diff --git a/data/.nlp/supported_default_settings/clm.json b/data/tst similarity index 100% rename from data/.nlp/supported_default_settings/clm.json rename to data/tst