Merge pull request ESCOMP#983 from jtruesdal/fv3fmsupdate

cam6_3_157: Update FV3 and FMS externals, new FV3 interface pulls from NOAA-GFDL and now FMS version synced with CESM

Update FV3 to allow syncing FMS version with CESM
ditch NCAR fork of FV3 in favor of pulling in library code from GFDL
Clean up FV3 makefile
closes FMS external version needs to match version used in CESM ESCOMP#950

Externals.cfg

@@ -87,8 +87,7 @@ externals = Externals_CLM.cfg
 required = True
 
 [fms]
-# Older tag than CESM as there is a compilation error mismatch
-tag = fi_20211011
+tag = fi_230818
 protocol = git
 repo_url = https://github.com/ESCOMP/FMS_interface
 local_path = libraries/FMS

Externals_CAM.cfg

@@ -56,11 +56,12 @@ repo_url = https://github.com/ESCOMP/atmospheric_physics
 required = True
 local_path = src/atmos_phys
 
-[atmos_cubed_sphere]
-tag = fv3_cesm.04
+[fv3]
+tag = fv3int_022824
 protocol = git
-repo_url = https://github.com/ESCOMP/FV3_CESM.git
-local_path = src/dynamics/fv3/atmos_cubed_sphere
+repo_url = https://github.com/ESCOMP/CAM_FV3_interface.git
+local_path = src/dynamics/fv3
+externals = Externals_FV3.cfg
 required = True
 
 [mpas]

bld/configure

@@ -2101,6 +2101,7 @@ sub write_fv3core_filepath
     my $camsrcdir   = $cfg_ref->get('cam_dir');
     my $CASEROOT = "$ENV{'CASEROOT'}";
     print $fh "$CASEROOT/SourceMods/src.cam\n";
+    print $fh "$camsrcdir/src/dynamics/fv3/src_override\n";
     print $fh "$camsrcdir/src/dynamics/fv3/microphys\n";
     print $fh "$camsrcdir/src/dynamics/fv3/atmos_cubed_sphere/model\n";
     print $fh "$camsrcdir/src/dynamics/fv3/atmos_cubed_sphere/tools\n";
@@ -2168,7 +2169,7 @@ sub write_filepath
 
     # Weak scaling fix. This has to come before physics/cam and before dycores
     # It also has to come before utils (which is already near the end).
-    if ($dyn eq 'se' or $dyn eq 'mpas') {
+    if ($dyn eq 'se' or $dyn eq 'mpas' or $dyn eq 'fv3') {
       print $fh "$camsrcdir/src/infrastructure\n";
     }
 

cime_config/buildlib

@@ -25,6 +25,50 @@ from CIME.build import get_standard_makefile_args
 
 logger = logging.getLogger(__name__)
 
+###############################################################################
+def _build_fms(caseroot, libroot, bldroot):
+    ###############################################################################
+
+    with Case(caseroot) as case:
+
+        # Only need FMS for fv3 dycore
+        cam_dycore = case.get_value("CAM_DYCORE")
+        srcroot = case.get_value("SRCROOT")
+        if cam_dycore == "fv3":
+            # first check for the external FMS library and build it
+            # Check to see if some other component built it already
+            fmsbuildlib = os.path.join(srcroot, "libraries", "FMS", "buildlib")
+            librootfms = os.path.join(libroot, "libfms.a")
+            if not os.path.exists(librootfms):
+                if case.get_value("DEBUG"):
+                    strdebug = "debug"
+                else:
+                    strdebug = "nodebug"
+
+                if case.get_value("BUILD_THREADED"):
+                    strthread = "threads"
+                else:
+                    strthread = "nothreads"
+
+                mpilib = case.get_value("MPILIB")
+                sharedpath = os.path.join(case.get_value("COMPILER"), mpilib,
+                                          strdebug, strthread, "nuopc")
+                slr = os.path.abspath(case.get_value("SHAREDLIBROOT"))
+                fmsbuildroot = os.path.join(slr, sharedpath)
+                fmsinstallpath = os.path.join(fmsbuildroot, "FMS")
+                install_libfms = os.path.join(fmsinstallpath, "libfms.a")
+
+                if not os.path.exists(install_libfms):
+                    if not os.path.exists(fmsbuildlib):
+                        #todo: call checkout_externals to get this component
+                        expect(False, "FMS external not found")
+                    else:
+                        stat, _, err = run_cmd(f"{fmsbuildlib} {fmsbuildroot} {fmsinstallpath} {caseroot}", verbose=True)
+                        expect(stat==0, f"FMS build Failed {err}")
+
+                if os.path.exists(install_libfms):
+                    shutil.copy(install_libfms, libroot)
+
 ###############################################################################
 def _build_cam(caseroot, libroot, bldroot):
     ###############################################################################
@@ -64,12 +108,11 @@ def _build_cam(caseroot, libroot, bldroot):
             threaded = "threads" if case.get_value("BUILD_THREADED") or case.get_value("FORCE_BUILD_SMP") else "nothreads"
             comp_interface = case.get_value("COMP_INTERFACE")
             fmsbuilddir = os.path.join(
-                slr, compiler, mpilib, debug, threaded, comp_interface)
+                slr, compiler, mpilib, debug, threaded, comp_interface, "FMS")
             user_incldir = '"-I{} -I{} -I{}"'.format(
                 os.path.join(srcroot, "libraries", "FMS", "src", "include"),
                 os.path.join(srcroot, "libraries", "FMS", "src", "mpp", "include"),
-                fmsbuilddir,
-            )
+                fmsbuilddir)
 
         # -------------------------------------------------------
         # Filepath is created in caseroot/camconf by the call
@@ -120,6 +163,7 @@ def _build_cam(caseroot, libroot, bldroot):
 
 def _main_func():
     caseroot, libroot, bldroot = parse_input(sys.argv)
+    _build_fms(caseroot, libroot, bldroot)
     _build_cam(caseroot, libroot, bldroot)
 
 

cime_config/config_pes.xml

@@ -1790,14 +1790,14 @@
       <pes pesize="any" compset="any">
         <comment>none</comment>
 	      <ntasks>
-	        <ntasks_atm>128</ntasks_atm>
-	        <ntasks_lnd>128</ntasks_lnd>
-	        <ntasks_rof>128</ntasks_rof>
-	        <ntasks_ice>128</ntasks_ice>
-	        <ntasks_ocn>128</ntasks_ocn>
-	        <ntasks_glc>128</ntasks_glc>
-	        <ntasks_wav>128</ntasks_wav>
-	        <ntasks_cpl>128</ntasks_cpl>
+	        <ntasks_atm>-3</ntasks_atm>
+	        <ntasks_lnd>-3</ntasks_lnd>
+	        <ntasks_rof>-3</ntasks_rof>
+	        <ntasks_ice>-3</ntasks_ice>
+	        <ntasks_ocn>-3</ntasks_ocn>
+	        <ntasks_glc>-3</ntasks_glc>
+	        <ntasks_wav>-3</ntasks_wav>
+	        <ntasks_cpl>-3</ntasks_cpl>
 	      </ntasks>
 	      <nthrds>
 	        <nthrds_atm>1</nthrds_atm>

cime_config/testdefs/testlist_cam.xml

@@ -301,8 +301,10 @@
     <machines>
       <machine name="derecho" compiler="intel" category="prealpha"/>
       <machine name="derecho" compiler="intel" category="fv3_cam"/>
+      <machine name="izumi" compiler="gnu" category="aux_cam"/>
     </machines>
     <options>
+      <option name="comment" >CAM6-FV3 aquaplanet exact restart test</option>
       <option name="wallclock">00:10:00</option>
     </options>
   </test>

cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3/shell_commands

@@ -2,5 +2,5 @@
 ./xmlchange GLC_NCPL=\$ATM_NCPL
 ./xmlchange CAM_CONFIG_OPTS=' -microphys mg3' --append
 if [ "`./xmlquery ATM_GRID --value`" == "C96" ]; then
-  ./xmlchange NTASKS=-2
+  ./xmlchange NTASKS=-3
 fi

doc/ChangeLog

@@ -1,5 +1,92 @@
 ===============================================================
 
+Tag name: cam6_3_157
+Originator(s): jet
+Date: Apr 17, 2024
+One-line Summary: Update FV3 FMS externals, added FV3_CAM interface external, now importing core FV3 from GFDL
+Github PR URL: https://github.com/ESCOMP/CAM/pull/983
+
+Purpose of changes (include the issue number and title text for each relevant GitHub issue):
+ - Update FV3 to allow syncing FMS version with CESM
+ - Ditch NCAR fork of FV3 in favor of pulling in library code from GFDL
+ - Clean up FV3 makfile
+ - Closes issue #950 : FMS external version needs to match version used in CESM
+
+Describe any changes made to build system:
+ - Replace FV3 fork external with FV3_interface external that inturn imports FV3 from GFDL
+
+Describe any changes made to the namelist: N/A
+
+List any changes to the defaults for the boundary datasets: N/A
+
+Describe any substantial timing or memory changes: N/A
+
+Code reviewed by: cacraig, nusbaume, jedwards
+
+List all files eliminated:
+	src/dynamics/fv3/dimensions_mod.F90
+	src/dynamics/fv3/dp_coupling.F90
+	src/dynamics/fv3/dycore_budget.F90
+	src/dynamics/fv3/dycore.F90
+	src/dynamics/fv3/dyn_comp.F90
+	src/dynamics/fv3/dyn_grid.F90
+	src/dynamics/fv3/interp_mod.F90
+	src/dynamics/fv3/Makefile.in.fv3
+	src/dynamics/fv3/pmgrid.F90
+	src/dynamics/fv3/restart_dynamics.F90
+	src/dynamics/fv3/spmd_dyn.F90
+	src/dynamics/fv3/stepon.F90
+	src/dynamics/fv3/microphys/gfdl_cloud_microphys.F90
+	src/dynamics/fv3/microphys/module_mp_radar.F90
+
+List all files added and what they do: N/A
+
+List all existing files that have been modified, and describe the changes:
+M       Externals.cfg
+          - updated FMS tag
+M       Externals_CAM.cfg
+          - removed FV3 fork external and replace with FV3_CAM_INTERFACE external
+M	bld/configure
+	  - add src_override directory for interfacing GFDL lib code to CAM
+M	cime_config/bldlib
+	  - add bld_fms target to use common FMS library
+M	cime_config/config_pes.xml
+	- update FV3 default C96 PE's for Derecho
+M	cime_config/testdefs/testlist_cam.xml
+	- add izumi gnu fv3 test
+M	cime_config/testdefs/testmods_dirs/cam/outfrq9xs_mg3/shell_commands
+	- fix C96 PE default for derecho
+M	test/system/TR8.sh
+	- add ignore for src_override directory of new FV3_CAM_INTERFACE external
+
+If there were any failures reported from running test_driver.sh on any test
+platform, and checkin with these failures has been OK'd by the gatekeeper,
+then copy the lines from the td.*.status files for the failed tests to the
+appropriate machine below.  All failed tests must be justified.
+
+All (coupled) jobs had errors about MEMCOMP failing due to missing files - to
+be fixed in upcoming CIME tag
+
+Many tests also had TPUTCOMP errors which are not reported here.  The current
+working assumption is that there is an error with the test itself not the CAM code.
+
+derecho/intel/aux_cam:
+  ERP_Ln9.f09_f09_mg17.FCSD_HCO.derecho_intel.cam-outfrq9s (Overall: FAIL) details:
+  SMS_Ld1.f09_f09_mg17.FCHIST_GC.derecho_intel.cam-outfrq1d (Overall: FAIL) details:
+    - pre-existing failures
+
+  ERP_Ln9.C96_C96_mg17.F2000climo.derecho_intel.cam-outfrq9s_mg3 (Overall: DIFF) details:
+    - FV3 diff failures are expected due to lack of a baseline file to compare against.
+
+
+izumi/nag/aux_cam:
+  DAE.f45_f45_mg37.FHS94.izumi_nag.cam-dae (Overall: FAIL) details:
+    - pre-existing failure
+
+izumi/gnu/aux_cam: All PASS
+
+===============================================================
+
 Tag name: cam6_3_156
 Originator(s): fvitt
 Date: 16 Apr 2024