From 698b373653f71831997560a5e31edb8300cd7e82 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 15 Sep 2023 13:26:58 -0700 Subject: [PATCH 01/57] Removes default coverage reporting and offloads to actions --- .github/workflows/testing.yml | 4 ++-- setup.cfg | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index e92b0b8c337..949f4c04fa6 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -129,7 +129,7 @@ jobs: init_cime - pytest -vvv --machine docker --no-fortran-run CIME/tests/test_unit* + pytest -vvv --cov=CIME --machine docker --no-fortran-run CIME/tests/test_unit* # Run system tests system-testing: @@ -182,7 +182,7 @@ jobs: conda activate base - pytest -vvv --machine docker --no-fortran-run --no-teardown CIME/tests/test_sys* + pytest -vvv --cov=CIME --machine docker --no-fortran-run --no-teardown CIME/tests/test_sys* - name: Create testing log archive if: ${{ failure() }} shell: bash diff --git a/setup.cfg b/setup.cfg index 772767f44b9..1c4058ebd85 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,7 +16,6 @@ console_scripts = [tool:pytest] junit_family=xunit2 -addopts = --cov=CIME --cov-report term-missing --cov-report html:test_coverage/html --cov-report xml:test_coverage/coverage.xml -s python_files = test_*.py testpaths = CIME/tests From e144fea5b591ef21cc11e6134552be906545ff81 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 15 Sep 2023 13:29:10 -0700 Subject: [PATCH 02/57] Adds test_coverage to ignore list --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 58e9dd92b66..f6351cf8996 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ scripts/Tools/JENKINS* components libraries share +test_coverage/** From 2245e742ee274b58dee13345bcec6feb495387a8 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Tue, 19 Sep 2023 17:06:58 -0700 Subject: [PATCH 03/57] Adds similar line as TPUTCOMP to inform user of difference --- CIME/SystemTests/system_tests_common.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index f15fbe959e0..7f9bbbd338e 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -775,6 +775,10 @@ def _compare_memory(self): tolerance = self._case.get_value("TEST_MEMLEAK_TOLERANCE") if tolerance is None: tolerance = 0.1 + comment = "MEMCOMP: Memory usage highwater has changed by {:.2f}% relative to baseline".format( + diff * 100 + ) + append_testlog(comment, self._orig_caseroot) if ( diff < tolerance and self._test_status.get_status(MEMCOMP_PHASE) is None From a6b8d8f0d56849c8cf61f042bce157f71456feb1 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Wed, 20 Sep 2023 13:07:09 -0700 Subject: [PATCH 04/57] Updates how performance baselines are written and compared to tests --- CIME/SystemTests/system_tests_common.py | 344 ++++++++++++++++++------ CIME/tests/test_unit_system_tests.py | 267 ++++++++++++++++++ 2 files changed, 533 insertions(+), 78 deletions(-) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index 7f9bbbd338e..eb367e33cee 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -751,6 +751,9 @@ def _get_latest_cpl_logs(self): return lastcpllogs def _compare_memory(self): + """ + Compares current test memory usage to baseline. + """ with self._test_status: # compare memory usage to baseline baseline_name = self._case.get_value("BASECMP_CASE") @@ -758,44 +761,68 @@ def _compare_memory(self): self._case.get_value("BASELINE_ROOT"), baseline_name ) newestcpllogfiles = self._get_latest_cpl_logs() - if len(newestcpllogfiles) > 0: - memlist = self._get_mem_usage(newestcpllogfiles[0]) + + # do we need this loop? when is there more than one coupler file? for cpllog in newestcpllogfiles: - m = re.search(r"/({}.*.log).*.gz".format(self._cpllog), cpllog) - if m is not None: - baselog = os.path.join(basecmp_dir, m.group(1)) + ".gz" - if baselog is None or not os.path.isfile(baselog): - # for backward compatibility - baselog = os.path.join(basecmp_dir, self._cpllog + ".log") - if os.path.isfile(baselog) and len(memlist) > 3: - blmem = self._get_mem_usage(baselog) - blmem = 0 if blmem == [] else blmem[-1][1] - curmem = memlist[-1][1] - diff = 0.0 if blmem == 0 else (curmem - blmem) / blmem - tolerance = self._case.get_value("TEST_MEMLEAK_TOLERANCE") - if tolerance is None: - tolerance = 0.1 - comment = "MEMCOMP: Memory usage highwater has changed by {:.2f}% relative to baseline".format( - diff * 100 - ) - append_testlog(comment, self._orig_caseroot) - if ( - diff < tolerance - and self._test_status.get_status(MEMCOMP_PHASE) is None - ): - self._test_status.set_status(MEMCOMP_PHASE, TEST_PASS_STATUS) - elif ( - self._test_status.get_status(MEMCOMP_PHASE) != TEST_FAIL_STATUS - ): - comment = "Error: Memory usage increase >{:d}% from baseline's {:f} to {:f}".format( - int(tolerance * 100), blmem, curmem - ) - self._test_status.set_status( - MEMCOMP_PHASE, TEST_FAIL_STATUS, comments=comment - ) - append_testlog(comment, self._orig_caseroot) + try: + baseline_mem = self._read_baseline_mem(basecmp_dir) + except FileNotFoundError as e: + logger.debug("Could not read baseline memory usage: %s", e) + + continue + else: + self._compare_current_memory(baseline_mem, cpllog) + + def _compare_current_memory(self, baseline, cpllog): + """ + Compares current test memory usage against ``baseline``. + + If the difference cannot be calculated then it's defaulted to "0.0". + + Parameters + ---------- + baseline : float + Baseline throughput value. + cpllog : str + Path to the current coupler log. + """ + memlist = self._get_mem_usage(cpllog) + + if len(memlist) > 3: + try: + current = memlist[-1][1] + except IndexError: + current = 0 + + try: + diff = (current - baseline) / baseline + except ZeroDivisionError: + diff = 0.0 + + tolerance = self._case.get_value("TEST_MEMLEAK_TOLERANCE") + if tolerance is None: + tolerance = 0.1 + comment = "MEMCOMP: Memory usage highwater has changed by {:.2f}% relative to baseline".format( + diff * 100 + ) + append_testlog(comment, self._orig_caseroot) + if diff < tolerance and self._test_status.get_status(MEMCOMP_PHASE) is None: + self._test_status.set_status(MEMCOMP_PHASE, TEST_PASS_STATUS) + elif self._test_status.get_status(MEMCOMP_PHASE) != TEST_FAIL_STATUS: + comment = "Error: Memory usage increase >{:d}% from baseline's {:f} to {:f}".format( + int(tolerance * 100), baseline, current + ) + self._test_status.set_status( + MEMCOMP_PHASE, TEST_FAIL_STATUS, comments=comment + ) + append_testlog(comment, self._orig_caseroot) + else: + logger.debug("Found less than 4 memory usage data points") def _compare_throughput(self): + """ + Compares current test throughput to baseline. + """ with self._test_status: # compare memory usage to baseline baseline_name = self._case.get_value("BASECMP_CASE") @@ -803,50 +830,68 @@ def _compare_throughput(self): self._case.get_value("BASELINE_ROOT"), baseline_name ) newestcpllogfiles = self._get_latest_cpl_logs() + # do we need this loop? when is there more than one coupler file? for cpllog in newestcpllogfiles: - m = re.search(r"/({}.*.log).*.gz".format(self._cpllog), cpllog) - if m is not None: - baselog = os.path.join(basecmp_dir, m.group(1)) + ".gz" - if baselog is None or not os.path.isfile(baselog): - # for backward compatibility - baselog = os.path.join(basecmp_dir, self._cpllog) - - if os.path.isfile(baselog): - # compare throughput to baseline - current = self._get_throughput(cpllog) - baseline = self._get_throughput(baselog) - # comparing ypd so bigger is better - if baseline is not None and current is not None: - diff = (baseline - current) / baseline - tolerance = self._case.get_value("TEST_TPUT_TOLERANCE") - if tolerance is None: - tolerance = 0.1 - expect( - tolerance > 0.0, - "Bad value for throughput tolerance in test", - ) - comment = "TPUTCOMP: Computation time changed by {:.2f}% relative to baseline".format( - diff * 100 - ) - append_testlog(comment, self._orig_caseroot) - if ( - diff < tolerance - and self._test_status.get_status(THROUGHPUT_PHASE) is None - ): - self._test_status.set_status( - THROUGHPUT_PHASE, TEST_PASS_STATUS - ) - elif ( - self._test_status.get_status(THROUGHPUT_PHASE) - != TEST_FAIL_STATUS - ): - comment = "Error: TPUTCOMP: Computation time increase > {:d}% from baseline".format( - int(tolerance * 100) - ) - self._test_status.set_status( - THROUGHPUT_PHASE, TEST_FAIL_STATUS, comments=comment - ) - append_testlog(comment, self._orig_caseroot) + try: + baseline_tput = self._read_baseline_tput(basecmp_dir) + except (FileNotFoundError, IndexError, ValueError) as e: + logger.debug("Could not read baseline throughput: %s", e) + + # Default behavior was to skip comparison if baseline file did not exist or no values found + continue + else: + self._compare_current_throughput(baseline_tput, cpllog) + + def _compare_current_throughput(self, baseline, cpllog): + """ + Compares current test throughput against ``baseline``. + + If the difference cannot be calculated then the phase is skipped. + + Parameters + ---------- + baseline : float + Baseline throughput value. + cpllog : str + Path to the current coupler log. + """ + current = self._get_throughput(cpllog) + + try: + # comparing ypd so bigger is better + diff = (baseline - current) / baseline + except ValueError: + # Should we default the diff to 0.0 as with _compare_current_memory? + logger.debug( + "Could not determine change in throughput between baseline %s and current %s", + baseline, + current, + ) + else: + tolerance = self._case.get_value("TEST_TPUT_TOLERANCE") + if tolerance is None: + tolerance = 0.1 + expect( + tolerance > 0.0, + "Bad value for throughput tolerance in test", + ) + comment = "TPUTCOMP: Computation time changed by {:.2f}% relative to baseline".format( + diff * 100 + ) + append_testlog(comment, self._orig_caseroot) + if ( + diff < tolerance + and self._test_status.get_status(THROUGHPUT_PHASE) is None + ): + self._test_status.set_status(THROUGHPUT_PHASE, TEST_PASS_STATUS) + elif self._test_status.get_status(THROUGHPUT_PHASE) != TEST_FAIL_STATUS: + comment = "Error: TPUTCOMP: Computation time increase > {:d}% from baseline".format( + int(tolerance * 100) + ) + self._test_status.set_status( + THROUGHPUT_PHASE, TEST_FAIL_STATUS, comments=comment + ) + append_testlog(comment, self._orig_caseroot) def _compare_baseline(self): """ @@ -900,6 +945,149 @@ def _generate_baseline(self): preserve_meta=False, ) + self._write_baseline_tput(basegen_dir, cpllog) + + self._write_baseline_mem(basegen_dir, cpllog) + + def _write_baseline_tput(self, baseline_dir, cpllog): + """ + Writes baseline throughput to file. + + A "-1" indicates that no throughput data was available from the coupler log. + + Parameters + ---------- + baseline_dir : str + Path to the baseline directory. + cpllog : str + Path to the current coupler log. + """ + tput_file = os.path.join(baseline_dir, "cpl-tput.log") + + tput = self._get_throughput(cpllog) + + with open(tput_file, "w") as fd: + fd.write("# Throughput in simulated years per compute day\n") + fd.write( + "# A -1 indicates no throughput data was available from the test\n" + ) + + if tput is None: + fd.write("-1") + else: + fd.write(str(tput)) + + def _write_baseline_mem(self, baseline_dir, cpllog): + """ + Writes baseline memory usage highwater to file. + + A "-1" indicates that no memory usage data was available from the coupler log. + + Parameters + ---------- + baseline_dir : str + Path to the baseline directory. + cpllog : str + Path to the current coupler log. + """ + mem_file = os.path.join(baseline_dir, "cpl-mem.log") + + mem = self._get_mem_usage(cpllog) + + with open(mem_file, "w") as fd: + fd.write("# Memory usage highwater\n") + fd.write( + "# A -1 indicates no memory usage data was available from the test\n" + ) + + try: + fd.write(str(mem[-1][1])) + except IndexError: + fd.write("-1") + + def _read_baseline_tput(self, baseline_dir): + """ + Reads throughput baseline. + + Parameters + ---------- + baseline_dir : str + Path to the baseline directory. + + Returns + ------- + float + Value of the throughput. + + Raises + ------ + FileNotFoundError + If baseline file does not exist. + IndexError + If no throughput value is found. + ValueError + If throughput value is not a float. + """ + return self._read_baseline_value(os.path.join(baseline_dir, "cpl-tput.log")) + + def _read_baseline_mem(self, baseline_dir): + """ + Reads memory usage highwater baseline. + + The default behvaior was to return 0 if no usage data was found. + + Parameters + ---------- + baseline_dir : str + Path to the baseline directory. + + Returns + ------- + float + Value of the highwater memory usage. + + Raises + ------ + FileNotFoundError + If baseline file does not exist. + """ + try: + memory = self._read_baseline_value( + os.path.join(baseline_dir, "cpl-mem.log") + ) + except (IndexError, ValueError): + memory = 0 + + return memory + + def _read_baseline_value(self, baseline_file): + """ + Read baseline value from file. + + Parameters + ---------- + baseline_file : str + Path to baseline file. + + Returns + ------- + float + Baseline value. + + Raises + ------ + FileNotFoundError + If ``baseline_file`` is not found. + IndexError + If not values are present in ``baseline_file``. + ValueError + If value in ``baseline_file`` is not a float. + """ + with open(baseline_file) as fd: + lines = [x for x in fd.readlines() if not x.startswith("#")] + + return float(lines[0]) + class FakeTest(SystemTestsCommon): """ diff --git a/CIME/tests/test_unit_system_tests.py b/CIME/tests/test_unit_system_tests.py index 3bd091900e3..e456b675047 100644 --- a/CIME/tests/test_unit_system_tests.py +++ b/CIME/tests/test_unit_system_tests.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import os +import tempfile +import gzip from re import A import unittest from unittest import mock @@ -10,8 +12,273 @@ from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo from CIME.SystemTests.system_tests_compare_n import SystemTestsCompareN +CPLLOG = """ + tStamp_write: model date = 00010102 0 wall clock = 2023-09-19 19:39:42 avg dt = 0.33 dt = 0.33 + memory_write: model date = 00010102 0 memory = 1673.89 MB (highwater) 387.77 MB (usage) (pe= 0 comps= cpl ATM LND ICE OCN GLC ROF WAV IAC ESP) + tStamp_write: model date = 00010103 0 wall clock = 2023-09-19 19:39:42 avg dt = 0.33 dt = 0.33 + memory_write: model date = 00010103 0 memory = 1673.89 MB (highwater) 390.09 MB (usage) (pe= 0 comps= cpl ATM LND ICE OCN GLC ROF WAV IAC ESP) + tStamp_write: model date = 00010104 0 wall clock = 2023-09-19 19:39:42 avg dt = 0.33 dt = 0.33 + memory_write: model date = 00010104 0 memory = 1673.89 MB (highwater) 391.64 MB (usage) (pe= 0 comps= cpl ATM LND ICE OCN GLC ROF WAV IAC ESP) + tStamp_write: model date = 00010105 0 wall clock = 2023-09-19 19:39:43 avg dt = 0.33 dt = 0.33 + memory_write: model date = 00010105 0 memory = 1673.89 MB (highwater) 392.67 MB (usage) (pe= 0 comps= cpl ATM LND ICE OCN GLC ROF WAV IAC ESP) + tStamp_write: model date = 00010106 0 wall clock = 2023-09-19 19:39:43 avg dt = 0.33 dt = 0.33 + memory_write: model date = 00010106 0 memory = 1673.89 MB (highwater) 393.44 MB (usage) (pe= 0 comps= cpl ATM LND ICE OCN GLC ROF WAV IAC ESP) + +(seq_mct_drv): =============== SUCCESSFUL TERMINATION OF CPL7-e3sm =============== +(seq_mct_drv): =============== at YMD,TOD = 00010106 0 =============== +(seq_mct_drv): =============== # simulated days (this run) = 5.000 =============== +(seq_mct_drv): =============== compute time (hrs) = 0.000 =============== +(seq_mct_drv): =============== # simulated years / cmp-day = 719.635 =============== +(seq_mct_drv): =============== pes min memory highwater (MB) 851.957 =============== +(seq_mct_drv): =============== pes max memory highwater (MB) 1673.891 =============== +(seq_mct_drv): =============== pes min memory last usage (MB) 182.742 =============== +(seq_mct_drv): =============== pes max memory last usage (MB) 393.441 =============== +""" + + +def create_mock_case(tempdir, idx=None, cpllog_data=None): + if idx is None: + idx = 0 + + case = mock.MagicMock() + + caseroot = Path(tempdir, str(idx), "caseroot") + baseline_root = caseroot.parent / "baselines" + run_dir = caseroot / "run" + run_dir.mkdir(parents=True, exist_ok=False) + + if cpllog_data is not None: + cpllog = run_dir / "cpl.log.gz" + + with gzip.open(cpllog, "w") as fd: + fd.write(cpllog_data.encode("utf-8")) + + case.get_latest_cpl_log.return_value = str(cpllog) + + hist_file = run_dir / "cpl.hi.2023-01-01.nc" + hist_file.touch() + + case.get_env.return_value.get_latest_hist_files.return_value = [str(hist_file)] + + case.get_compset_components.return_value = [] + + return case, caseroot, baseline_root, run_dir + class TestCaseSubmit(unittest.TestCase): + @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") + def test_compare_throughput(self, append_testlog): + with tempfile.TemporaryDirectory() as tempdir: + case, caseroot, baseline_root, run_dir = create_mock_case( + tempdir, cpllog_data=CPLLOG + ) + + base_case, base_caseroot, _, _ = create_mock_case( + tempdir, idx=1, cpllog_data=CPLLOG + ) + + case.get_value.side_effect = ( + str(caseroot), + "ERIO.ne30_g16_rx1.A.docker_gnu", + "mct", + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + str(baseline_root), + str(run_dir), + 0.05, + ) + + baseline_dir = baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" + baseline_dir.mkdir(parents=True, exist_ok=False) + baseline_mem = baseline_dir / "cpl-mem.log" + + with open(baseline_mem, "w") as fd: + fd.write(str("1673.89")) + + common = SystemTestsCommon(case) + + common._compare_memory() + + assert common._test_status.get_overall_test_status() == ("PASS", None) + + append_testlog.assert_any_call( + "MEMCOMP: Memory usage highwater has changed by 0.00% relative to baseline", + str(caseroot), + ) + + @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") + def test_compare_throughput_fail(self, append_testlog): + with tempfile.TemporaryDirectory() as tempdir: + case, caseroot, baseline_root, run_dir = create_mock_case( + tempdir, cpllog_data=CPLLOG + ) + + base_case, base_caseroot, _, _ = create_mock_case( + tempdir, idx=1, cpllog_data=CPLLOG + ) + + case.get_value.side_effect = ( + str(caseroot), + "ERIO.ne30_g16_rx1.A.docker_gnu", + "mct", + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + str(baseline_root), + str(run_dir), + 0.05, + ) + + baseline_dir = baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" + baseline_dir.mkdir(parents=True, exist_ok=False) + baseline_mem = baseline_dir / "cpl-mem.log" + + with open(baseline_mem, "w") as fd: + fd.write(str("1473.89")) + + common = SystemTestsCommon(case) + + common._compare_memory() + + assert common._test_status.get_overall_test_status() == ("PASS", None) + + append_testlog.assert_any_call( + "MEMCOMP: Memory usage highwater has changed by 13.57% relative to baseline", + str(caseroot), + ) + append_testlog.assert_any_call( + "Error: Memory usage increase >5% from baseline's 1473.890000 to 1673.890000", + str(caseroot), + ) + + @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") + def test_compare_memory(self, append_testlog): + with tempfile.TemporaryDirectory() as tempdir: + case, caseroot, baseline_root, run_dir = create_mock_case( + tempdir, cpllog_data=CPLLOG + ) + + base_case, base_caseroot, _, _ = create_mock_case( + tempdir, idx=1, cpllog_data=CPLLOG + ) + + case.get_value.side_effect = ( + str(caseroot), + "ERIO.ne30_g16_rx1.A.docker_gnu", + "mct", + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + str(baseline_root), + str(run_dir), + 0.05, + ) + + baseline_dir = baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" + baseline_dir.mkdir(parents=True, exist_ok=False) + baseline_tput = baseline_dir / "cpl-tput.log" + + with open(baseline_tput, "w") as fd: + fd.write(str("719.635")) + + common = SystemTestsCommon(case) + + common._compare_throughput() + + assert common._test_status.get_overall_test_status() == ("PASS", None) + + append_testlog.assert_any_call( + "TPUTCOMP: Computation time changed by 0.00% relative to baseline", + str(caseroot), + ) + + @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") + def test_compare_memory_fail(self, append_testlog): + with tempfile.TemporaryDirectory() as tempdir: + case, caseroot, baseline_root, run_dir = create_mock_case( + tempdir, cpllog_data=CPLLOG + ) + + base_case, base_caseroot, _, _ = create_mock_case( + tempdir, idx=1, cpllog_data=CPLLOG + ) + + case.get_value.side_effect = ( + str(caseroot), + "ERIO.ne30_g16_rx1.A.docker_gnu", + "mct", + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + str(baseline_root), + str(run_dir), + 0.05, + ) + + baseline_dir = baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" + baseline_dir.mkdir(parents=True, exist_ok=False) + baseline_tput = baseline_dir / "cpl-tput.log" + + with open(baseline_tput, "w") as fd: + fd.write(str("900.635")) + + common = SystemTestsCommon(case) + + common._compare_throughput() + + assert common._test_status.get_overall_test_status() == ("PASS", None) + + append_testlog.assert_any_call( + "TPUTCOMP: Computation time changed by 20.10% relative to baseline", + str(caseroot), + ) + append_testlog.assert_any_call( + "Error: TPUTCOMP: Computation time increase > 5% from baseline", + str(caseroot), + ) + + def test_generate_baseline(self): + with tempfile.TemporaryDirectory() as tempdir: + case, caseroot, baseline_root, run_dir = create_mock_case( + tempdir, cpllog_data=CPLLOG + ) + + case.get_value.side_effect = ( + str(caseroot), + "ERIO.ne30_g16_rx1.A.docker_gnu", + "mct", + str(run_dir), + "case.std", + str(baseline_root), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + "ERIO.ne30_g16_rx1.A.docker_gnu.G.20230919_193255_z9hg2w", + "mct", + str(run_dir), + "ERIO", + "ERIO.ne30_g16_rx1.A.docker_gnu", + os.getcwd(), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + str(baseline_root), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + str(run_dir), + ) + + common = SystemTestsCommon(case) + + common._generate_baseline() + + baseline_dir = baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" + + assert (baseline_dir / "cpl.log.gz").exists() + assert (baseline_dir / "cpl-tput.log").exists() + assert (baseline_dir / "cpl-mem.log").exists() + assert (baseline_dir / "cpl.hi.2023-01-01.nc").exists() + + with open(baseline_dir / "cpl-tput.log") as fd: + lines = fd.readlines() + + assert len(lines) == 3 + assert lines[-1] == "719.635" + + with open(baseline_dir / "cpl-mem.log") as fd: + lines = fd.readlines() + + assert len(lines) == 3 + assert lines[-1] == "1673.89" + def test_kwargs(self): case = mock.MagicMock() From a367ab9b76bb8edb11bec505872b00941d929a1b Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Wed, 20 Sep 2023 14:45:04 -0700 Subject: [PATCH 05/57] Updates MEMCOMP and TPUTCOMP to DIFF if they're checked and fail --- CIME/test_status.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CIME/test_status.py b/CIME/test_status.py index 90714631eb8..5f306b7db0e 100644 --- a/CIME/test_status.py +++ b/CIME/test_status.py @@ -460,7 +460,7 @@ def _get_overall_status_based_on_phases( if rv == TEST_PASS_STATUS: rv = NAMELIST_FAIL_STATUS - elif phase == BASELINE_PHASE: + elif phase in [BASELINE_PHASE, THROUGHPUT_PHASE, MEMCOMP_PHASE]: if rv in [NAMELIST_FAIL_STATUS, TEST_PASS_STATUS]: phase_responsible_for_status = phase rv = TEST_DIFF_STATUS @@ -512,7 +512,9 @@ def get_overall_test_status( >>> _test_helper2('PASS ERS.foo.A RUN\nFAIL ERS.foo.A TPUTCOMP') ('PASS', 'RUN') >>> _test_helper2('PASS ERS.foo.A RUN\nFAIL ERS.foo.A TPUTCOMP', check_throughput=True) - ('FAIL', 'TPUTCOMP') + ('DIFF', 'TPUTCOMP') + >>> _test_helper2('PASS ERS.foo.A RUN\nFAIL ERS.foo.A MEMCOMP', check_memory=True) + ('DIFF', 'MEMCOMP') >>> _test_helper2('PASS ERS.foo.A MODEL_BUILD\nPASS ERS.foo.A RUN\nFAIL ERS.foo.A NLCOMP') ('NLFAIL', 'RUN') >>> _test_helper2('PASS ERS.foo.A MODEL_BUILD\nPEND ERS.foo.A RUN\nFAIL ERS.foo.A NLCOMP') From 2cc83524770a3bb7b7b66c77f9f9a913933953fd Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 21 Sep 2023 12:42:05 -0700 Subject: [PATCH 06/57] Refactors _compare_throughput and _compare_memory methods --- CIME/SystemTests/system_tests_common.py | 379 +++----------------- CIME/baselines.py | 320 +++++++++++++++++ CIME/tests/test_unit_baselines.py | 437 ++++++++++++++++++++++++ CIME/tests/test_unit_system_tests.py | 209 ++++++------ 4 files changed, 913 insertions(+), 432 deletions(-) create mode 100644 CIME/baselines.py create mode 100644 CIME/tests/test_unit_baselines.py diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index eb367e33cee..37dd63ad49c 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -26,6 +26,14 @@ from CIME.config import Config from CIME.provenance import save_test_time, get_test_success from CIME.locked_files import LOCKED_DIR, lock_file, is_locked +from CIME.baselines import ( + get_latest_cpl_logs, + get_mem_usage, + compare_memory, + compare_throughput, + write_baseline_mem, + write_baseline_tput, +) import CIME.build as build import glob, gzip, time, traceback, os @@ -493,7 +501,7 @@ def run_indv( self._case.case_st_archive(resubmit=True) def _coupler_log_indicates_run_complete(self): - newestcpllogfiles = self._get_latest_cpl_logs() + newestcpllogfiles = get_latest_cpl_logs(self._case) logger.debug("Latest Coupler log file(s) {}".format(newestcpllogfiles)) # Exception is raised if the file is not compressed allgood = len(newestcpllogfiles) @@ -598,43 +606,6 @@ def _st_archive_case_test(self): else: self._test_status.set_status(STARCHIVE_PHASE, TEST_FAIL_STATUS) - def _get_mem_usage(self, cpllog): - """ - Examine memory usage as recorded in the cpl log file and look for unexpected - increases. - """ - memlist = [] - meminfo = re.compile( - r".*model date =\s+(\w+).*memory =\s+(\d+\.?\d+).*highwater" - ) - if cpllog is not None and os.path.isfile(cpllog): - if ".gz" == cpllog[-3:]: - fopen = gzip.open - else: - fopen = open - with fopen(cpllog, "rb") as f: - for line in f: - m = meminfo.match(line.decode("utf-8")) - if m: - memlist.append((float(m.group(1)), float(m.group(2)))) - # Remove the last mem record, it's sometimes artificially high - if len(memlist) > 0: - memlist.pop() - return memlist - - def _get_throughput(self, cpllog): - """ - Examine memory usage as recorded in the cpl log file and look for unexpected - increases. - """ - if cpllog is not None and os.path.isfile(cpllog): - with gzip.open(cpllog, "rb") as f: - cpltext = f.read().decode("utf-8") - m = re.search(r"# simulated years / cmp-day =\s+(\d+\.\d+)\s", cpltext) - if m: - return float(m.group(1)) - return None - def _phase_modifying_call(self, phase, function): """ Ensures that unexpected exceptions from phases will result in a FAIL result @@ -662,9 +633,9 @@ def _check_for_memleak(self): increases. """ with self._test_status: - latestcpllogs = self._get_latest_cpl_logs() + latestcpllogs = get_latest_cpl_logs(self._case) for cpllog in latestcpllogs: - memlist = self._get_mem_usage(cpllog) + memlist = get_mem_usage(cpllog) if len(memlist) < 3: self._test_status.set_status( @@ -728,170 +699,61 @@ def compare_env_run(self, expected=None): return False return True - def _get_latest_cpl_logs(self): - """ - find and return the latest cpl log file in the run directory - """ - coupler_log_path = self._case.get_value("RUNDIR") - cpllogs = glob.glob( - os.path.join(coupler_log_path, "{}*.log.*".format(self._cpllog)) - ) - lastcpllogs = [] - if cpllogs: - lastcpllogs.append(max(cpllogs, key=os.path.getctime)) - basename = os.path.basename(lastcpllogs[0]) - suffix = basename.split(".", 1)[1] - for log in cpllogs: - if log in lastcpllogs: - continue - - if log.endswith(suffix): - lastcpllogs.append(log) - - return lastcpllogs - def _compare_memory(self): """ Compares current test memory usage to baseline. """ with self._test_status: - # compare memory usage to baseline - baseline_name = self._case.get_value("BASECMP_CASE") - basecmp_dir = os.path.join( - self._case.get_value("BASELINE_ROOT"), baseline_name + below_tolerance, diff, tolerance, baseline, current = compare_memory( + self._case ) - newestcpllogfiles = self._get_latest_cpl_logs() - - # do we need this loop? when is there more than one coupler file? - for cpllog in newestcpllogfiles: - try: - baseline_mem = self._read_baseline_mem(basecmp_dir) - except FileNotFoundError as e: - logger.debug("Could not read baseline memory usage: %s", e) - - continue - else: - self._compare_current_memory(baseline_mem, cpllog) - - def _compare_current_memory(self, baseline, cpllog): - """ - Compares current test memory usage against ``baseline``. - If the difference cannot be calculated then it's defaulted to "0.0". - - Parameters - ---------- - baseline : float - Baseline throughput value. - cpllog : str - Path to the current coupler log. - """ - memlist = self._get_mem_usage(cpllog) - - if len(memlist) > 3: - try: - current = memlist[-1][1] - except IndexError: - current = 0 - - try: - diff = (current - baseline) / baseline - except ZeroDivisionError: - diff = 0.0 - - tolerance = self._case.get_value("TEST_MEMLEAK_TOLERANCE") - if tolerance is None: - tolerance = 0.1 - comment = "MEMCOMP: Memory usage highwater has changed by {:.2f}% relative to baseline".format( - diff * 100 - ) - append_testlog(comment, self._orig_caseroot) - if diff < tolerance and self._test_status.get_status(MEMCOMP_PHASE) is None: - self._test_status.set_status(MEMCOMP_PHASE, TEST_PASS_STATUS) - elif self._test_status.get_status(MEMCOMP_PHASE) != TEST_FAIL_STATUS: - comment = "Error: Memory usage increase >{:d}% from baseline's {:f} to {:f}".format( - int(tolerance * 100), baseline, current - ) - self._test_status.set_status( - MEMCOMP_PHASE, TEST_FAIL_STATUS, comments=comment + if below_tolerance is not None: + comment = "MEMCOMP: Memory usage highwater has changed by {:.2f}% relative to baseline".format( + diff * 100 ) + append_testlog(comment, self._orig_caseroot) - else: - logger.debug("Found less than 4 memory usage data points") + + if ( + below_tolerance + and self._test_status.get_status(MEMCOMP_PHASE) is None + ): + self._test_status.set_status(MEMCOMP_PHASE, TEST_PASS_STATUS) + elif self._test_status.get_status(MEMCOMP_PHASE) != TEST_FAIL_STATUS: + comment = "Error: Memory usage increase >{:d}% from baseline's {:f} to {:f}".format( + int(tolerance * 100), baseline, current + ) + self._test_status.set_status( + MEMCOMP_PHASE, TEST_FAIL_STATUS, comments=comment + ) + append_testlog(comment, self._orig_caseroot) def _compare_throughput(self): """ Compares current test throughput to baseline. """ with self._test_status: - # compare memory usage to baseline - baseline_name = self._case.get_value("BASECMP_CASE") - basecmp_dir = os.path.join( - self._case.get_value("BASELINE_ROOT"), baseline_name - ) - newestcpllogfiles = self._get_latest_cpl_logs() - # do we need this loop? when is there more than one coupler file? - for cpllog in newestcpllogfiles: - try: - baseline_tput = self._read_baseline_tput(basecmp_dir) - except (FileNotFoundError, IndexError, ValueError) as e: - logger.debug("Could not read baseline throughput: %s", e) - - # Default behavior was to skip comparison if baseline file did not exist or no values found - continue - else: - self._compare_current_throughput(baseline_tput, cpllog) - - def _compare_current_throughput(self, baseline, cpllog): - """ - Compares current test throughput against ``baseline``. - - If the difference cannot be calculated then the phase is skipped. - - Parameters - ---------- - baseline : float - Baseline throughput value. - cpllog : str - Path to the current coupler log. - """ - current = self._get_throughput(cpllog) + below_tolerance, diff, tolerance, _, _ = compare_throughput(self._case) - try: - # comparing ypd so bigger is better - diff = (baseline - current) / baseline - except ValueError: - # Should we default the diff to 0.0 as with _compare_current_memory? - logger.debug( - "Could not determine change in throughput between baseline %s and current %s", - baseline, - current, - ) - else: - tolerance = self._case.get_value("TEST_TPUT_TOLERANCE") - if tolerance is None: - tolerance = 0.1 - expect( - tolerance > 0.0, - "Bad value for throughput tolerance in test", - ) - comment = "TPUTCOMP: Computation time changed by {:.2f}% relative to baseline".format( - diff * 100 - ) - append_testlog(comment, self._orig_caseroot) - if ( - diff < tolerance - and self._test_status.get_status(THROUGHPUT_PHASE) is None - ): - self._test_status.set_status(THROUGHPUT_PHASE, TEST_PASS_STATUS) - elif self._test_status.get_status(THROUGHPUT_PHASE) != TEST_FAIL_STATUS: - comment = "Error: TPUTCOMP: Computation time increase > {:d}% from baseline".format( - int(tolerance * 100) - ) - self._test_status.set_status( - THROUGHPUT_PHASE, TEST_FAIL_STATUS, comments=comment + if below_tolerance is not None: + comment = "TPUTCOMP: Computation time changed by {:.2f}% relative to baseline".format( + diff * 100 ) append_testlog(comment, self._orig_caseroot) + if ( + below_tolerance + and self._test_status.get_status(THROUGHPUT_PHASE) is None + ): + self._test_status.set_status(THROUGHPUT_PHASE, TEST_PASS_STATUS) + elif self._test_status.get_status(THROUGHPUT_PHASE) != TEST_FAIL_STATUS: + comment = "Error: TPUTCOMP: Computation time increase > {:d}% from baseline".format( + int(tolerance * 100) + ) + self._test_status.set_status( + THROUGHPUT_PHASE, TEST_FAIL_STATUS, comments=comment + ) + append_testlog(comment, self._orig_caseroot) def _compare_baseline(self): """ @@ -933,7 +795,7 @@ def _generate_baseline(self): ) # copy latest cpl log to baseline # drop the date so that the name is generic - newestcpllogfiles = self._get_latest_cpl_logs() + newestcpllogfiles = get_latest_cpl_logs(self._case) with SharedArea(): for cpllog in newestcpllogfiles: m = re.search(r"/({}.*.log).*.gz".format(self._cpllog), cpllog) @@ -945,148 +807,9 @@ def _generate_baseline(self): preserve_meta=False, ) - self._write_baseline_tput(basegen_dir, cpllog) - - self._write_baseline_mem(basegen_dir, cpllog) - - def _write_baseline_tput(self, baseline_dir, cpllog): - """ - Writes baseline throughput to file. - - A "-1" indicates that no throughput data was available from the coupler log. - - Parameters - ---------- - baseline_dir : str - Path to the baseline directory. - cpllog : str - Path to the current coupler log. - """ - tput_file = os.path.join(baseline_dir, "cpl-tput.log") - - tput = self._get_throughput(cpllog) - - with open(tput_file, "w") as fd: - fd.write("# Throughput in simulated years per compute day\n") - fd.write( - "# A -1 indicates no throughput data was available from the test\n" - ) - - if tput is None: - fd.write("-1") - else: - fd.write(str(tput)) - - def _write_baseline_mem(self, baseline_dir, cpllog): - """ - Writes baseline memory usage highwater to file. - - A "-1" indicates that no memory usage data was available from the coupler log. - - Parameters - ---------- - baseline_dir : str - Path to the baseline directory. - cpllog : str - Path to the current coupler log. - """ - mem_file = os.path.join(baseline_dir, "cpl-mem.log") - - mem = self._get_mem_usage(cpllog) - - with open(mem_file, "w") as fd: - fd.write("# Memory usage highwater\n") - fd.write( - "# A -1 indicates no memory usage data was available from the test\n" - ) - - try: - fd.write(str(mem[-1][1])) - except IndexError: - fd.write("-1") - - def _read_baseline_tput(self, baseline_dir): - """ - Reads throughput baseline. - - Parameters - ---------- - baseline_dir : str - Path to the baseline directory. - - Returns - ------- - float - Value of the throughput. - - Raises - ------ - FileNotFoundError - If baseline file does not exist. - IndexError - If no throughput value is found. - ValueError - If throughput value is not a float. - """ - return self._read_baseline_value(os.path.join(baseline_dir, "cpl-tput.log")) - - def _read_baseline_mem(self, baseline_dir): - """ - Reads memory usage highwater baseline. - - The default behvaior was to return 0 if no usage data was found. - - Parameters - ---------- - baseline_dir : str - Path to the baseline directory. - - Returns - ------- - float - Value of the highwater memory usage. - - Raises - ------ - FileNotFoundError - If baseline file does not exist. - """ - try: - memory = self._read_baseline_value( - os.path.join(baseline_dir, "cpl-mem.log") - ) - except (IndexError, ValueError): - memory = 0 - - return memory - - def _read_baseline_value(self, baseline_file): - """ - Read baseline value from file. - - Parameters - ---------- - baseline_file : str - Path to baseline file. - - Returns - ------- - float - Baseline value. - - Raises - ------ - FileNotFoundError - If ``baseline_file`` is not found. - IndexError - If not values are present in ``baseline_file``. - ValueError - If value in ``baseline_file`` is not a float. - """ - with open(baseline_file) as fd: - lines = [x for x in fd.readlines() if not x.startswith("#")] + write_baseline_tput(basegen_dir, cpllog) - return float(lines[0]) + write_baseline_mem(basegen_dir, cpllog) class FakeTest(SystemTestsCommon): diff --git a/CIME/baselines.py b/CIME/baselines.py new file mode 100644 index 00000000000..e371488f294 --- /dev/null +++ b/CIME/baselines.py @@ -0,0 +1,320 @@ +import os +import glob +import re +import gzip +import logging +from CIME.utils import expect + +logger = logging.getLogger(__name__) + +def get_latest_cpl_logs(case): + """ + find and return the latest cpl log file in the run directory + """ + coupler_log_path = case.get_value("RUNDIR") + cpllog_name = "drv" if case.get_value("COMP_INTERFACE") == "nuopc" else "cpl" + cpllogs = glob.glob( + os.path.join(coupler_log_path, "{}*.log.*".format(cpllog_name)) + ) + lastcpllogs = [] + if cpllogs: + lastcpllogs.append(max(cpllogs, key=os.path.getctime)) + basename = os.path.basename(lastcpllogs[0]) + suffix = basename.split(".", 1)[1] + for log in cpllogs: + if log in lastcpllogs: + continue + + if log.endswith(suffix): + lastcpllogs.append(log) + + return lastcpllogs + +def compare_memory(case): + baseline_root = case.get_value("BASELINE_ROOT") + + baseline_name = case.get_value("BASECMP_CASE") + + baseline_dir = os.path.join(baseline_root, baseline_name) + + latest_cpl_logs = get_latest_cpl_logs(case) + + diff, baseline, current = [None]*3 + + for cpllog in latest_cpl_logs: + try: + baseline = read_baseline_mem(baseline_dir) + except FileNotFoundError as e: + logger.debug("Could not read baseline memory usage: %s", e) + + continue + + if baseline is None: + baseline = 0.0 + + memlist = get_mem_usage(cpllog) + + if len(memlist) <= 3: + logger.debug(f"Found {len(memlist)} memory usage samples, need atleast 4") + + continue + + current = memlist[-1][1] + + try: + diff = (current - baseline) / baseline + except ZeroDivisionError: + diff = 0.0 + + tolerance = case.get_value("TEST_MEMLEAK_TOLERANCE") + + if tolerance is None: + tolerance = 0.1 + + # Should we check if tolerance is above 0 + below_tolerance = None + + if diff is not None: + below_tolerance = diff < tolerance + + return below_tolerance, diff, tolerance, baseline, current + +def compare_throughput(case): + baseline_root = case.get_value("BASELINE_ROOT") + + baseline_name = case.get_value("BASECMP_CASE") + + baseline_dir = os.path.join(baseline_root, baseline_name) + + latest_cpl_logs = get_latest_cpl_logs(case) + + diff, baseline, current = [None]*3 + + # Do we need this loop, when are there multiple cpl logs? + for cpllog in latest_cpl_logs: + try: + baseline = read_baseline_tput(baseline_dir) + except (FileNotFoundError, IndexError, ValueError) as e: + logger.debug("Could not read baseline throughput: %s", e) + + continue + + current = get_throughput(cpllog) + + try: + # comparing ypd so bigger is better + diff = (baseline - current) / baseline + except (ValueError, TypeError): + # Should we default the diff to 0.0 as with _compare_current_memory? + logger.debug( + "Could not determine change in throughput between baseline %s and current %s", + baseline, + current, + ) + + continue + + tolerance = case.get_value("TEST_TPUT_TOLERANCE") + + if tolerance is None: + tolerance = 0.1 + + expect( + tolerance > 0.0, + "Bad value for throughput tolerance in test", + ) + + below_tolerance = None + + if diff is not None: + below_tolerance = diff < tolerance + + return below_tolerance, diff, tolerance, baseline, current + +def write_baseline_tput(baseline_dir, cpllog): + """ + Writes baseline throughput to file. + + A "-1" indicates that no throughput data was available from the coupler log. + + Parameters + ---------- + baseline_dir : str + Path to the baseline directory. + cpllog : str + Path to the current coupler log. + """ + tput_file = os.path.join(baseline_dir, "cpl-tput.log") + + tput = get_throughput(cpllog) + + with open(tput_file, "w") as fd: + fd.write("# Throughput in simulated years per compute day\n") + fd.write( + "# A -1 indicates no throughput data was available from the test\n" + ) + + if tput is None: + fd.write("-1") + else: + fd.write(str(tput)) + +def write_baseline_mem(baseline_dir, cpllog): + """ + Writes baseline memory usage highwater to file. + + A "-1" indicates that no memory usage data was available from the coupler log. + + Parameters + ---------- + baseline_dir : str + Path to the baseline directory. + cpllog : str + Path to the current coupler log. + """ + mem_file = os.path.join(baseline_dir, "cpl-mem.log") + + mem = get_mem_usage(cpllog) + + with open(mem_file, "w") as fd: + fd.write("# Memory usage highwater\n") + fd.write( + "# A -1 indicates no memory usage data was available from the test\n" + ) + + try: + fd.write(str(mem[-1][1])) + except IndexError: + fd.write("-1") + +def read_baseline_tput(baseline_dir): + """ + Reads throughput baseline. + + Parameters + ---------- + baseline_dir : str + Path to the baseline directory. + + Returns + ------- + float + Value of the throughput. + + Raises + ------ + FileNotFoundError + If baseline file does not exist. + IndexError + If no throughput value is found. + ValueError + If throughput value is not a float. + """ + try: + tput = read_baseline_value(os.path.join(baseline_dir, "cpl-tput.log")) + except (IndexError, ValueError): + tput = None + + if tput == -1: + tput = None + + return tput + +def read_baseline_mem(baseline_dir): + """ + Reads memory usage highwater baseline. + + The default behvaior was to return 0 if no usage data was found. + + Parameters + ---------- + baseline_dir : str + Path to the baseline directory. + + Returns + ------- + float + Value of the highwater memory usage. + + Raises + ------ + FileNotFoundError + If baseline file does not exist. + """ + try: + memory = read_baseline_value( + os.path.join(baseline_dir, "cpl-mem.log") + ) + except (IndexError, ValueError): + memory = 0 + + if memory == -1: + memory = 0 + + return memory + +def read_baseline_value(baseline_file): + """ + Read baseline value from file. + + Parameters + ---------- + baseline_file : str + Path to baseline file. + + Returns + ------- + float + Baseline value. + + Raises + ------ + FileNotFoundError + If ``baseline_file`` is not found. + IndexError + If not values are present in ``baseline_file``. + ValueError + If value in ``baseline_file`` is not a float. + """ + with open(baseline_file) as fd: + lines = [x for x in fd.readlines() if not x.startswith("#")] + + return float(lines[0]) + +def get_mem_usage(cpllog): + """ + Examine memory usage as recorded in the cpl log file and look for unexpected + increases. + """ + memlist = [] + meminfo = re.compile( + r".*model date =\s+(\w+).*memory =\s+(\d+\.?\d+).*highwater" + ) + if cpllog is not None and os.path.isfile(cpllog): + if ".gz" == cpllog[-3:]: + fopen = gzip.open + else: + fopen = open + with fopen(cpllog, "rb") as f: + for line in f: + m = meminfo.match(line.decode("utf-8")) + if m: + memlist.append((float(m.group(1)), float(m.group(2)))) + # Remove the last mem record, it's sometimes artificially high + if len(memlist) > 0: + memlist.pop() + return memlist + +def get_throughput(cpllog): + """ + Examine memory usage as recorded in the cpl log file and look for unexpected + increases. + """ + if cpllog is not None and os.path.isfile(cpllog): + with gzip.open(cpllog, "rb") as f: + cpltext = f.read().decode("utf-8") + m = re.search(r"# simulated years / cmp-day =\s+(\d+\.\d+)\s", cpltext) + if m: + return float(m.group(1)) + return None + diff --git a/CIME/tests/test_unit_baselines.py b/CIME/tests/test_unit_baselines.py new file mode 100644 index 00000000000..77ce61ac20f --- /dev/null +++ b/CIME/tests/test_unit_baselines.py @@ -0,0 +1,437 @@ +#!/usr/bin/env python3 + +import gzip +import tempfile +import unittest +from unittest import mock +from pathlib import Path + +from CIME import baselines +from CIME.tests.test_unit_system_tests import CPLLOG + +def create_mock_case(tempdir, get_latest_cpl_logs): + caseroot = Path(tempdir, "0", "caseroot") + rundir = caseroot / "run" + + get_latest_cpl_logs.return_value = ( + str(rundir / "cpl.log.gz"), + ) + + baseline_root = Path(tempdir, "baselines") + + case = mock.MagicMock() + + return case, caseroot, rundir, baseline_root + +class TestUnitBaseline(unittest.TestCase): + def test_get_throughput_no_file(self): + throughput = baselines.get_throughput("/tmp/cpl.log") + + assert throughput is None + + def test_get_throughput(self): + with tempfile.TemporaryDirectory() as tempdir: + cpl_log_path = Path(tempdir, "cpl.log.gz") + + with gzip.open(cpl_log_path, "w") as fd: + fd.write(CPLLOG.encode("utf-8")) + + throughput = baselines.get_throughput(str(cpl_log_path)) + + assert throughput == 719.635 + + def test_get_mem_usage_gz(self): + with tempfile.TemporaryDirectory() as tempdir: + cpl_log_path = Path(tempdir, "cpl.log.gz") + + with gzip.open(cpl_log_path, "w") as fd: + fd.write(CPLLOG.encode("utf-8")) + + mem_usage = baselines.get_mem_usage(str(cpl_log_path)) + + assert mem_usage == [(10102.0, 1673.89), (10103.0, 1673.89), (10104.0, 1673.89), (10105.0, 1673.89)] + + @mock.patch("CIME.baselines.os.path.isfile") + def test_get_mem_usage(self, isfile): + isfile.return_value = True + + with mock.patch("builtins.open", mock.mock_open(read_data=CPLLOG.encode("utf-8"))) as mock_file: + mem_usage = baselines.get_mem_usage("/tmp/cpl.log") + + assert mem_usage == [(10102.0, 1673.89), (10103.0, 1673.89), (10104.0, 1673.89), (10105.0, 1673.89)] + + def test_read_baseline_mem_empty(self): + with mock.patch("builtins.open", mock.mock_open(read_data="")) as mock_file: + baseline = baselines.read_baseline_mem("/tmp/cpl-mem.log") + + assert baseline is 0 + + def test_read_baseline_mem_none(self): + with mock.patch("builtins.open", mock.mock_open(read_data="-1")) as mock_file: + baseline = baselines.read_baseline_mem("/tmp/cpl-mem.log") + + assert baseline is 0 + + def test_read_baseline_mem(self): + with mock.patch("builtins.open", mock.mock_open(read_data="200")) as mock_file: + baseline = baselines.read_baseline_mem("/tmp/cpl-mem.log") + + assert baseline == 200 + + def test_read_baseline_tput_empty(self): + with mock.patch("builtins.open", mock.mock_open(read_data="")) as mock_file: + baseline = baselines.read_baseline_tput("/tmp/cpl-tput.log") + + assert baseline is None + + def test_read_baseline_tput_none(self): + with mock.patch("builtins.open", mock.mock_open(read_data="-1")) as mock_file: + baseline = baselines.read_baseline_tput("/tmp/cpl-tput.log") + + assert baseline is None + + def test_read_baseline_tput(self): + with mock.patch("builtins.open", mock.mock_open(read_data="200")) as mock_file: + baseline = baselines.read_baseline_tput("/tmp/cpl-tput.log") + + assert baseline == 200 + + @mock.patch("CIME.baselines.get_mem_usage") + def test_write_baseline_mem_no_value(self, get_mem_usage): + get_mem_usage.return_value = [] + + with mock.patch("builtins.open", mock.mock_open()) as mock_file: + baselines.write_baseline_mem("/tmp", "/tmp/cpl.log") + + mock_file.assert_called_with("/tmp/cpl-mem.log", "w") + mock_file.return_value.write.assert_called_with("-1") + + @mock.patch("CIME.baselines.get_mem_usage") + def test_write_baseline_mem(self, get_mem_usage): + get_mem_usage.return_value = [(1, 200)] + + with mock.patch("builtins.open", mock.mock_open()) as mock_file: + baselines.write_baseline_mem("/tmp", "/tmp/cpl.log") + + mock_file.assert_called_with("/tmp/cpl-mem.log", "w") + mock_file.return_value.write.assert_called_with("200") + + @mock.patch("CIME.baselines.get_throughput") + def test_write_baseline_tput_no_value(self, get_throughput): + get_throughput.return_value = None + + with mock.patch("builtins.open", mock.mock_open()) as mock_file: + baselines.write_baseline_tput("/tmp", "/tmp/cpl.log") + + mock_file.assert_called_with("/tmp/cpl-tput.log", "w") + mock_file.return_value.write.assert_called_with("-1") + + @mock.patch("CIME.baselines.get_throughput") + def test_write_baseline_tput(self, get_throughput): + get_throughput.return_value = 200 + + with mock.patch("builtins.open", mock.mock_open()) as mock_file: + baselines.write_baseline_tput("/tmp", "/tmp/cpl.log") + + mock_file.assert_called_with("/tmp/cpl-tput.log", "w") + mock_file.return_value.write.assert_called_with("200") + + @mock.patch("CIME.baselines.get_throughput") + @mock.patch("CIME.baselines.read_baseline_tput") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_compare_throughput_no_baseline_file(self, get_latest_cpl_logs, read_baseline_tput, get_throughput): + read_baseline_tput.side_effect = FileNotFoundError + + get_throughput.return_value = 504 + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + case.get_value.side_effect = ( + str(baseline_root), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + 0.05, + ) + + below_tolerance, diff, tolerance, baseline, current = baselines.compare_throughput(case) + + assert below_tolerance is None + assert diff == None + assert tolerance == 0.05 + assert baseline == None + assert current == None + + @mock.patch("CIME.baselines.get_throughput") + @mock.patch("CIME.baselines.read_baseline_tput") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_compare_throughput_no_baseline(self, get_latest_cpl_logs, read_baseline_tput, get_throughput): + read_baseline_tput.return_value = None + + get_throughput.return_value = 504 + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + case.get_value.side_effect = ( + str(baseline_root), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + 0.05, + ) + + below_tolerance, diff, tolerance, baseline, current = baselines.compare_throughput(case) + + assert below_tolerance is None + assert diff == None + assert tolerance == 0.05 + assert baseline == None + assert current == 504 + + @mock.patch("CIME.baselines.get_throughput") + @mock.patch("CIME.baselines.read_baseline_tput") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_compare_throughput_no_tolerance(self, get_latest_cpl_logs, read_baseline_tput, get_throughput): + read_baseline_tput.return_value = 500 + + get_throughput.return_value = 504 + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + case.get_value.side_effect = ( + str(baseline_root), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + None + ) + + below_tolerance, diff, tolerance, baseline, current = baselines.compare_throughput(case) + + assert below_tolerance + assert diff == -0.008 + assert tolerance == 0.1 + assert baseline == 500 + assert current == 504 + + @mock.patch("CIME.baselines.get_throughput") + @mock.patch("CIME.baselines.read_baseline_tput") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_compare_throughput(self, get_latest_cpl_logs, read_baseline_tput, get_throughput): + read_baseline_tput.return_value = 500 + + get_throughput.return_value = 504 + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + case.get_value.side_effect = ( + str(baseline_root), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + 0.05, + ) + + below_tolerance, diff, tolerance, baseline, current = baselines.compare_throughput(case) + + assert below_tolerance + assert diff == -0.008 + assert tolerance == 0.05 + assert baseline == 500 + assert current == 504 + + @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.read_baseline_mem") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_compare_memory_no_baseline(self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage): + read_baseline_mem.return_value = None + + get_mem_usage.return_value = [ + (1, 1000.0), + (2, 1001.0), + (3, 1002.0), + (4, 1003.0), + ] + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + case.get_value.side_effect = ( + str(baseline_root), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + 0.05, + ) + + below_tolerance, diff, tolerance, baseline, current = baselines.compare_memory(case) + + assert below_tolerance + assert diff == 0.0 + assert tolerance == 0.05 + assert baseline == 0.0 + assert current == 1003.0 + + @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.read_baseline_mem") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_compare_memory_not_enough_samples(self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage): + read_baseline_mem.return_value = 1000.0 + + get_mem_usage.return_value = [ + (1, 1000.0), + (2, 1001.0), + ] + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + case.get_value.side_effect = ( + str(baseline_root), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + 0.05, + ) + + below_tolerance, diff, tolerance, baseline, current = baselines.compare_memory(case) + + assert below_tolerance is None + assert diff == None + assert tolerance == 0.05 + assert baseline == 1000.0 + assert current == None + + @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.read_baseline_mem") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_compare_memory_no_baseline_file(self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage): + read_baseline_mem.side_effect = FileNotFoundError + + get_mem_usage.return_value = [ + (1, 1000.0), + (2, 1001.0), + (3, 1002.0), + (4, 1003.0), + ] + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + case.get_value.side_effect = ( + str(baseline_root), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + 0.05, + ) + + below_tolerance, diff, tolerance, baseline, current = baselines.compare_memory(case) + + assert below_tolerance is None + assert diff == None + assert tolerance == 0.05 + assert baseline == None + assert current == None + + @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.read_baseline_mem") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_compare_memory_no_tolerance(self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage): + read_baseline_mem.return_value = 1000.0 + + get_mem_usage.return_value = [ + (1, 1000.0), + (2, 1001.0), + (3, 1002.0), + (4, 1003.0), + ] + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + case.get_value.side_effect = ( + str(baseline_root), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + None, + ) + + below_tolerance, diff, tolerance, baseline, current = baselines.compare_memory(case) + + assert below_tolerance + assert diff == 0.003 + assert tolerance == 0.1 + assert baseline == 1000.0 + assert current == 1003.0 + + @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.read_baseline_mem") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_compare_memory(self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage): + read_baseline_mem.return_value = 1000.0 + + get_mem_usage.return_value = [ + (1, 1000.0), + (2, 1001.0), + (3, 1002.0), + (4, 1003.0), + ] + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + case.get_value.side_effect = ( + str(baseline_root), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + 0.05, + ) + + below_tolerance, diff, tolerance, baseline, current = baselines.compare_memory(case) + + assert below_tolerance + assert diff == 0.003 + assert tolerance == 0.05 + assert baseline == 1000.0 + assert current == 1003.0 + + def test_get_latest_cpl_logs_found_multiple(self): + with tempfile.TemporaryDirectory() as tempdir: + run_dir = Path(tempdir) / "run" + run_dir.mkdir(parents=True, exist_ok=False) + + cpl_log_path = run_dir / "cpl.log.gz" + cpl_log_path.touch() + + cpl_log_2_path = run_dir / "cpl-2023-01-01.log.gz" + cpl_log_2_path.touch() + + case = mock.MagicMock() + case.get_value.side_effect = ( + str(run_dir), + "mct", + ) + + latest_cpl_logs = baselines.get_latest_cpl_logs(case) + + assert len(latest_cpl_logs) == 2 + assert sorted(latest_cpl_logs) == sorted([str(cpl_log_path), str(cpl_log_2_path)]) + + def test_get_latest_cpl_logs_found_single(self): + with tempfile.TemporaryDirectory() as tempdir: + run_dir = Path(tempdir) / "run" + run_dir.mkdir(parents=True, exist_ok=False) + + cpl_log_path = run_dir / "cpl.log.gz" + cpl_log_path.touch() + + case = mock.MagicMock() + case.get_value.side_effect = ( + str(run_dir), + "mct", + ) + + latest_cpl_logs = baselines.get_latest_cpl_logs(case) + + assert len(latest_cpl_logs) == 1 + assert latest_cpl_logs[0] == str(cpl_log_path) + + def test_get_latest_cpl_logs(self): + case = mock.MagicMock() + case.get_value.side_effect = ( + f"/tmp/run", + "mct", + ) + + latest_cpl_logs = baselines.get_latest_cpl_logs(case) + + assert len(latest_cpl_logs) == 0 diff --git a/CIME/tests/test_unit_system_tests.py b/CIME/tests/test_unit_system_tests.py index e456b675047..e81bd6bca10 100644 --- a/CIME/tests/test_unit_system_tests.py +++ b/CIME/tests/test_unit_system_tests.py @@ -66,169 +66,169 @@ def create_mock_case(tempdir, idx=None, cpllog_data=None): class TestCaseSubmit(unittest.TestCase): + @mock.patch("CIME.SystemTests.system_tests_common.compare_throughput") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - def test_compare_throughput(self, append_testlog): - with tempfile.TemporaryDirectory() as tempdir: - case, caseroot, baseline_root, run_dir = create_mock_case( - tempdir, cpllog_data=CPLLOG - ) + def test_compare_throughput(self, append_testlog, compare_throughput): + compare_throughput.return_value = (True, 0.02, 0.05, 200, 201) - base_case, base_caseroot, _, _ = create_mock_case( - tempdir, idx=1, cpllog_data=CPLLOG - ) + with tempfile.TemporaryDirectory() as tempdir: + caseroot = Path(tempdir) / "caseroot" + caseroot.mkdir(parents=True, exist_ok=False) + case = mock.MagicMock() case.get_value.side_effect = ( - str(caseroot), + str(Path(tempdir) / "caseroot"), "ERIO.ne30_g16_rx1.A.docker_gnu", "mct", - "master/ERIO.ne30_g16_rx1.A.docker_gnu", - str(baseline_root), - str(run_dir), - 0.05, ) - baseline_dir = baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" - baseline_dir.mkdir(parents=True, exist_ok=False) - baseline_mem = baseline_dir / "cpl-mem.log" + common = SystemTestsCommon(case) - with open(baseline_mem, "w") as fd: - fd.write(str("1673.89")) + common._compare_throughput() - common = SystemTestsCommon(case) + assert common._test_status.get_overall_test_status() == ("PASS", None) - common._compare_memory() + append_testlog.assert_any_call( + "TPUTCOMP: Computation time changed by 2.00% relative to baseline", + str(caseroot), + ) - assert common._test_status.get_overall_test_status() == ("PASS", None) + @mock.patch("CIME.SystemTests.system_tests_common.compare_throughput") + @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") + def test_compare_throughput_error_diff(self, append_testlog, compare_throughput): + compare_throughput.return_value = (None, 0.02, 0.05, 200, 201) - append_testlog.assert_any_call( - "MEMCOMP: Memory usage highwater has changed by 0.00% relative to baseline", - str(caseroot), + with tempfile.TemporaryDirectory() as tempdir: + caseroot = Path(tempdir) / "caseroot" + caseroot.mkdir(parents=True, exist_ok=False) + + case = mock.MagicMock() + case.get_value.side_effect = ( + str(Path(tempdir) / "caseroot"), + "ERIO.ne30_g16_rx1.A.docker_gnu", + "mct", ) + common = SystemTestsCommon(case) + + common._compare_throughput() + + assert common._test_status.get_overall_test_status() == ("PASS", None) + + append_testlog.assert_not_called() + + @mock.patch("CIME.SystemTests.system_tests_common.compare_throughput") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - def test_compare_throughput_fail(self, append_testlog): + def test_compare_throughput_fail(self, append_testlog, compare_throughput): + compare_throughput.return_value = (False, 0.02, 0.05, 200, 201) + with tempfile.TemporaryDirectory() as tempdir: - case, caseroot, baseline_root, run_dir = create_mock_case( - tempdir, cpllog_data=CPLLOG - ) + caseroot = Path(tempdir) / "caseroot" + caseroot.mkdir(parents=True, exist_ok=False) - base_case, base_caseroot, _, _ = create_mock_case( - tempdir, idx=1, cpllog_data=CPLLOG + case = mock.MagicMock() + case.get_value.side_effect = ( + str(Path(tempdir) / "caseroot"), + "ERIO.ne30_g16_rx1.A.docker_gnu", + "mct", ) + common = SystemTestsCommon(case) + + common._compare_throughput() + + assert common._test_status.get_overall_test_status() == ("PASS", None) + + append_testlog.assert_any_call( + "TPUTCOMP: Computation time changed by 2.00% relative to baseline", + str(caseroot), + ) + append_testlog.assert_any_call( + "Error: TPUTCOMP: Computation time increase > 5% from baseline", + str(caseroot), + ) + + @mock.patch("CIME.SystemTests.system_tests_common.compare_memory") + @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") + def test_compare_memory(self, append_testlog, compare_memory): + compare_memory.return_value = (True, 0.02, 0.05, 1000, 1002) + + with tempfile.TemporaryDirectory() as tempdir: + caseroot = Path(tempdir) / "caseroot" + caseroot.mkdir(parents=True, exist_ok=False) + + case = mock.MagicMock() case.get_value.side_effect = ( str(caseroot), "ERIO.ne30_g16_rx1.A.docker_gnu", "mct", - "master/ERIO.ne30_g16_rx1.A.docker_gnu", - str(baseline_root), - str(run_dir), - 0.05, ) - baseline_dir = baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" - baseline_dir.mkdir(parents=True, exist_ok=False) - baseline_mem = baseline_dir / "cpl-mem.log" - - with open(baseline_mem, "w") as fd: - fd.write(str("1473.89")) - common = SystemTestsCommon(case) common._compare_memory() - assert common._test_status.get_overall_test_status() == ("PASS", None) + assert common._test_status.get_overall_test_status() == ("PASS", None) - append_testlog.assert_any_call( - "MEMCOMP: Memory usage highwater has changed by 13.57% relative to baseline", - str(caseroot), - ) - append_testlog.assert_any_call( - "Error: Memory usage increase >5% from baseline's 1473.890000 to 1673.890000", - str(caseroot), - ) + append_testlog.assert_any_call( + "MEMCOMP: Memory usage highwater has changed by 2.00% relative to baseline", + str(caseroot), + ) + @mock.patch("CIME.SystemTests.system_tests_common.compare_memory") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - def test_compare_memory(self, append_testlog): - with tempfile.TemporaryDirectory() as tempdir: - case, caseroot, baseline_root, run_dir = create_mock_case( - tempdir, cpllog_data=CPLLOG - ) + def test_compare_memory_erorr_diff(self, append_testlog, compare_memory): + compare_memory.return_value = (None, 0.02, 0.05, 1000, 1002) - base_case, base_caseroot, _, _ = create_mock_case( - tempdir, idx=1, cpllog_data=CPLLOG - ) + with tempfile.TemporaryDirectory() as tempdir: + caseroot = Path(tempdir) / "caseroot" + caseroot.mkdir(parents=True, exist_ok=False) + case = mock.MagicMock() case.get_value.side_effect = ( str(caseroot), "ERIO.ne30_g16_rx1.A.docker_gnu", "mct", - "master/ERIO.ne30_g16_rx1.A.docker_gnu", - str(baseline_root), - str(run_dir), - 0.05, ) - baseline_dir = baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" - baseline_dir.mkdir(parents=True, exist_ok=False) - baseline_tput = baseline_dir / "cpl-tput.log" - - with open(baseline_tput, "w") as fd: - fd.write(str("719.635")) - common = SystemTestsCommon(case) - common._compare_throughput() + common._compare_memory() - assert common._test_status.get_overall_test_status() == ("PASS", None) + assert common._test_status.get_overall_test_status() == ("PASS", None) - append_testlog.assert_any_call( - "TPUTCOMP: Computation time changed by 0.00% relative to baseline", - str(caseroot), - ) + append_testlog.assert_not_called() + @mock.patch("CIME.SystemTests.system_tests_common.compare_memory") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - def test_compare_memory_fail(self, append_testlog): - with tempfile.TemporaryDirectory() as tempdir: - case, caseroot, baseline_root, run_dir = create_mock_case( - tempdir, cpllog_data=CPLLOG - ) + def test_compare_memory_erorr_fail(self, append_testlog, compare_memory): + compare_memory.return_value = (False, 0.02, 0.05, 1000, 1002) - base_case, base_caseroot, _, _ = create_mock_case( - tempdir, idx=1, cpllog_data=CPLLOG - ) + with tempfile.TemporaryDirectory() as tempdir: + caseroot = Path(tempdir) / "caseroot" + caseroot.mkdir(parents=True, exist_ok=False) + case = mock.MagicMock() case.get_value.side_effect = ( str(caseroot), "ERIO.ne30_g16_rx1.A.docker_gnu", "mct", - "master/ERIO.ne30_g16_rx1.A.docker_gnu", - str(baseline_root), - str(run_dir), - 0.05, ) - baseline_dir = baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" - baseline_dir.mkdir(parents=True, exist_ok=False) - baseline_tput = baseline_dir / "cpl-tput.log" - - with open(baseline_tput, "w") as fd: - fd.write(str("900.635")) - common = SystemTestsCommon(case) - common._compare_throughput() + common._compare_memory() - assert common._test_status.get_overall_test_status() == ("PASS", None) + assert common._test_status.get_overall_test_status() == ("PASS", None) - append_testlog.assert_any_call( - "TPUTCOMP: Computation time changed by 20.10% relative to baseline", - str(caseroot), - ) - append_testlog.assert_any_call( - "Error: TPUTCOMP: Computation time increase > 5% from baseline", - str(caseroot), - ) + append_testlog.assert_any_call( + "MEMCOMP: Memory usage highwater has changed by 2.00% relative to baseline", + str(caseroot), + ) + append_testlog.assert_any_call( + "Error: Memory usage increase >5% from baseline's 1000.000000 to 1002.000000", + str(caseroot), + ) def test_generate_baseline(self): with tempfile.TemporaryDirectory() as tempdir: @@ -254,6 +254,7 @@ def test_generate_baseline(self): str(baseline_root), "master/ERIO.ne30_g16_rx1.A.docker_gnu", str(run_dir), + "mct", ) common = SystemTestsCommon(case) From 5ac9e1ea3748ae81729fea0116781344e0447abf Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 22 Sep 2023 15:34:40 -0700 Subject: [PATCH 07/57] Refactors compare_throughput and compare_memory functions --- CIME/SystemTests/system_tests_common.py | 31 ++-- CIME/baselines.py | 107 ++++++----- CIME/tests/test_unit_baselines.py | 230 +++++++++++++++--------- CIME/tests/test_unit_system_tests.py | 32 ++-- 4 files changed, 240 insertions(+), 160 deletions(-) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index 37dd63ad49c..e1c5d59388c 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -28,6 +28,7 @@ from CIME.locked_files import LOCKED_DIR, lock_file, is_locked from CIME.baselines import ( get_latest_cpl_logs, + get_throughput, get_mem_usage, compare_memory, compare_throughput, @@ -704,15 +705,9 @@ def _compare_memory(self): Compares current test memory usage to baseline. """ with self._test_status: - below_tolerance, diff, tolerance, baseline, current = compare_memory( - self._case - ) + below_tolerance, comment = compare_memory(self._case) if below_tolerance is not None: - comment = "MEMCOMP: Memory usage highwater has changed by {:.2f}% relative to baseline".format( - diff * 100 - ) - append_testlog(comment, self._orig_caseroot) if ( @@ -721,39 +716,29 @@ def _compare_memory(self): ): self._test_status.set_status(MEMCOMP_PHASE, TEST_PASS_STATUS) elif self._test_status.get_status(MEMCOMP_PHASE) != TEST_FAIL_STATUS: - comment = "Error: Memory usage increase >{:d}% from baseline's {:f} to {:f}".format( - int(tolerance * 100), baseline, current - ) self._test_status.set_status( MEMCOMP_PHASE, TEST_FAIL_STATUS, comments=comment ) - append_testlog(comment, self._orig_caseroot) def _compare_throughput(self): """ Compares current test throughput to baseline. """ with self._test_status: - below_tolerance, diff, tolerance, _, _ = compare_throughput(self._case) + below_tolerance, comment = compare_throughput(self._case) if below_tolerance is not None: - comment = "TPUTCOMP: Computation time changed by {:.2f}% relative to baseline".format( - diff * 100 - ) append_testlog(comment, self._orig_caseroot) + if ( below_tolerance and self._test_status.get_status(THROUGHPUT_PHASE) is None ): self._test_status.set_status(THROUGHPUT_PHASE, TEST_PASS_STATUS) elif self._test_status.get_status(THROUGHPUT_PHASE) != TEST_FAIL_STATUS: - comment = "Error: TPUTCOMP: Computation time increase > {:d}% from baseline".format( - int(tolerance * 100) - ) self._test_status.set_status( THROUGHPUT_PHASE, TEST_FAIL_STATUS, comments=comment ) - append_testlog(comment, self._orig_caseroot) def _compare_baseline(self): """ @@ -807,9 +792,13 @@ def _generate_baseline(self): preserve_meta=False, ) - write_baseline_tput(basegen_dir, cpllog) + tput = get_throughput(cpllog) + + write_baseline_tput(basegen_dir, tput) + + mem = get_mem_usage(cpllog) - write_baseline_mem(basegen_dir, cpllog) + write_baseline_mem(basegen_dir, mem) class FakeTest(SystemTestsCommon): diff --git a/CIME/baselines.py b/CIME/baselines.py index e371488f294..cbdb3a2cf61 100644 --- a/CIME/baselines.py +++ b/CIME/baselines.py @@ -7,15 +7,14 @@ logger = logging.getLogger(__name__) + def get_latest_cpl_logs(case): """ find and return the latest cpl log file in the run directory """ coupler_log_path = case.get_value("RUNDIR") cpllog_name = "drv" if case.get_value("COMP_INTERFACE") == "nuopc" else "cpl" - cpllogs = glob.glob( - os.path.join(coupler_log_path, "{}*.log.*".format(cpllog_name)) - ) + cpllogs = glob.glob(os.path.join(coupler_log_path, "{}*.log.*".format(cpllog_name))) lastcpllogs = [] if cpllogs: lastcpllogs.append(max(cpllogs, key=os.path.getctime)) @@ -30,22 +29,27 @@ def get_latest_cpl_logs(case): return lastcpllogs -def compare_memory(case): - baseline_root = case.get_value("BASELINE_ROOT") - baseline_name = case.get_value("BASECMP_CASE") +def compare_memory(case, baseline_dir=None): + if baseline_dir is None: + baseline_root = case.get_value("BASELINE_ROOT") - baseline_dir = os.path.join(baseline_root, baseline_name) + baseline_name = case.get_value("BASECMP_CASE") + + baseline_dir = os.path.join(baseline_root, baseline_name) latest_cpl_logs = get_latest_cpl_logs(case) - diff, baseline, current = [None]*3 + diff, baseline, current = [None] * 3 + comment = "" for cpllog in latest_cpl_logs: try: baseline = read_baseline_mem(baseline_dir) except FileNotFoundError as e: - logger.debug("Could not read baseline memory usage: %s", e) + comment = f"Could not read baseline memory usage: {e!s}" + + logger.debug(comment) continue @@ -55,7 +59,9 @@ def compare_memory(case): memlist = get_mem_usage(cpllog) if len(memlist) <= 3: - logger.debug(f"Found {len(memlist)} memory usage samples, need atleast 4") + comment = f"Found {len(memlist)} memory usage samples, need atleast 4" + + logger.debug(comment) continue @@ -77,25 +83,39 @@ def compare_memory(case): if diff is not None: below_tolerance = diff < tolerance - return below_tolerance, diff, tolerance, baseline, current + if below_tolerance: + comment = "MEMCOMP: Memory usage highwater has changed by {:.2f}% relative to baseline".format( + diff * 100 + ) + else: + comment = "Error: Memory usage increase >{:d}% from baseline's {:f} to {:f}".format( + int(tolerance * 100), baseline, current + ) -def compare_throughput(case): - baseline_root = case.get_value("BASELINE_ROOT") + return below_tolerance, comment - baseline_name = case.get_value("BASECMP_CASE") - baseline_dir = os.path.join(baseline_root, baseline_name) +def compare_throughput(case, baseline_dir=None): + if baseline_dir is None: + baseline_root = case.get_value("BASELINE_ROOT") + + baseline_name = case.get_value("BASECMP_CASE") + + baseline_dir = os.path.join(baseline_root, baseline_name) latest_cpl_logs = get_latest_cpl_logs(case) - diff, baseline, current = [None]*3 + diff, baseline, current = [None] * 3 + comment = "" # Do we need this loop, when are there multiple cpl logs? for cpllog in latest_cpl_logs: try: baseline = read_baseline_tput(baseline_dir) - except (FileNotFoundError, IndexError, ValueError) as e: - logger.debug("Could not read baseline throughput: %s", e) + except FileNotFoundError as e: + comment = f"Could not read baseline throughput file: {e!s}" + + logger.debug(comment) continue @@ -106,11 +126,9 @@ def compare_throughput(case): diff = (baseline - current) / baseline except (ValueError, TypeError): # Should we default the diff to 0.0 as with _compare_current_memory? - logger.debug( - "Could not determine change in throughput between baseline %s and current %s", - baseline, - current, - ) + comment = f"Could not determine diff with baseline {baseline!r} and current {current!r}" + + logger.debug(comment) continue @@ -129,9 +147,19 @@ def compare_throughput(case): if diff is not None: below_tolerance = diff < tolerance - return below_tolerance, diff, tolerance, baseline, current + if below_tolerance: + comment = "TPUTCOMP: Computation time changed by {:.2f}% relative to baseline".format( + diff * 100 + ) + else: + comment = "Error: TPUTCOMP: Computation time increase > {:d}% from baseline".format( + int(tolerance * 100) + ) + + return below_tolerance, comment + -def write_baseline_tput(baseline_dir, cpllog): +def write_baseline_tput(baseline_dir, tput): """ Writes baseline throughput to file. @@ -146,20 +174,17 @@ def write_baseline_tput(baseline_dir, cpllog): """ tput_file = os.path.join(baseline_dir, "cpl-tput.log") - tput = get_throughput(cpllog) - with open(tput_file, "w") as fd: fd.write("# Throughput in simulated years per compute day\n") - fd.write( - "# A -1 indicates no throughput data was available from the test\n" - ) + fd.write("# A -1 indicates no throughput data was available from the test\n") if tput is None: fd.write("-1") else: fd.write(str(tput)) -def write_baseline_mem(baseline_dir, cpllog): + +def write_baseline_mem(baseline_dir, mem): """ Writes baseline memory usage highwater to file. @@ -174,19 +199,16 @@ def write_baseline_mem(baseline_dir, cpllog): """ mem_file = os.path.join(baseline_dir, "cpl-mem.log") - mem = get_mem_usage(cpllog) - with open(mem_file, "w") as fd: fd.write("# Memory usage highwater\n") - fd.write( - "# A -1 indicates no memory usage data was available from the test\n" - ) + fd.write("# A -1 indicates no memory usage data was available from the test\n") try: fd.write(str(mem[-1][1])) except IndexError: fd.write("-1") + def read_baseline_tput(baseline_dir): """ Reads throughput baseline. @@ -220,6 +242,7 @@ def read_baseline_tput(baseline_dir): return tput + def read_baseline_mem(baseline_dir): """ Reads memory usage highwater baseline. @@ -242,9 +265,7 @@ def read_baseline_mem(baseline_dir): If baseline file does not exist. """ try: - memory = read_baseline_value( - os.path.join(baseline_dir, "cpl-mem.log") - ) + memory = read_baseline_value(os.path.join(baseline_dir, "cpl-mem.log")) except (IndexError, ValueError): memory = 0 @@ -253,6 +274,7 @@ def read_baseline_mem(baseline_dir): return memory + def read_baseline_value(baseline_file): """ Read baseline value from file. @@ -281,15 +303,14 @@ def read_baseline_value(baseline_file): return float(lines[0]) + def get_mem_usage(cpllog): """ Examine memory usage as recorded in the cpl log file and look for unexpected increases. """ memlist = [] - meminfo = re.compile( - r".*model date =\s+(\w+).*memory =\s+(\d+\.?\d+).*highwater" - ) + meminfo = re.compile(r".*model date =\s+(\w+).*memory =\s+(\d+\.?\d+).*highwater") if cpllog is not None and os.path.isfile(cpllog): if ".gz" == cpllog[-3:]: fopen = gzip.open @@ -305,6 +326,7 @@ def get_mem_usage(cpllog): memlist.pop() return memlist + def get_throughput(cpllog): """ Examine memory usage as recorded in the cpl log file and look for unexpected @@ -317,4 +339,3 @@ def get_throughput(cpllog): if m: return float(m.group(1)) return None - diff --git a/CIME/tests/test_unit_baselines.py b/CIME/tests/test_unit_baselines.py index 77ce61ac20f..1d4f11d1c1e 100644 --- a/CIME/tests/test_unit_baselines.py +++ b/CIME/tests/test_unit_baselines.py @@ -9,13 +9,12 @@ from CIME import baselines from CIME.tests.test_unit_system_tests import CPLLOG + def create_mock_case(tempdir, get_latest_cpl_logs): caseroot = Path(tempdir, "0", "caseroot") rundir = caseroot / "run" - get_latest_cpl_logs.return_value = ( - str(rundir / "cpl.log.gz"), - ) + get_latest_cpl_logs.return_value = (str(rundir / "cpl.log.gz"),) baseline_root = Path(tempdir, "baselines") @@ -23,6 +22,7 @@ def create_mock_case(tempdir, get_latest_cpl_logs): return case, caseroot, rundir, baseline_root + class TestUnitBaseline(unittest.TestCase): def test_get_throughput_no_file(self): throughput = baselines.get_throughput("/tmp/cpl.log") @@ -49,16 +49,28 @@ def test_get_mem_usage_gz(self): mem_usage = baselines.get_mem_usage(str(cpl_log_path)) - assert mem_usage == [(10102.0, 1673.89), (10103.0, 1673.89), (10104.0, 1673.89), (10105.0, 1673.89)] + assert mem_usage == [ + (10102.0, 1673.89), + (10103.0, 1673.89), + (10104.0, 1673.89), + (10105.0, 1673.89), + ] @mock.patch("CIME.baselines.os.path.isfile") def test_get_mem_usage(self, isfile): isfile.return_value = True - with mock.patch("builtins.open", mock.mock_open(read_data=CPLLOG.encode("utf-8"))) as mock_file: + with mock.patch( + "builtins.open", mock.mock_open(read_data=CPLLOG.encode("utf-8")) + ) as mock_file: mem_usage = baselines.get_mem_usage("/tmp/cpl.log") - assert mem_usage == [(10102.0, 1673.89), (10103.0, 1673.89), (10104.0, 1673.89), (10105.0, 1673.89)] + assert mem_usage == [ + (10102.0, 1673.89), + (10103.0, 1673.89), + (10104.0, 1673.89), + (10105.0, 1673.89), + ] def test_read_baseline_mem_empty(self): with mock.patch("builtins.open", mock.mock_open(read_data="")) as mock_file: @@ -96,42 +108,30 @@ def test_read_baseline_tput(self): assert baseline == 200 - @mock.patch("CIME.baselines.get_mem_usage") - def test_write_baseline_mem_no_value(self, get_mem_usage): - get_mem_usage.return_value = [] - + def test_write_baseline_mem_no_value(self): with mock.patch("builtins.open", mock.mock_open()) as mock_file: - baselines.write_baseline_mem("/tmp", "/tmp/cpl.log") + baselines.write_baseline_mem("/tmp", []) mock_file.assert_called_with("/tmp/cpl-mem.log", "w") mock_file.return_value.write.assert_called_with("-1") - @mock.patch("CIME.baselines.get_mem_usage") - def test_write_baseline_mem(self, get_mem_usage): - get_mem_usage.return_value = [(1, 200)] - + def test_write_baseline_mem(self): with mock.patch("builtins.open", mock.mock_open()) as mock_file: - baselines.write_baseline_mem("/tmp", "/tmp/cpl.log") + baselines.write_baseline_mem("/tmp", [(1, 200)]) mock_file.assert_called_with("/tmp/cpl-mem.log", "w") mock_file.return_value.write.assert_called_with("200") - @mock.patch("CIME.baselines.get_throughput") - def test_write_baseline_tput_no_value(self, get_throughput): - get_throughput.return_value = None - + def test_write_baseline_tput_no_value(self): with mock.patch("builtins.open", mock.mock_open()) as mock_file: - baselines.write_baseline_tput("/tmp", "/tmp/cpl.log") + baselines.write_baseline_tput("/tmp", None) mock_file.assert_called_with("/tmp/cpl-tput.log", "w") mock_file.return_value.write.assert_called_with("-1") - @mock.patch("CIME.baselines.get_throughput") - def test_write_baseline_tput(self, get_throughput): - get_throughput.return_value = 200 - + def test_write_baseline_tput(self): with mock.patch("builtins.open", mock.mock_open()) as mock_file: - baselines.write_baseline_tput("/tmp", "/tmp/cpl.log") + baselines.write_baseline_tput("/tmp", 200) mock_file.assert_called_with("/tmp/cpl-tput.log", "w") mock_file.return_value.write.assert_called_with("200") @@ -139,7 +139,9 @@ def test_write_baseline_tput(self, get_throughput): @mock.patch("CIME.baselines.get_throughput") @mock.patch("CIME.baselines.read_baseline_tput") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_throughput_no_baseline_file(self, get_latest_cpl_logs, read_baseline_tput, get_throughput): + def test_compare_throughput_no_baseline_file( + self, get_latest_cpl_logs, read_baseline_tput, get_throughput + ): read_baseline_tput.side_effect = FileNotFoundError get_throughput.return_value = 504 @@ -153,18 +155,17 @@ def test_compare_throughput_no_baseline_file(self, get_latest_cpl_logs, read_bas 0.05, ) - below_tolerance, diff, tolerance, baseline, current = baselines.compare_throughput(case) + (below_tolerance, comment) = baselines.compare_throughput(case) assert below_tolerance is None - assert diff == None - assert tolerance == 0.05 - assert baseline == None - assert current == None + assert comment == "Could not read baseline throughput file: " @mock.patch("CIME.baselines.get_throughput") @mock.patch("CIME.baselines.read_baseline_tput") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_throughput_no_baseline(self, get_latest_cpl_logs, read_baseline_tput, get_throughput): + def test_compare_throughput_no_baseline( + self, get_latest_cpl_logs, read_baseline_tput, get_throughput + ): read_baseline_tput.return_value = None get_throughput.return_value = 504 @@ -178,18 +179,17 @@ def test_compare_throughput_no_baseline(self, get_latest_cpl_logs, read_baseline 0.05, ) - below_tolerance, diff, tolerance, baseline, current = baselines.compare_throughput(case) + (below_tolerance, comment) = baselines.compare_throughput(case) assert below_tolerance is None - assert diff == None - assert tolerance == 0.05 - assert baseline == None - assert current == 504 + assert comment == "Could not determine diff with baseline None and current 504" @mock.patch("CIME.baselines.get_throughput") @mock.patch("CIME.baselines.read_baseline_tput") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_throughput_no_tolerance(self, get_latest_cpl_logs, read_baseline_tput, get_throughput): + def test_compare_throughput_no_tolerance( + self, get_latest_cpl_logs, read_baseline_tput, get_throughput + ): read_baseline_tput.return_value = 500 get_throughput.return_value = 504 @@ -200,21 +200,49 @@ def test_compare_throughput_no_tolerance(self, get_latest_cpl_logs, read_baselin case.get_value.side_effect = ( str(baseline_root), "master/ERIO.ne30_g16_rx1.A.docker_gnu", - None + None, ) - below_tolerance, diff, tolerance, baseline, current = baselines.compare_throughput(case) + (below_tolerance, comment) = baselines.compare_throughput(case) assert below_tolerance - assert diff == -0.008 - assert tolerance == 0.1 - assert baseline == 500 - assert current == 504 + assert ( + comment + == "TPUTCOMP: Computation time changed by -0.80% relative to baseline" + ) @mock.patch("CIME.baselines.get_throughput") @mock.patch("CIME.baselines.read_baseline_tput") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_throughput(self, get_latest_cpl_logs, read_baseline_tput, get_throughput): + def test_compare_throughput_above_threshold( + self, get_latest_cpl_logs, read_baseline_tput, get_throughput + ): + read_baseline_tput.return_value = 1000 + + get_throughput.return_value = 504 + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + case.get_value.side_effect = ( + str(baseline_root), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + 0.05, + ) + + (below_tolerance, comment) = baselines.compare_throughput(case) + + assert not below_tolerance + assert ( + comment == "Error: TPUTCOMP: Computation time increase > 5% from baseline" + ) + + @mock.patch("CIME.baselines.get_throughput") + @mock.patch("CIME.baselines.read_baseline_tput") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_compare_throughput( + self, get_latest_cpl_logs, read_baseline_tput, get_throughput + ): read_baseline_tput.return_value = 500 get_throughput.return_value = 504 @@ -228,18 +256,20 @@ def test_compare_throughput(self, get_latest_cpl_logs, read_baseline_tput, get_t 0.05, ) - below_tolerance, diff, tolerance, baseline, current = baselines.compare_throughput(case) + (below_tolerance, comment) = baselines.compare_throughput(case) assert below_tolerance - assert diff == -0.008 - assert tolerance == 0.05 - assert baseline == 500 - assert current == 504 + assert ( + comment + == "TPUTCOMP: Computation time changed by -0.80% relative to baseline" + ) @mock.patch("CIME.baselines.get_mem_usage") @mock.patch("CIME.baselines.read_baseline_mem") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_memory_no_baseline(self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage): + def test_compare_memory_no_baseline( + self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage + ): read_baseline_mem.return_value = None get_mem_usage.return_value = [ @@ -258,18 +288,20 @@ def test_compare_memory_no_baseline(self, get_latest_cpl_logs, read_baseline_mem 0.05, ) - below_tolerance, diff, tolerance, baseline, current = baselines.compare_memory(case) + (below_tolerance, comment) = baselines.compare_memory(case) assert below_tolerance - assert diff == 0.0 - assert tolerance == 0.05 - assert baseline == 0.0 - assert current == 1003.0 + assert ( + comment + == "MEMCOMP: Memory usage highwater has changed by 0.00% relative to baseline" + ) @mock.patch("CIME.baselines.get_mem_usage") @mock.patch("CIME.baselines.read_baseline_mem") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_memory_not_enough_samples(self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage): + def test_compare_memory_not_enough_samples( + self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage + ): read_baseline_mem.return_value = 1000.0 get_mem_usage.return_value = [ @@ -286,18 +318,17 @@ def test_compare_memory_not_enough_samples(self, get_latest_cpl_logs, read_basel 0.05, ) - below_tolerance, diff, tolerance, baseline, current = baselines.compare_memory(case) + (below_tolerance, comment) = baselines.compare_memory(case) assert below_tolerance is None - assert diff == None - assert tolerance == 0.05 - assert baseline == 1000.0 - assert current == None + assert comment == "Found 2 memory usage samples, need atleast 4" @mock.patch("CIME.baselines.get_mem_usage") @mock.patch("CIME.baselines.read_baseline_mem") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_memory_no_baseline_file(self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage): + def test_compare_memory_no_baseline_file( + self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage + ): read_baseline_mem.side_effect = FileNotFoundError get_mem_usage.return_value = [ @@ -316,18 +347,17 @@ def test_compare_memory_no_baseline_file(self, get_latest_cpl_logs, read_baselin 0.05, ) - below_tolerance, diff, tolerance, baseline, current = baselines.compare_memory(case) + (below_tolerance, comment) = baselines.compare_memory(case) assert below_tolerance is None - assert diff == None - assert tolerance == 0.05 - assert baseline == None - assert current == None + assert comment == "Could not read baseline memory usage: " @mock.patch("CIME.baselines.get_mem_usage") @mock.patch("CIME.baselines.read_baseline_mem") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_memory_no_tolerance(self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage): + def test_compare_memory_no_tolerance( + self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage + ): read_baseline_mem.return_value = 1000.0 get_mem_usage.return_value = [ @@ -346,18 +376,52 @@ def test_compare_memory_no_tolerance(self, get_latest_cpl_logs, read_baseline_me None, ) - below_tolerance, diff, tolerance, baseline, current = baselines.compare_memory(case) + (below_tolerance, comment) = baselines.compare_memory(case) assert below_tolerance - assert diff == 0.003 - assert tolerance == 0.1 - assert baseline == 1000.0 - assert current == 1003.0 + assert ( + comment + == "MEMCOMP: Memory usage highwater has changed by 0.30% relative to baseline" + ) + + @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.read_baseline_mem") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_compare_memory_above_threshold( + self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage + ): + read_baseline_mem.return_value = 1000.0 + + get_mem_usage.return_value = [ + (1, 2000.0), + (2, 2001.0), + (3, 2002.0), + (4, 2003.0), + ] + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + case.get_value.side_effect = ( + str(baseline_root), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + 0.05, + ) + + (below_tolerance, comment) = baselines.compare_memory(case) + + assert not below_tolerance + assert ( + comment + == "Error: Memory usage increase >5% from baseline's 1000.000000 to 2003.000000" + ) @mock.patch("CIME.baselines.get_mem_usage") @mock.patch("CIME.baselines.read_baseline_mem") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_memory(self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage): + def test_compare_memory( + self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage + ): read_baseline_mem.return_value = 1000.0 get_mem_usage.return_value = [ @@ -376,13 +440,13 @@ def test_compare_memory(self, get_latest_cpl_logs, read_baseline_mem, get_mem_us 0.05, ) - below_tolerance, diff, tolerance, baseline, current = baselines.compare_memory(case) + (below_tolerance, comment) = baselines.compare_memory(case) assert below_tolerance - assert diff == 0.003 - assert tolerance == 0.05 - assert baseline == 1000.0 - assert current == 1003.0 + assert ( + comment + == "MEMCOMP: Memory usage highwater has changed by 0.30% relative to baseline" + ) def test_get_latest_cpl_logs_found_multiple(self): with tempfile.TemporaryDirectory() as tempdir: @@ -404,7 +468,9 @@ def test_get_latest_cpl_logs_found_multiple(self): latest_cpl_logs = baselines.get_latest_cpl_logs(case) assert len(latest_cpl_logs) == 2 - assert sorted(latest_cpl_logs) == sorted([str(cpl_log_path), str(cpl_log_2_path)]) + assert sorted(latest_cpl_logs) == sorted( + [str(cpl_log_path), str(cpl_log_2_path)] + ) def test_get_latest_cpl_logs_found_single(self): with tempfile.TemporaryDirectory() as tempdir: diff --git a/CIME/tests/test_unit_system_tests.py b/CIME/tests/test_unit_system_tests.py index e81bd6bca10..a43b7ef8f80 100644 --- a/CIME/tests/test_unit_system_tests.py +++ b/CIME/tests/test_unit_system_tests.py @@ -69,7 +69,10 @@ class TestCaseSubmit(unittest.TestCase): @mock.patch("CIME.SystemTests.system_tests_common.compare_throughput") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") def test_compare_throughput(self, append_testlog, compare_throughput): - compare_throughput.return_value = (True, 0.02, 0.05, 200, 201) + compare_throughput.return_value = ( + True, + "TPUTCOMP: Computation time changed by 2.00% relative to baseline", + ) with tempfile.TemporaryDirectory() as tempdir: caseroot = Path(tempdir) / "caseroot" @@ -96,7 +99,7 @@ def test_compare_throughput(self, append_testlog, compare_throughput): @mock.patch("CIME.SystemTests.system_tests_common.compare_throughput") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") def test_compare_throughput_error_diff(self, append_testlog, compare_throughput): - compare_throughput.return_value = (None, 0.02, 0.05, 200, 201) + compare_throughput.return_value = (None, "Error diff value") with tempfile.TemporaryDirectory() as tempdir: caseroot = Path(tempdir) / "caseroot" @@ -120,7 +123,10 @@ def test_compare_throughput_error_diff(self, append_testlog, compare_throughput) @mock.patch("CIME.SystemTests.system_tests_common.compare_throughput") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") def test_compare_throughput_fail(self, append_testlog, compare_throughput): - compare_throughput.return_value = (False, 0.02, 0.05, 200, 201) + compare_throughput.return_value = ( + False, + "Error: TPUTCOMP: Computation time increase > 5% from baseline", + ) with tempfile.TemporaryDirectory() as tempdir: caseroot = Path(tempdir) / "caseroot" @@ -139,10 +145,6 @@ def test_compare_throughput_fail(self, append_testlog, compare_throughput): assert common._test_status.get_overall_test_status() == ("PASS", None) - append_testlog.assert_any_call( - "TPUTCOMP: Computation time changed by 2.00% relative to baseline", - str(caseroot), - ) append_testlog.assert_any_call( "Error: TPUTCOMP: Computation time increase > 5% from baseline", str(caseroot), @@ -151,7 +153,10 @@ def test_compare_throughput_fail(self, append_testlog, compare_throughput): @mock.patch("CIME.SystemTests.system_tests_common.compare_memory") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") def test_compare_memory(self, append_testlog, compare_memory): - compare_memory.return_value = (True, 0.02, 0.05, 1000, 1002) + compare_memory.return_value = ( + True, + "MEMCOMP: Memory usage highwater has changed by 2.00% relative to baseline", + ) with tempfile.TemporaryDirectory() as tempdir: caseroot = Path(tempdir) / "caseroot" @@ -178,7 +183,7 @@ def test_compare_memory(self, append_testlog, compare_memory): @mock.patch("CIME.SystemTests.system_tests_common.compare_memory") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") def test_compare_memory_erorr_diff(self, append_testlog, compare_memory): - compare_memory.return_value = (None, 0.02, 0.05, 1000, 1002) + compare_memory.return_value = (None, "Error diff value") with tempfile.TemporaryDirectory() as tempdir: caseroot = Path(tempdir) / "caseroot" @@ -202,7 +207,10 @@ def test_compare_memory_erorr_diff(self, append_testlog, compare_memory): @mock.patch("CIME.SystemTests.system_tests_common.compare_memory") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") def test_compare_memory_erorr_fail(self, append_testlog, compare_memory): - compare_memory.return_value = (False, 0.02, 0.05, 1000, 1002) + compare_memory.return_value = ( + False, + "Error: Memory usage increase >5% from baseline's 1000.000000 to 1002.000000", + ) with tempfile.TemporaryDirectory() as tempdir: caseroot = Path(tempdir) / "caseroot" @@ -221,10 +229,6 @@ def test_compare_memory_erorr_fail(self, append_testlog, compare_memory): assert common._test_status.get_overall_test_status() == ("PASS", None) - append_testlog.assert_any_call( - "MEMCOMP: Memory usage highwater has changed by 2.00% relative to baseline", - str(caseroot), - ) append_testlog.assert_any_call( "Error: Memory usage increase >5% from baseline's 1000.000000 to 1002.000000", str(caseroot), From 9bab6396e9f1007282c7b168618d380752a493ca Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 22 Sep 2023 22:28:35 -0700 Subject: [PATCH 08/57] Adds --tput-only and --mem-only --- CIME/Tools/bless_test_results | 14 +++- CIME/bless_test_results.py | 127 ++++++++++++++++++++++++++++++++-- 2 files changed, 135 insertions(+), 6 deletions(-) diff --git a/CIME/Tools/bless_test_results b/CIME/Tools/bless_test_results index d630aff69bd..e4236c172f6 100755 --- a/CIME/Tools/bless_test_results +++ b/CIME/Tools/bless_test_results @@ -59,6 +59,12 @@ OR "--hist-only", action="store_true", help="Only analyze history files." ) + parser.add_argument( + "--tput-only", action="store_true", help="Only analyze throughput." + ) + + parser.add_argument("--mem-only", action="store_true", help="Only analyze memory.") + parser.add_argument( "-b", "--baseline-name", @@ -140,7 +146,7 @@ OR "Makes no sense to use -r and -f simultaneously", ) expect( - not (args.namelists_only and args.hist_only), + sum([args.namelists_only, args.hist_only, args.tput_only, args.mem_only]) <= 1, "Makes no sense to use --namelists-only and --hist-only simultaneously", ) @@ -152,6 +158,8 @@ OR args.test_id, args.namelists_only, args.hist_only, + args.tput_only, + args.mem_only, args.report_only, args.force, args.pes_file, @@ -173,6 +181,8 @@ def _main_func(description): test_id, namelists_only, hist_only, + tput_only, + mem_only, report_only, force, pes_file, @@ -190,6 +200,8 @@ def _main_func(description): test_id=test_id, namelists_only=namelists_only, hist_only=hist_only, + tput_only=tput_only, + mem_only=mem_only, report_only=report_only, force=force, pesfile=pes_file, diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index 62637851cb4..a44c2ec5fe5 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -11,10 +11,94 @@ from CIME.hist_utils import generate_baseline, compare_baseline from CIME.case import Case from CIME.test_utils import get_test_status_files +from CIME.baselines import ( + get_latest_cpl_logs, + get_mem_usage, + get_throughput, + compare_throughput, + compare_memory, + write_baseline_tput, + write_baseline_mem, +) import os, time logger = logging.getLogger(__name__) + +def bless_throughput( + case, + test_name, + baseline_root, + baseline_name, + report_only, + force, +): + success = True + reason = None + + baseline_dir = os.path.join( + baseline_root, baseline_name, case.get_value("CASEBASEID") + ) + + below_threshold, comment = compare_throughput(case, baseline_dir=baseline_dir) + + if below_threshold: + logger.info("Diff appears to have been already resolved.") + else: + logger.info(comment) + + if not report_only and ( + force or input("Update this diff (y/n)? ").upper() in ["Y", "YES"] + ): + try: + latest_cpl_logs = get_latest_cpl_logs(case) + current = get_throughput(latest_cpl_logs[0]) + write_baseline_tput(baseline_dir, current) + except Exception as e: + success = False + + reason = f"Failed to write baseline throughput for {test_name!r}: {e!s}" + + return success, reason + + +def bless_memory( + case, + test_name, + baseline_root, + baseline_name, + report_only, + force, +): + success = True + reason = None + + baseline_dir = os.path.join( + baseline_root, baseline_name, case.get_value("CASEBASEID") + ) + + below_threshold, comment = compare_memory(case, baseline_dir=baseline_dir) + + if below_threshold: + logger.info("Diff appears to have been already resolved.") + else: + logger.info(comment) + + if not report_only and ( + force or input("Update this diff (y/n)? ").upper() in ["Y", "YES"] + ): + try: + latest_cpl_logs = get_latest_cpl_logs(case) + current = get_mem_usage(latest_cpl_logs[0]) + write_baseline_mem(baseline_dir, current) + except Exception as e: + success = False + + reason = f"Failed to write baseline memory usage for test {test_name!r}: {e!s}" + + return success, reason + + ############################################################################### def bless_namelists( test_name, @@ -112,6 +196,8 @@ def bless_test_results( test_id=None, namelists_only=False, hist_only=False, + tput_only=False, + mem_only=False, report_only=False, force=False, pesfile=None, @@ -172,9 +258,15 @@ def bless_test_results( if bless_tests in [[], None] or CIME.utils.match_any( test_name, bless_tests_counts ): - overall_result, phase = ts.get_overall_test_status( - ignore_namelists=True, ignore_memleak=True - ) + ts_kwargs = dict(ignore_namelists=True, ignore_memleak=True) + + if tput_only: + ts_kwargs["check_throughput"] = True + + if mem_only: + ts_kwargs["check_memory"] = True + + overall_result, phase = ts.get_overall_test_status(**ts_kwargs) # See if we need to bless namelist if not hist_only: @@ -219,14 +311,13 @@ def bless_test_results( hist_bless = False # Now, do the bless - if not nl_bless and not hist_bless: + if not nl_bless and not hist_bless and not tput_only and not mem_only: logger.info( "Nothing to bless for test: {}, overall status: {}".format( test_name, overall_result ) ) else: - logger.info( "###############################################################################" ) @@ -305,6 +396,32 @@ def bless_test_results( if not success: broken_blesses.append((test_name, reason)) + if tput_only: + success, reason = bless_throughput( + case, + test_name, + baseline_root, + baseline_name, + report_only, + force, + ) + + if not success: + broken_blesses.append((test_name, reason)) + + if mem_only: + success, reason = bless_memory( + case, + test_name, + baseline_root, + baseline_name, + report_only, + force, + ) + + if not success: + broken_blesses.append((test_name, reason)) + # Emit a warning if items in bless_tests did not match anything if bless_tests: for bless_test, bless_count in bless_tests_counts.items(): From 52c8d7b56a887efb0d53febfbf9a8e006e7b25fa Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 29 Sep 2023 12:36:36 -0700 Subject: [PATCH 09/57] Fixes failing test --- CIME/tests/test_unit_system_tests.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/CIME/tests/test_unit_system_tests.py b/CIME/tests/test_unit_system_tests.py index a43b7ef8f80..066d4fbd193 100644 --- a/CIME/tests/test_unit_system_tests.py +++ b/CIME/tests/test_unit_system_tests.py @@ -8,6 +8,7 @@ from unittest import mock from pathlib import Path +from CIME.config import Config from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo from CIME.SystemTests.system_tests_compare_n import SystemTestsCompareN @@ -240,7 +241,7 @@ def test_generate_baseline(self): tempdir, cpllog_data=CPLLOG ) - case.get_value.side_effect = ( + get_value_calls = [ str(caseroot), "ERIO.ne30_g16_rx1.A.docker_gnu", "mct", @@ -253,13 +254,17 @@ def test_generate_baseline(self): str(run_dir), "ERIO", "ERIO.ne30_g16_rx1.A.docker_gnu", - os.getcwd(), "master/ERIO.ne30_g16_rx1.A.docker_gnu", str(baseline_root), "master/ERIO.ne30_g16_rx1.A.docker_gnu", str(run_dir), "mct", - ) + ] + + if Config.instance().create_bless_log: + get_value_calls.insert(12, os.getcwd()) + + case.get_value.side_effect = get_value_calls common = SystemTestsCommon(case) From 6dc623cb3493ef5265d5b99db2f207727eae994f Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Mon, 2 Oct 2023 17:03:09 -0700 Subject: [PATCH 10/57] Refactors and updates bless_test_results --- CIME/Tools/bless_test_results | 202 ++++++++++++++-------------------- CIME/bless_test_results.py | 1 + CIME/utils.py | 11 +- 3 files changed, 90 insertions(+), 124 deletions(-) diff --git a/CIME/Tools/bless_test_results b/CIME/Tools/bless_test_results index e4236c172f6..cb6bd2f972a 100755 --- a/CIME/Tools/bless_test_results +++ b/CIME/Tools/bless_test_results @@ -8,20 +8,21 @@ blessing of diffs. You may need to load modules for cprnc to work. """ - from standard_script_setup import * from CIME.utils import expect from CIME.XML.machines import Machines from CIME.bless_test_results import bless_test_results -import argparse, sys, os +import argparse +import sys +import os +import logging _MACHINE = Machines() -############################################################################### + def parse_command_line(args, description): - ############################################################################### parser = argparse.ArgumentParser( usage="""\n{0} [-n] [-r ] [-b ] [-c ] [ ...] [--verbose] OR @@ -45,41 +46,18 @@ OR formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - default_compiler = _MACHINE.get_default_compiler() - scratch_root = _MACHINE.get_value("CIME_OUTPUT_ROOT") - default_testroot = os.path.join(scratch_root) + create_bless_options(parser) - CIME.utils.setup_standard_logging_options(parser) + create_baseline_options(parser) - parser.add_argument( - "-n", "--namelists-only", action="store_true", help="Only analyze namelists." - ) + create_test_options(parser) - parser.add_argument( - "--hist-only", action="store_true", help="Only analyze history files." - ) - - parser.add_argument( - "--tput-only", action="store_true", help="Only analyze throughput." - ) - - parser.add_argument("--mem-only", action="store_true", help="Only analyze memory.") - - parser.add_argument( - "-b", - "--baseline-name", - help="Name of baselines to use. Default will use BASELINE_NAME_CMP first if possible, otherwise branch name.", - ) - - parser.add_argument( - "--baseline-root", - help="Root of baselines. Default will use the BASELINE_ROOT from the case.", - ) + CIME.utils.setup_standard_logging_options(parser) parser.add_argument( "-c", "--compiler", - default=default_compiler, + default=_MACHINE.get_default_compiler(), help="Compiler of run you want to bless", ) @@ -91,36 +69,15 @@ OR "This option forces the bless to happen regardless.", ) - parser.add_argument( + mutual_execution = parser.add_mutually_exclusive_group() + + mutual_execution.add_argument( "--report-only", action="store_true", help="Only report what files will be overwritten and why. Caution is a good thing when updating baselines", ) - parser.add_argument( - "-r", - "--test-root", - default=default_testroot, - help="Path to test results that are being blessed", - ) - - parser.add_argument( - "--new-test-root", - help="If bless_test_results needs to create cases (for blessing namelists), use this root area", - ) - - parser.add_argument( - "--new-test-id", - help="If bless_test_results needs to create cases (for blessing namelists), use this test id", - ) - - parser.add_argument( - "-t", - "--test-id", - help="Limit processes to case dirs matching this test-id. Can be useful if mutiple runs dumped into the same dir.", - ) - - parser.add_argument( + mutual_execution.add_argument( "-f", "--force", action="store_true", @@ -141,79 +98,82 @@ OR args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) - expect( - not (args.report_only and args.force), - "Makes no sense to use -r and -f simultaneously", + return vars(args) + + +def create_bless_options(parser): + bless_group = parser.add_argument_group("Bless options") + + mutual_bless_group = bless_group.add_mutually_exclusive_group(required=True) + + mutual_bless_group.add_argument( + "-n", "--namelists-only", action="store_true", help="Only analyze namelists." ) - expect( - sum([args.namelists_only, args.hist_only, args.tput_only, args.mem_only]) <= 1, - "Makes no sense to use --namelists-only and --hist-only simultaneously", + + mutual_bless_group.add_argument( + "--hist-only", action="store_true", help="Only analyze history files." + ) + + mutual_bless_group.add_argument( + "--tput-only", action="store_true", help="Only analyze throughput." ) - return ( - args.baseline_name, - args.baseline_root, - args.test_root, - args.compiler, - args.test_id, - args.namelists_only, - args.hist_only, - args.tput_only, - args.mem_only, - args.report_only, - args.force, - args.pes_file, - args.bless_tests, - args.no_skip_pass, - args.new_test_root, - args.new_test_id, + mutual_bless_group.add_argument( + "--mem-only", action="store_true", help="Only analyze memory." ) -############################################################################### -def _main_func(description): - ############################################################################### - ( - baseline_name, - baseline_root, - test_root, - compiler, - test_id, - namelists_only, - hist_only, - tput_only, - mem_only, - report_only, - force, - pes_file, - bless_tests, - no_skip_pass, - new_test_root, - new_test_id, - ) = parse_command_line(sys.argv, description) - - success = bless_test_results( - baseline_name, - baseline_root, - test_root, - compiler, - test_id=test_id, - namelists_only=namelists_only, - hist_only=hist_only, - tput_only=tput_only, - mem_only=mem_only, - report_only=report_only, - force=force, - pesfile=pes_file, - bless_tests=bless_tests, - no_skip_pass=no_skip_pass, - new_test_root=new_test_root, - new_test_id=new_test_id, +def create_baseline_options(parser): + baseline_group = parser.add_argument_group("Baseline options") + + baseline_group.add_argument( + "-b", + "--baseline-name", + help="Name of baselines to use. Default will use BASELINE_NAME_CMP first if possible, otherwise branch name.", ) - sys.exit(0 if success else 1) + baseline_group.add_argument( + "--baseline-root", + help="Root of baselines. Default will use the BASELINE_ROOT from the case.", + ) + + +def create_test_options(parser): + default_testroot = _MACHINE.get_value("CIME_OUTPUT_ROOT") + + test_group = parser.add_argument_group("Test options") + + test_group.add_argument( + "-r", + "--test-root", + default=default_testroot, + help="Path to test results that are being blessed", + ) + + test_group.add_argument( + "--new-test-root", + help="If bless_test_results needs to create cases (for blessing namelists), use this root area", + ) + + test_group.add_argument( + "--new-test-id", + help="If bless_test_results needs to create cases (for blessing namelists), use this test id", + ) + + test_group.add_argument( + "-t", + "--test-id", + help="Limit processes to case dirs matching this test-id. Can be useful if mutiple runs dumped into the same dir.", + ) + + +def _main_func(description): + kwargs = parse_command_line(sys.argv, description) + + success = bless_test_results(**kwargs) + + sys.exit(0 if success else 1) -############################################################################### if __name__ == "__main__": _main_func(__doc__) diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index a44c2ec5fe5..2c7d5e671a7 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -205,6 +205,7 @@ def bless_test_results( no_skip_pass=False, new_test_root=None, new_test_id=None, + **_, # Capture all for extra ): ############################################################################### test_status_files = get_test_status_files(test_root, compiler, test_id=test_id) diff --git a/CIME/utils.py b/CIME/utils.py index 1293f6d3171..1a32319c8d1 100644 --- a/CIME/utils.py +++ b/CIME/utils.py @@ -1625,20 +1625,25 @@ def find_files(rootdir, pattern): def setup_standard_logging_options(parser): + group = parser.add_argument_group("Logging options") + helpfile = os.path.join(os.getcwd(), os.path.basename("{}.log".format(sys.argv[0]))) - parser.add_argument( + + group.add_argument( "-d", "--debug", action="store_true", help="Print debug information (very verbose) to file {}".format(helpfile), ) - parser.add_argument( + + group.add_argument( "-v", "--verbose", action="store_true", help="Add additional context (time and file) to log messages", ) - parser.add_argument( + + group.add_argument( "-s", "--silent", action="store_true", From 94502568282a5ac578313925825ac97c2a17c4ca Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 5 Oct 2023 10:00:58 -0700 Subject: [PATCH 11/57] Refactors base config class --- CIME/config.py | 227 +++++++++++++++++++++++++------------------------ 1 file changed, 118 insertions(+), 109 deletions(-) diff --git a/CIME/config.py b/CIME/config.py index 3cef6cc0530..d2306d354d0 100644 --- a/CIME/config.py +++ b/CIME/config.py @@ -9,19 +9,132 @@ logger = logging.getLogger(__name__) -class Config: +class ConfigBase: def __new__(cls): if not hasattr(cls, "_instance"): - cls._instance = super(Config, cls).__new__(cls) + cls._instance = super(ConfigBase, cls).__new__(cls) return cls._instance def __init__(self): - if getattr(self, "_loaded", False): - return - self._attribute_config = {} + @property + def loaded(self): + return getattr(self, "_loaded", False) + + @classmethod + def instance(cls): + """Access singleton. + + Explicit way to access singleton, same as calling constructor. + """ + return cls() + + @classmethod + def load(cls, customize_path): + obj = cls() + + logger.debug("Searching %r for files to load", customize_path) + + customize_files = glob.glob(f"{customize_path}/**/*.py", recursive=True) + + # filter out any tests + customize_files = [ + x for x in customize_files if "tests" not in x and "conftest" not in x + ] + + customize_module_spec = importlib.machinery.ModuleSpec("cime_customize", None) + + customize_module = importlib.util.module_from_spec(customize_module_spec) + + sys.modules["CIME.customize"] = customize_module + + for x in sorted(customize_files): + obj._load_file(x, customize_module) + + setattr(obj, "_loaded", True) + + return obj + + def _load_file(self, file_path, customize_module): + logger.debug("Loading file %r", file_path) + + raw_config = utils.import_from_file("raw_config", file_path) + + # filter user define variables and functions + user_defined = [x for x in dir(raw_config) if not x.endswith("__")] + + # set values on this object, will overwrite existing + for x in user_defined: + try: + value = getattr(raw_config, x) + except AttributeError: + # should never hit this + logger.fatal("Attribute %r missing on obejct", x) + + sys.exit(1) + else: + setattr(customize_module, x, value) + + self._set_attribute(x, value) + + def _set_attribute(self, name, value, desc=None): + if hasattr(self, name): + logger.debug("Overwriting %r attribute", name) + + logger.debug("Setting attribute %r with value %r", name, value) + + setattr(self, name, value) + + self._attribute_config[name] = { + "desc": desc, + "default": value, + } + + def print_rst_table(self): + max_variable = max([len(x) for x in self._attribute_config.keys()]) + max_default = max( + [len(str(x["default"])) for x in self._attribute_config.values()] + ) + max_type = max( + [len(type(x["default"]).__name__) for x in self._attribute_config.values()] + ) + max_desc = max([len(x["desc"]) for x in self._attribute_config.values()]) + + divider_row = ( + f"{'='*max_variable} {'='*max_default} {'='*max_type} {'='*max_desc}" + ) + + rows = [ + divider_row, + f"Variable{' '*(max_variable-8)} Default{' '*(max_default-7)} Type{' '*(max_type-4)} Description{' '*(max_desc-11)}", + divider_row, + ] + + for variable, value in sorted( + self._attribute_config.items(), key=lambda x: x[0] + ): + variable_fill = max_variable - len(variable) + default_fill = max_default - len(str(value["default"])) + type_fill = max_type - len(type(value["default"]).__name__) + + rows.append( + f"{variable}{' '*variable_fill} {value['default']}{' '*default_fill} {type(value['default']).__name__}{' '*type_fill} {value['desc']}" + ) + + rows.append(divider_row) + + print("\n".join(rows)) + + +class Config(ConfigBase): + def __init__(self): + super().__init__() + + if self.loaded: + return + self._set_attribute( "additional_archive_components", ("drv", "dart"), @@ -195,107 +308,3 @@ def __init__(self): "{srcroot}/libraries/mct", desc="Sets the path to the mct library.", ) - - @classmethod - def instance(cls): - """Access singleton. - - Explicit way to access singleton, same as calling constructor. - """ - return cls() - - @classmethod - def load(cls, customize_path): - obj = cls() - - logger.debug("Searching %r for files to load", customize_path) - - customize_files = glob.glob(f"{customize_path}/**/*.py", recursive=True) - - # filter out any tests - customize_files = [ - x for x in customize_files if "tests" not in x and "conftest" not in x - ] - - customize_module_spec = importlib.machinery.ModuleSpec("cime_customize", None) - - customize_module = importlib.util.module_from_spec(customize_module_spec) - - sys.modules["CIME.customize"] = customize_module - - for x in sorted(customize_files): - obj._load_file(x, customize_module) - - setattr(obj, "_loaded", True) - - return obj - - def _load_file(self, file_path, customize_module): - logger.debug("Loading file %r", file_path) - - raw_config = utils.import_from_file("raw_config", file_path) - - # filter user define variables and functions - user_defined = [x for x in dir(raw_config) if not x.endswith("__")] - - # set values on this object, will overwrite existing - for x in user_defined: - try: - value = getattr(raw_config, x) - except AttributeError: - # should never hit this - logger.fatal("Attribute %r missing on obejct", x) - - sys.exit(1) - else: - setattr(customize_module, x, value) - - self._set_attribute(x, value) - - def _set_attribute(self, name, value, desc=None): - if hasattr(self, name): - logger.debug("Overwriting %r attribute", name) - - logger.debug("Setting attribute %r with value %r", name, value) - - setattr(self, name, value) - - self._attribute_config[name] = { - "desc": desc, - "default": value, - } - - def print_rst_table(self): - max_variable = max([len(x) for x in self._attribute_config.keys()]) - max_default = max( - [len(str(x["default"])) for x in self._attribute_config.values()] - ) - max_type = max( - [len(type(x["default"]).__name__) for x in self._attribute_config.values()] - ) - max_desc = max([len(x["desc"]) for x in self._attribute_config.values()]) - - divider_row = ( - f"{'='*max_variable} {'='*max_default} {'='*max_type} {'='*max_desc}" - ) - - rows = [ - divider_row, - f"Variable{' '*(max_variable-8)} Default{' '*(max_default-7)} Type{' '*(max_type-4)} Description{' '*(max_desc-11)}", - divider_row, - ] - - for variable, value in sorted( - self._attribute_config.items(), key=lambda x: x[0] - ): - variable_fill = max_variable - len(variable) - default_fill = max_default - len(str(value["default"])) - type_fill = max_type - len(type(value["default"]).__name__) - - rows.append( - f"{variable}{' '*variable_fill} {value['default']}{' '*default_fill} {type(value['default']).__name__}{' '*type_fill} {value['desc']}" - ) - - rows.append(divider_row) - - print("\n".join(rows)) From d948986b4739af86485adb44867f29a95a583628 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 6 Oct 2023 00:25:25 -0700 Subject: [PATCH 12/57] Adds feature for coupler to define custom read/compare functions --- CIME/SystemTests/system_tests_common.py | 19 +- CIME/baselines.py | 359 +++++++++++++++--------- CIME/bless_test_results.py | 14 +- CIME/case/case.py | 7 + CIME/tests/test_unit_baselines.py | 205 +++++++++----- CIME/tests/test_unit_system_tests.py | 13 +- 6 files changed, 398 insertions(+), 219 deletions(-) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index e1c5d59388c..38263cd7cea 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -28,12 +28,10 @@ from CIME.locked_files import LOCKED_DIR, lock_file, is_locked from CIME.baselines import ( get_latest_cpl_logs, - get_throughput, - get_mem_usage, + default_get_mem_usage, compare_memory, compare_throughput, - write_baseline_mem, - write_baseline_tput, + write_baseline, ) import CIME.build as build @@ -636,7 +634,7 @@ def _check_for_memleak(self): with self._test_status: latestcpllogs = get_latest_cpl_logs(self._case) for cpllog in latestcpllogs: - memlist = get_mem_usage(cpllog) + memlist = default_get_mem_usage(self._case, cpllog) if len(memlist) < 3: self._test_status.set_status( @@ -782,23 +780,20 @@ def _generate_baseline(self): # drop the date so that the name is generic newestcpllogfiles = get_latest_cpl_logs(self._case) with SharedArea(): + # TODO ever actually more than one cpl log? for cpllog in newestcpllogfiles: m = re.search(r"/({}.*.log).*.gz".format(self._cpllog), cpllog) + if m is not None: baselog = os.path.join(basegen_dir, m.group(1)) + ".gz" + safe_copy( cpllog, os.path.join(basegen_dir, baselog), preserve_meta=False, ) - tput = get_throughput(cpllog) - - write_baseline_tput(basegen_dir, tput) - - mem = get_mem_usage(cpllog) - - write_baseline_mem(basegen_dir, mem) + write_baseline(self._case, basegen_dir, cpllog) class FakeTest(SystemTestsCommon): diff --git a/CIME/baselines.py b/CIME/baselines.py index cbdb3a2cf61..0f7d1c16b19 100644 --- a/CIME/baselines.py +++ b/CIME/baselines.py @@ -3,6 +3,7 @@ import re import gzip import logging +from CIME.config import Config from CIME.utils import expect logger = logging.getLogger(__name__) @@ -13,13 +14,20 @@ def get_latest_cpl_logs(case): find and return the latest cpl log file in the run directory """ coupler_log_path = case.get_value("RUNDIR") + cpllog_name = "drv" if case.get_value("COMP_INTERFACE") == "nuopc" else "cpl" + cpllogs = glob.glob(os.path.join(coupler_log_path, "{}*.log.*".format(cpllog_name))) + lastcpllogs = [] + if cpllogs: lastcpllogs.append(max(cpllogs, key=os.path.getctime)) + basename = os.path.basename(lastcpllogs[0]) + suffix = basename.split(".", 1)[1] + for log in cpllogs: if log in lastcpllogs: continue @@ -31,54 +39,75 @@ def get_latest_cpl_logs(case): def compare_memory(case, baseline_dir=None): - if baseline_dir is None: - baseline_root = case.get_value("BASELINE_ROOT") - - baseline_name = case.get_value("BASECMP_CASE") + """ + Compare current memory usage to baseline. - baseline_dir = os.path.join(baseline_root, baseline_name) + Parameters + ---------- + case : CIME.case.case.Case + Case object. + baseline_dir : str or None + Path to the baseline directory. - latest_cpl_logs = get_latest_cpl_logs(case) + Returns + ------- + below_tolerance : bool + Whether the current memory usage is below the baseline. + comments : str + Comments about baseline comparison. + """ + if baseline_dir is None: + baseline_dir = case.get_baseline_dir() - diff, baseline, current = [None] * 3 - comment = "" + config = load_coupler_customization(case) - for cpllog in latest_cpl_logs: + # TODO need better handling + try: try: - baseline = read_baseline_mem(baseline_dir) - except FileNotFoundError as e: - comment = f"Could not read baseline memory usage: {e!s}" + current = config.get_mem_usage(case) + except AttributeError: + current = default_get_mem_usage(case) + except RuntimeError as e: + return None, str(e) - logger.debug(comment) + try: + baseline = read_baseline_mem(baseline_dir) + except FileNotFoundError as e: + comment = f"Could not read baseline memory usage: {e!s}" - continue + logger.debug(comment) - if baseline is None: - baseline = 0.0 + return None, comment - memlist = get_mem_usage(cpllog) + if baseline is None: + baseline = 0.0 - if len(memlist) <= 3: - comment = f"Found {len(memlist)} memory usage samples, need atleast 4" + tolerance = case.get_value("TEST_MEMLEAK_TOLERANCE") - logger.debug(comment) + if tolerance is None: + tolerance = 0.1 - continue + try: + below_tolerance, comments = config.compare_memory_baseline( + current, baseline, tolerance + ) + except AttributeError: + below_tolerance, comments = compare_memory_baseline( + current, baseline, tolerance + ) - current = memlist[-1][1] + return below_tolerance, comments - try: - diff = (current - baseline) / baseline - except ZeroDivisionError: - diff = 0.0 - tolerance = case.get_value("TEST_MEMLEAK_TOLERANCE") - - if tolerance is None: - tolerance = 0.1 +def compare_memory_baseline(current, baseline, tolerance): + try: + diff = (current - baseline) / baseline + except ZeroDivisionError: + diff = 0.0 # Should we check if tolerance is above 0 below_tolerance = None + comment = "" if diff is not None: below_tolerance = diff < tolerance @@ -97,40 +126,23 @@ def compare_memory(case, baseline_dir=None): def compare_throughput(case, baseline_dir=None): if baseline_dir is None: - baseline_root = case.get_value("BASELINE_ROOT") - - baseline_name = case.get_value("BASECMP_CASE") - - baseline_dir = os.path.join(baseline_root, baseline_name) + baseline_dir = case.get_baseline_dir() - latest_cpl_logs = get_latest_cpl_logs(case) + config = load_coupler_customization(case) - diff, baseline, current = [None] * 3 - comment = "" - - # Do we need this loop, when are there multiple cpl logs? - for cpllog in latest_cpl_logs: - try: - baseline = read_baseline_tput(baseline_dir) - except FileNotFoundError as e: - comment = f"Could not read baseline throughput file: {e!s}" - - logger.debug(comment) - - continue - - current = get_throughput(cpllog) + try: + current = config.get_throughput(case) + except AttributeError: + current = default_get_throughput(case) - try: - # comparing ypd so bigger is better - diff = (baseline - current) / baseline - except (ValueError, TypeError): - # Should we default the diff to 0.0 as with _compare_current_memory? - comment = f"Could not determine diff with baseline {baseline!r} and current {current!r}" + try: + baseline = read_baseline_tput(baseline_dir) + except FileNotFoundError as e: + comment = f"Could not read baseline throughput file: {e!s}" - logger.debug(comment) + logger.debug(comment) - continue + return None, comment tolerance = case.get_value("TEST_TPUT_TOLERANCE") @@ -142,6 +154,38 @@ def compare_throughput(case, baseline_dir=None): "Bad value for throughput tolerance in test", ) + try: + below_tolerance, comment = config.compare_baseline_throughput( + current, baseline, tolerance + ) + except AttributeError: + below_tolerance, comment = compare_baseline_throughput( + current, baseline, tolerance + ) + + return below_tolerance, comment + + +def load_coupler_customization(case): + comp_root_dir_cpl = case.get_value("COMP_ROOT_DIR_CPL") + + cpl_customize = os.path.join(comp_root_dir_cpl, "cime_config", "customize") + + return Config.load(cpl_customize) + + +def compare_baseline_throughput(current, baseline, tolerance): + try: + # comparing ypd so bigger is better + diff = (baseline - current) / baseline + except (ValueError, TypeError): + # Should we default the diff to 0.0 as with _compare_current_memory? + comment = f"Could not determine diff with baseline {baseline!r} and current {current!r}" + + logger.debug(comment) + + diff = None + below_tolerance = None if diff is not None: @@ -159,59 +203,124 @@ def compare_throughput(case, baseline_dir=None): return below_tolerance, comment +def write_baseline(case, basegen_dir, throughput=True, memory=True): + config = load_coupler_customization(case) + + if throughput: + try: + tput = config.get_throughput(case) + except AttributeError: + tput = str(default_get_throughput(case)) + + write_baseline_tput(basegen_dir, tput) + + if memory: + try: + mem = config.get_mem_usage(case) + except AttributeError: + mem = str(default_get_mem_usage(case)) + + write_baseline_mem(basegen_dir, mem) + + +def default_get_throughput(case): + """ + Parameters + ---------- + cpllog : str + Path to the coupler log. + + Returns + ------- + str + Last recorded highwater memory usage. + """ + cpllog = get_latest_cpl_logs(case) + + try: + tput = get_cpl_throughput(cpllog[0]) + except (FileNotFoundError, IndexError): + tput = None + + return tput + + +def default_get_mem_usage(case, cpllog=None): + """ + Parameters + ---------- + cpllog : str + Path to the coupler log. + + Returns + ------- + str + Last recorded highwater memory usage. + + Raises + ------ + RuntimeError + If not enough sample were found. + """ + if cpllog is None: + cpllog = get_latest_cpl_logs(case) + + try: + memlist = get_cpl_mem_usage(cpllog[0]) + except (FileNotFoundError, IndexError): + memlist = [(None, None)] + else: + if len(memlist) <= 3: + raise RuntimeError( + f"Found {len(memlist)} memory usage samples, need atleast 4" + ) + + return memlist[-1][1] + + def write_baseline_tput(baseline_dir, tput): """ - Writes baseline throughput to file. + Writes throughput to baseline file. - A "-1" indicates that no throughput data was available from the coupler log. + The format is arbitrary, it's the callers responsibilty + to decode the data. Parameters ---------- baseline_dir : str Path to the baseline directory. - cpllog : str - Path to the current coupler log. + tput : str + Model throughput. """ tput_file = os.path.join(baseline_dir, "cpl-tput.log") with open(tput_file, "w") as fd: - fd.write("# Throughput in simulated years per compute day\n") - fd.write("# A -1 indicates no throughput data was available from the test\n") - - if tput is None: - fd.write("-1") - else: - fd.write(str(tput)) + fd.write(tput) def write_baseline_mem(baseline_dir, mem): """ - Writes baseline memory usage highwater to file. + Writes memory usage to baseline file. - A "-1" indicates that no memory usage data was available from the coupler log. + The format is arbitrary, it's the callers responsibilty + to decode the data. Parameters ---------- baseline_dir : str Path to the baseline directory. - cpllog : str - Path to the current coupler log. + mem : str + Model memory usage. """ mem_file = os.path.join(baseline_dir, "cpl-mem.log") with open(mem_file, "w") as fd: - fd.write("# Memory usage highwater\n") - fd.write("# A -1 indicates no memory usage data was available from the test\n") - - try: - fd.write(str(mem[-1][1])) - except IndexError: - fd.write("-1") + fd.write(mem) def read_baseline_tput(baseline_dir): """ - Reads throughput baseline. + Reads the raw lines of the throughput baseline file. Parameters ---------- @@ -220,34 +329,20 @@ def read_baseline_tput(baseline_dir): Returns ------- - float - Value of the throughput. + list + Contents of the throughput baseline file. Raises ------ FileNotFoundError If baseline file does not exist. - IndexError - If no throughput value is found. - ValueError - If throughput value is not a float. """ - try: - tput = read_baseline_value(os.path.join(baseline_dir, "cpl-tput.log")) - except (IndexError, ValueError): - tput = None - - if tput == -1: - tput = None - - return tput + return read_baseline_value(os.path.join(baseline_dir, "cpl-tput.log")) def read_baseline_mem(baseline_dir): """ - Reads memory usage highwater baseline. - - The default behvaior was to return 0 if no usage data was found. + Read the raw lines of the memory baseline file. Parameters ---------- @@ -256,28 +351,20 @@ def read_baseline_mem(baseline_dir): Returns ------- - float - Value of the highwater memory usage. + list + Contents of the memory baseline file. Raises ------ FileNotFoundError If baseline file does not exist. """ - try: - memory = read_baseline_value(os.path.join(baseline_dir, "cpl-mem.log")) - except (IndexError, ValueError): - memory = 0 - - if memory == -1: - memory = 0 - - return memory + return read_baseline_value(os.path.join(baseline_dir, "cpl-mem.log")) def read_baseline_value(baseline_file): """ - Read baseline value from file. + Generic read function, ignores lines prepended by `#`. Parameters ---------- @@ -286,56 +373,78 @@ def read_baseline_value(baseline_file): Returns ------- - float - Baseline value. + list + Lines contained in the baseline file without comments. Raises ------ FileNotFoundError If ``baseline_file`` is not found. - IndexError - If not values are present in ``baseline_file``. - ValueError - If value in ``baseline_file`` is not a float. """ with open(baseline_file) as fd: lines = [x for x in fd.readlines() if not x.startswith("#")] - return float(lines[0]) + return lines -def get_mem_usage(cpllog): +def get_cpl_mem_usage(cpllog): """ - Examine memory usage as recorded in the cpl log file and look for unexpected - increases. + Read memory usage from coupler log. + + Parameters + ---------- + cpllog : str + Path to the coupler log. + + Returns + ------- + list + Memory usage (data, highwater) as recorded by the coupler or empty list. """ memlist = [] + meminfo = re.compile(r".*model date =\s+(\w+).*memory =\s+(\d+\.?\d+).*highwater") + if cpllog is not None and os.path.isfile(cpllog): if ".gz" == cpllog[-3:]: fopen = gzip.open else: fopen = open + with fopen(cpllog, "rb") as f: for line in f: m = meminfo.match(line.decode("utf-8")) + if m: memlist.append((float(m.group(1)), float(m.group(2)))) + # Remove the last mem record, it's sometimes artificially high if len(memlist) > 0: memlist.pop() + return memlist -def get_throughput(cpllog): +def get_cpl_throughput(cpllog): """ - Examine memory usage as recorded in the cpl log file and look for unexpected - increases. + Reads throuhgput from coupler log. + + Parameters + ---------- + cpllog : str + Path to the coupler log. + + Returns + ------- + int or None + Throughput as recorded by the coupler or None """ if cpllog is not None and os.path.isfile(cpllog): with gzip.open(cpllog, "rb") as f: cpltext = f.read().decode("utf-8") + m = re.search(r"# simulated years / cmp-day =\s+(\d+\.\d+)\s", cpltext) + if m: return float(m.group(1)) return None diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index 2c7d5e671a7..d2a88d4b2c8 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -12,13 +12,9 @@ from CIME.case import Case from CIME.test_utils import get_test_status_files from CIME.baselines import ( - get_latest_cpl_logs, - get_mem_usage, - get_throughput, compare_throughput, compare_memory, - write_baseline_tput, - write_baseline_mem, + write_baseline, ) import os, time @@ -51,9 +47,7 @@ def bless_throughput( force or input("Update this diff (y/n)? ").upper() in ["Y", "YES"] ): try: - latest_cpl_logs = get_latest_cpl_logs(case) - current = get_throughput(latest_cpl_logs[0]) - write_baseline_tput(baseline_dir, current) + write_baseline(case, baseline_dir, memory=False) except Exception as e: success = False @@ -88,9 +82,7 @@ def bless_memory( force or input("Update this diff (y/n)? ").upper() in ["Y", "YES"] ): try: - latest_cpl_logs = get_latest_cpl_logs(case) - current = get_mem_usage(latest_cpl_logs[0]) - write_baseline_mem(baseline_dir, current) + write_baseline(case, baseline_dir, throughput=False) except Exception as e: success = False diff --git a/CIME/case/case.py b/CIME/case/case.py index 6de8bb2a217..14bb0069024 100644 --- a/CIME/case/case.py +++ b/CIME/case/case.py @@ -207,6 +207,13 @@ def __init__(self, case_root=None, read_only=True, record=False, non_local=False self.initialize_derived_attributes() + def get_baseline_dir(self): + baseline_root = self.get_value("BASELINE_ROOT") + + baseline_name = self.get_value("BASECMP_self") + + return os.path.join(baseline_root, baseline_name) + def check_if_comp_var(self, vid): for env_file in self._env_entryid_files: new_vid, new_comp, iscompvar = env_file.check_if_comp_var(vid) diff --git a/CIME/tests/test_unit_baselines.py b/CIME/tests/test_unit_baselines.py index 1d4f11d1c1e..644c6b28dcb 100644 --- a/CIME/tests/test_unit_baselines.py +++ b/CIME/tests/test_unit_baselines.py @@ -12,42 +12,45 @@ def create_mock_case(tempdir, get_latest_cpl_logs): caseroot = Path(tempdir, "0", "caseroot") + rundir = caseroot / "run" get_latest_cpl_logs.return_value = (str(rundir / "cpl.log.gz"),) baseline_root = Path(tempdir, "baselines") + baseline_root.mkdir(parents=True, exist_ok=False) + case = mock.MagicMock() return case, caseroot, rundir, baseline_root class TestUnitBaseline(unittest.TestCase): - def test_get_throughput_no_file(self): - throughput = baselines.get_throughput("/tmp/cpl.log") + def test_get_cpl_throughput_no_file(self): + throughput = baselines.get_cpl_throughput("/tmp/cpl.log") assert throughput is None - def test_get_throughput(self): + def test_get_cpl_throughput(self): with tempfile.TemporaryDirectory() as tempdir: cpl_log_path = Path(tempdir, "cpl.log.gz") with gzip.open(cpl_log_path, "w") as fd: fd.write(CPLLOG.encode("utf-8")) - throughput = baselines.get_throughput(str(cpl_log_path)) + throughput = baselines.get_cpl_throughput(str(cpl_log_path)) assert throughput == 719.635 - def test_get_mem_usage_gz(self): + def test_get_cpl_mem_usage_gz(self): with tempfile.TemporaryDirectory() as tempdir: cpl_log_path = Path(tempdir, "cpl.log.gz") with gzip.open(cpl_log_path, "w") as fd: fd.write(CPLLOG.encode("utf-8")) - mem_usage = baselines.get_mem_usage(str(cpl_log_path)) + mem_usage = baselines.get_cpl_mem_usage(str(cpl_log_path)) assert mem_usage == [ (10102.0, 1673.89), @@ -57,13 +60,13 @@ def test_get_mem_usage_gz(self): ] @mock.patch("CIME.baselines.os.path.isfile") - def test_get_mem_usage(self, isfile): + def test_get_cpl_mem_usage(self, isfile): isfile.return_value = True with mock.patch( "builtins.open", mock.mock_open(read_data=CPLLOG.encode("utf-8")) ) as mock_file: - mem_usage = baselines.get_mem_usage("/tmp/cpl.log") + mem_usage = baselines.get_cpl_mem_usage("/tmp/cpl.log") assert mem_usage == [ (10102.0, 1673.89), @@ -76,75 +79,116 @@ def test_read_baseline_mem_empty(self): with mock.patch("builtins.open", mock.mock_open(read_data="")) as mock_file: baseline = baselines.read_baseline_mem("/tmp/cpl-mem.log") - assert baseline is 0 + assert baseline == [] def test_read_baseline_mem_none(self): with mock.patch("builtins.open", mock.mock_open(read_data="-1")) as mock_file: baseline = baselines.read_baseline_mem("/tmp/cpl-mem.log") - assert baseline is 0 + assert baseline == ["-1"] def test_read_baseline_mem(self): with mock.patch("builtins.open", mock.mock_open(read_data="200")) as mock_file: baseline = baselines.read_baseline_mem("/tmp/cpl-mem.log") - assert baseline == 200 + assert baseline == ["200"] def test_read_baseline_tput_empty(self): with mock.patch("builtins.open", mock.mock_open(read_data="")) as mock_file: baseline = baselines.read_baseline_tput("/tmp/cpl-tput.log") - assert baseline is None + assert baseline == [] def test_read_baseline_tput_none(self): with mock.patch("builtins.open", mock.mock_open(read_data="-1")) as mock_file: baseline = baselines.read_baseline_tput("/tmp/cpl-tput.log") - assert baseline is None + assert baseline == ["-1"] def test_read_baseline_tput(self): with mock.patch("builtins.open", mock.mock_open(read_data="200")) as mock_file: baseline = baselines.read_baseline_tput("/tmp/cpl-tput.log") - assert baseline == 200 + assert baseline == ["200"] def test_write_baseline_mem_no_value(self): with mock.patch("builtins.open", mock.mock_open()) as mock_file: - baselines.write_baseline_mem("/tmp", []) + baselines.write_baseline_mem("/tmp", "-1") mock_file.assert_called_with("/tmp/cpl-mem.log", "w") mock_file.return_value.write.assert_called_with("-1") def test_write_baseline_mem(self): with mock.patch("builtins.open", mock.mock_open()) as mock_file: - baselines.write_baseline_mem("/tmp", [(1, 200)]) + baselines.write_baseline_mem("/tmp", "200") mock_file.assert_called_with("/tmp/cpl-mem.log", "w") mock_file.return_value.write.assert_called_with("200") def test_write_baseline_tput_no_value(self): with mock.patch("builtins.open", mock.mock_open()) as mock_file: - baselines.write_baseline_tput("/tmp", None) + baselines.write_baseline_tput("/tmp", "-1") mock_file.assert_called_with("/tmp/cpl-tput.log", "w") mock_file.return_value.write.assert_called_with("-1") def test_write_baseline_tput(self): with mock.patch("builtins.open", mock.mock_open()) as mock_file: - baselines.write_baseline_tput("/tmp", 200) + baselines.write_baseline_tput("/tmp", "200") mock_file.assert_called_with("/tmp/cpl-tput.log", "w") mock_file.return_value.write.assert_called_with("200") - @mock.patch("CIME.baselines.get_throughput") + @mock.patch("CIME.baselines.get_cpl_throughput") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_default_get_throughput(self, get_latest_cpl_logs, get_cpl_throughput): + get_cpl_throughput.side_effect = FileNotFoundError() + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + tput = baselines.default_get_throughput(case) + + assert tput == None + + @mock.patch("CIME.baselines.get_cpl_mem_usage") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_default_get_mem_usage(self, get_latest_cpl_logs, get_cpl_mem_usage): + get_cpl_mem_usage.side_effect = FileNotFoundError() + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + mem = baselines.default_get_mem_usage(case) + + assert mem == None + + @mock.patch("CIME.baselines.get_cpl_mem_usage") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_write_baseline(self, get_latest_cpl_logs, get_cpl_mem_usage): + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + get_cpl_mem_usage.return_value = [ + (1, 1000.0), + (2, 1001.0), + (3, 1002.0), + (4, 1003.0), + ] + + baselines.write_baseline( + case, baseline_root, get_latest_cpl_logs.return_value + ) + + @mock.patch("CIME.baselines.get_cpl_throughput") @mock.patch("CIME.baselines.read_baseline_tput") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_throughput_no_baseline_file( - self, get_latest_cpl_logs, read_baseline_tput, get_throughput + self, get_latest_cpl_logs, read_baseline_tput, get_cpl_throughput ): read_baseline_tput.side_effect = FileNotFoundError - get_throughput.return_value = 504 + get_cpl_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -152,6 +196,7 @@ def test_compare_throughput_no_baseline_file( case.get_value.side_effect = ( str(baseline_root), "master/ERIO.ne30_g16_rx1.A.docker_gnu", + "/tmp/components/cpl", 0.05, ) @@ -160,22 +205,25 @@ def test_compare_throughput_no_baseline_file( assert below_tolerance is None assert comment == "Could not read baseline throughput file: " - @mock.patch("CIME.baselines.get_throughput") + @mock.patch("CIME.baselines.default_get_throughput") @mock.patch("CIME.baselines.read_baseline_tput") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_throughput_no_baseline( - self, get_latest_cpl_logs, read_baseline_tput, get_throughput + self, get_latest_cpl_logs, read_baseline_tput, default_get_throughput ): read_baseline_tput.return_value = None - get_throughput.return_value = 504 + default_get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + case.get_baseline_dir.return_value = str( + baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" + ) + case.get_value.side_effect = ( - str(baseline_root), - "master/ERIO.ne30_g16_rx1.A.docker_gnu", + "/tmp/components/cpl", 0.05, ) @@ -184,22 +232,25 @@ def test_compare_throughput_no_baseline( assert below_tolerance is None assert comment == "Could not determine diff with baseline None and current 504" - @mock.patch("CIME.baselines.get_throughput") + @mock.patch("CIME.baselines.default_get_throughput") @mock.patch("CIME.baselines.read_baseline_tput") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_throughput_no_tolerance( - self, get_latest_cpl_logs, read_baseline_tput, get_throughput + self, get_latest_cpl_logs, read_baseline_tput, default_get_throughput ): read_baseline_tput.return_value = 500 - get_throughput.return_value = 504 + default_get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + case.get_baseline_dir.return_value = str( + baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" + ) + case.get_value.side_effect = ( - str(baseline_root), - "master/ERIO.ne30_g16_rx1.A.docker_gnu", + "/tmp/components/cpl", None, ) @@ -211,22 +262,25 @@ def test_compare_throughput_no_tolerance( == "TPUTCOMP: Computation time changed by -0.80% relative to baseline" ) - @mock.patch("CIME.baselines.get_throughput") + @mock.patch("CIME.baselines.default_get_throughput") @mock.patch("CIME.baselines.read_baseline_tput") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_throughput_above_threshold( - self, get_latest_cpl_logs, read_baseline_tput, get_throughput + self, get_latest_cpl_logs, read_baseline_tput, default_get_throughput ): read_baseline_tput.return_value = 1000 - get_throughput.return_value = 504 + default_get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + case.get_baseline_dir.return_value = str( + baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" + ) + case.get_value.side_effect = ( - str(baseline_root), - "master/ERIO.ne30_g16_rx1.A.docker_gnu", + "/tmp/components/cpl", 0.05, ) @@ -237,22 +291,25 @@ def test_compare_throughput_above_threshold( comment == "Error: TPUTCOMP: Computation time increase > 5% from baseline" ) - @mock.patch("CIME.baselines.get_throughput") + @mock.patch("CIME.baselines.default_get_throughput") @mock.patch("CIME.baselines.read_baseline_tput") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_throughput( - self, get_latest_cpl_logs, read_baseline_tput, get_throughput + self, get_latest_cpl_logs, read_baseline_tput, default_get_throughput ): read_baseline_tput.return_value = 500 - get_throughput.return_value = 504 + default_get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + case.get_baseline_dir.return_value = str( + baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" + ) + case.get_value.side_effect = ( - str(baseline_root), - "master/ERIO.ne30_g16_rx1.A.docker_gnu", + "/tmp/components/cpl", 0.05, ) @@ -264,15 +321,15 @@ def test_compare_throughput( == "TPUTCOMP: Computation time changed by -0.80% relative to baseline" ) - @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.read_baseline_mem") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_memory_no_baseline( - self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage + self, get_latest_cpl_logs, read_baseline_mem, get_cpl_mem_usage ): read_baseline_mem.return_value = None - get_mem_usage.return_value = [ + get_cpl_mem_usage.return_value = [ (1, 1000.0), (2, 1001.0), (3, 1002.0), @@ -282,9 +339,12 @@ def test_compare_memory_no_baseline( with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + case.get_baseline_dir.return_value = str( + baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" + ) + case.get_value.side_effect = ( - str(baseline_root), - "master/ERIO.ne30_g16_rx1.A.docker_gnu", + "/tmp/components/cpl", 0.05, ) @@ -296,15 +356,15 @@ def test_compare_memory_no_baseline( == "MEMCOMP: Memory usage highwater has changed by 0.00% relative to baseline" ) - @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.read_baseline_mem") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_memory_not_enough_samples( - self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage + self, get_latest_cpl_logs, read_baseline_mem, get_cpl_mem_usage ): read_baseline_mem.return_value = 1000.0 - get_mem_usage.return_value = [ + get_cpl_mem_usage.return_value = [ (1, 1000.0), (2, 1001.0), ] @@ -315,6 +375,7 @@ def test_compare_memory_not_enough_samples( case.get_value.side_effect = ( str(baseline_root), "master/ERIO.ne30_g16_rx1.A.docker_gnu", + "/tmp/components/cpl", 0.05, ) @@ -323,15 +384,15 @@ def test_compare_memory_not_enough_samples( assert below_tolerance is None assert comment == "Found 2 memory usage samples, need atleast 4" - @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.read_baseline_mem") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_memory_no_baseline_file( - self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage + self, get_latest_cpl_logs, read_baseline_mem, get_cpl_mem_usage ): read_baseline_mem.side_effect = FileNotFoundError - get_mem_usage.return_value = [ + get_cpl_mem_usage.return_value = [ (1, 1000.0), (2, 1001.0), (3, 1002.0), @@ -344,6 +405,7 @@ def test_compare_memory_no_baseline_file( case.get_value.side_effect = ( str(baseline_root), "master/ERIO.ne30_g16_rx1.A.docker_gnu", + "/tmp/components/cpl", 0.05, ) @@ -352,15 +414,15 @@ def test_compare_memory_no_baseline_file( assert below_tolerance is None assert comment == "Could not read baseline memory usage: " - @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.read_baseline_mem") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_memory_no_tolerance( - self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage + self, get_latest_cpl_logs, read_baseline_mem, get_cpl_mem_usage ): read_baseline_mem.return_value = 1000.0 - get_mem_usage.return_value = [ + get_cpl_mem_usage.return_value = [ (1, 1000.0), (2, 1001.0), (3, 1002.0), @@ -370,9 +432,12 @@ def test_compare_memory_no_tolerance( with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + case.get_baseline_dir.return_value = str( + baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" + ) + case.get_value.side_effect = ( - str(baseline_root), - "master/ERIO.ne30_g16_rx1.A.docker_gnu", + "/tmp/components/cpl", None, ) @@ -384,15 +449,15 @@ def test_compare_memory_no_tolerance( == "MEMCOMP: Memory usage highwater has changed by 0.30% relative to baseline" ) - @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.read_baseline_mem") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_memory_above_threshold( - self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage + self, get_latest_cpl_logs, read_baseline_mem, get_cpl_mem_usage ): read_baseline_mem.return_value = 1000.0 - get_mem_usage.return_value = [ + get_cpl_mem_usage.return_value = [ (1, 2000.0), (2, 2001.0), (3, 2002.0), @@ -402,9 +467,12 @@ def test_compare_memory_above_threshold( with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + case.get_baseline_dir.return_value = str( + baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" + ) + case.get_value.side_effect = ( - str(baseline_root), - "master/ERIO.ne30_g16_rx1.A.docker_gnu", + "/tmp/components/cpl", 0.05, ) @@ -416,15 +484,15 @@ def test_compare_memory_above_threshold( == "Error: Memory usage increase >5% from baseline's 1000.000000 to 2003.000000" ) - @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.read_baseline_mem") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_memory( - self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage + self, get_latest_cpl_logs, read_baseline_mem, get_cpl_mem_usage ): read_baseline_mem.return_value = 1000.0 - get_mem_usage.return_value = [ + get_cpl_mem_usage.return_value = [ (1, 1000.0), (2, 1001.0), (3, 1002.0), @@ -434,9 +502,12 @@ def test_compare_memory( with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + case.get_baseline_dir.return_value = str( + baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" + ) + case.get_value.side_effect = ( - str(baseline_root), - "master/ERIO.ne30_g16_rx1.A.docker_gnu", + "/tmp/components/cpl", 0.05, ) diff --git a/CIME/tests/test_unit_system_tests.py b/CIME/tests/test_unit_system_tests.py index 066d4fbd193..9ea088411bd 100644 --- a/CIME/tests/test_unit_system_tests.py +++ b/CIME/tests/test_unit_system_tests.py @@ -259,6 +259,11 @@ def test_generate_baseline(self): "master/ERIO.ne30_g16_rx1.A.docker_gnu", str(run_dir), "mct", + "/tmp/components/cpl", + str(run_dir), + "mct", + str(run_dir), + "mct", ] if Config.instance().create_bless_log: @@ -280,14 +285,14 @@ def test_generate_baseline(self): with open(baseline_dir / "cpl-tput.log") as fd: lines = fd.readlines() - assert len(lines) == 3 - assert lines[-1] == "719.635" + assert len(lines) == 1 + assert lines[0] == "719.635" with open(baseline_dir / "cpl-mem.log") as fd: lines = fd.readlines() - assert len(lines) == 3 - assert lines[-1] == "1673.89" + assert len(lines) == 1 + assert lines[0] == "1673.89" def test_kwargs(self): case = mock.MagicMock() From ec661ca64eaa95f6ec90a2c018f68b9b9eb1a07c Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 6 Oct 2023 14:47:06 -0700 Subject: [PATCH 13/57] Fixes failing tests --- CIME/SystemTests/system_tests_common.py | 66 +++++++++++++------------ CIME/baselines.py | 32 +++++++----- CIME/tests/test_unit_baselines.py | 2 +- 3 files changed, 55 insertions(+), 45 deletions(-) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index 38263cd7cea..45e7ae932d3 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -634,44 +634,46 @@ def _check_for_memleak(self): with self._test_status: latestcpllogs = get_latest_cpl_logs(self._case) for cpllog in latestcpllogs: - memlist = default_get_mem_usage(self._case, cpllog) - - if len(memlist) < 3: + try: + memlist = default_get_mem_usage(self._case, cpllog) + except RuntimeError: self._test_status.set_status( MEMLEAK_PHASE, TEST_PASS_STATUS, comments="insuffiencient data for memleak test", ) + + continue + + finaldate = int(memlist[-1][0]) + originaldate = int( + memlist[1][0] + ) # skip first day mem record, it can be too low while initializing + finalmem = float(memlist[-1][1]) + originalmem = float(memlist[1][1]) + memdiff = -1 + if originalmem > 0: + memdiff = (finalmem - originalmem) / originalmem + tolerance = self._case.get_value("TEST_MEMLEAK_TOLERANCE") + if tolerance is None: + tolerance = 0.1 + expect(tolerance > 0.0, "Bad value for memleak tolerance in test") + if memdiff < 0: + self._test_status.set_status( + MEMLEAK_PHASE, + TEST_PASS_STATUS, + comments="data for memleak test is insuffiencient", + ) + elif memdiff < tolerance: + self._test_status.set_status(MEMLEAK_PHASE, TEST_PASS_STATUS) else: - finaldate = int(memlist[-1][0]) - originaldate = int( - memlist[1][0] - ) # skip first day mem record, it can be too low while initializing - finalmem = float(memlist[-1][1]) - originalmem = float(memlist[1][1]) - memdiff = -1 - if originalmem > 0: - memdiff = (finalmem - originalmem) / originalmem - tolerance = self._case.get_value("TEST_MEMLEAK_TOLERANCE") - if tolerance is None: - tolerance = 0.1 - expect(tolerance > 0.0, "Bad value for memleak tolerance in test") - if memdiff < 0: - self._test_status.set_status( - MEMLEAK_PHASE, - TEST_PASS_STATUS, - comments="data for memleak test is insuffiencient", - ) - elif memdiff < tolerance: - self._test_status.set_status(MEMLEAK_PHASE, TEST_PASS_STATUS) - else: - comment = "memleak detected, memory went from {:f} to {:f} in {:d} days".format( - originalmem, finalmem, finaldate - originaldate - ) - append_testlog(comment, self._orig_caseroot) - self._test_status.set_status( - MEMLEAK_PHASE, TEST_FAIL_STATUS, comments=comment - ) + comment = "memleak detected, memory went from {:f} to {:f} in {:d} days".format( + originalmem, finalmem, finaldate - originaldate + ) + append_testlog(comment, self._orig_caseroot) + self._test_status.set_status( + MEMLEAK_PHASE, TEST_FAIL_STATUS, comments=comment + ) def compare_env_run(self, expected=None): """ diff --git a/CIME/baselines.py b/CIME/baselines.py index 0f7d1c16b19..e74483d5a6d 100644 --- a/CIME/baselines.py +++ b/CIME/baselines.py @@ -66,7 +66,7 @@ def compare_memory(case, baseline_dir=None): try: current = config.get_mem_usage(case) except AttributeError: - current = default_get_mem_usage(case) + current = default_get_mem_usage(case)[-1][1] except RuntimeError as e: return None, str(e) @@ -208,19 +208,25 @@ def write_baseline(case, basegen_dir, throughput=True, memory=True): if throughput: try: - tput = config.get_throughput(case) - except AttributeError: - tput = str(default_get_throughput(case)) - - write_baseline_tput(basegen_dir, tput) + try: + tput = config.get_throughput(case) + except AttributeError: + tput = str(default_get_throughput(case)) + except RuntimeError: + pass + else: + write_baseline_tput(basegen_dir, tput) if memory: try: - mem = config.get_mem_usage(case) - except AttributeError: - mem = str(default_get_mem_usage(case)) - - write_baseline_mem(basegen_dir, mem) + try: + mem = config.get_mem_usage(case) + except AttributeError: + mem = str(default_get_mem_usage(case)[-1][1]) + except RuntimeError: + pass + else: + write_baseline_mem(basegen_dir, mem) def default_get_throughput(case): @@ -264,6 +270,8 @@ def default_get_mem_usage(case, cpllog=None): """ if cpllog is None: cpllog = get_latest_cpl_logs(case) + else: + cpllog = [cpllog,] try: memlist = get_cpl_mem_usage(cpllog[0]) @@ -275,7 +283,7 @@ def default_get_mem_usage(case, cpllog=None): f"Found {len(memlist)} memory usage samples, need atleast 4" ) - return memlist[-1][1] + return memlist def write_baseline_tput(baseline_dir, tput): diff --git a/CIME/tests/test_unit_baselines.py b/CIME/tests/test_unit_baselines.py index 644c6b28dcb..91e7b72d8a3 100644 --- a/CIME/tests/test_unit_baselines.py +++ b/CIME/tests/test_unit_baselines.py @@ -161,7 +161,7 @@ def test_default_get_mem_usage(self, get_latest_cpl_logs, get_cpl_mem_usage): mem = baselines.default_get_mem_usage(case) - assert mem == None + assert mem == [(None, None)] @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.get_latest_cpl_logs") From 4bf66054ccdbaf26bfc1323e239788f175c6da78 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 6 Oct 2023 15:16:28 -0700 Subject: [PATCH 14/57] Fixes black formatting --- CIME/baselines.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CIME/baselines.py b/CIME/baselines.py index e74483d5a6d..e084fe54b3a 100644 --- a/CIME/baselines.py +++ b/CIME/baselines.py @@ -271,7 +271,9 @@ def default_get_mem_usage(case, cpllog=None): if cpllog is None: cpllog = get_latest_cpl_logs(case) else: - cpllog = [cpllog,] + cpllog = [ + cpllog, + ] try: memlist = get_cpl_mem_usage(cpllog[0]) From e61d6412640f92f41029948ef7149700f55f158e Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Mon, 9 Oct 2023 14:27:59 -0700 Subject: [PATCH 15/57] Updates permissions --- .github/workflows/testing.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 2f0b35042f5..7010406cae4 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -25,6 +25,7 @@ concurrency: permissions: contents: read # to fetch code (actions/checkout) + packages: read jobs: pre-commit: From b353d1d30402565a81d1e24f2bf0a213dcf13c92 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Mon, 9 Oct 2023 14:30:20 -0700 Subject: [PATCH 16/57] Removes --pull --- .github/workflows/testing.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 7010406cae4..53cead36411 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -59,7 +59,6 @@ jobs: credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - options: '--pull=always' strategy: matrix: python-version: ['3.8', '3.9', '3.10'] @@ -99,7 +98,6 @@ jobs: credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - options: '--pull=always' strategy: matrix: model: ["e3sm", "cesm"] From ecab15438bc673e548ebcae7ae6bb245efa407b0 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Wed, 11 Oct 2023 09:32:41 -0700 Subject: [PATCH 17/57] Fixes BASECMP_CASE name --- CIME/case/case.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CIME/case/case.py b/CIME/case/case.py index 14bb0069024..2bf14540205 100644 --- a/CIME/case/case.py +++ b/CIME/case/case.py @@ -210,7 +210,7 @@ def __init__(self, case_root=None, read_only=True, record=False, non_local=False def get_baseline_dir(self): baseline_root = self.get_value("BASELINE_ROOT") - baseline_name = self.get_value("BASECMP_self") + baseline_name = self.get_value("BASECMP_CASE") return os.path.join(baseline_root, baseline_name) From a786b10b35e429f8071e0603d2116b4f58eed03a Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Wed, 11 Oct 2023 23:35:38 -0700 Subject: [PATCH 18/57] Cleans up baselines --- CIME/SystemTests/system_tests_common.py | 8 +- CIME/baselines.py | 556 +++++++++++++----------- CIME/tests/test_unit_baselines.py | 189 ++++---- CIME/tests/test_unit_system_tests.py | 181 +++++++- 4 files changed, 584 insertions(+), 350 deletions(-) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index 45e7ae932d3..43c08d45c7f 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -28,7 +28,7 @@ from CIME.locked_files import LOCKED_DIR, lock_file, is_locked from CIME.baselines import ( get_latest_cpl_logs, - default_get_mem_usage, + get_default_mem_usage, compare_memory, compare_throughput, write_baseline, @@ -635,12 +635,12 @@ def _check_for_memleak(self): latestcpllogs = get_latest_cpl_logs(self._case) for cpllog in latestcpllogs: try: - memlist = default_get_mem_usage(self._case, cpllog) + memlist = get_default_mem_usage(self._case, cpllog) except RuntimeError: self._test_status.set_status( MEMLEAK_PHASE, TEST_PASS_STATUS, - comments="insuffiencient data for memleak test", + comments="insufficient data for memleak test", ) continue @@ -662,7 +662,7 @@ def _check_for_memleak(self): self._test_status.set_status( MEMLEAK_PHASE, TEST_PASS_STATUS, - comments="data for memleak test is insuffiencient", + comments="data for memleak test is insufficient", ) elif memdiff < tolerance: self._test_status.set_status(MEMLEAK_PHASE, TEST_PASS_STATUS) diff --git a/CIME/baselines.py b/CIME/baselines.py index e084fe54b3a..47f74a698fd 100644 --- a/CIME/baselines.py +++ b/CIME/baselines.py @@ -9,69 +9,89 @@ logger = logging.getLogger(__name__) -def get_latest_cpl_logs(case): +def compare_throughput(case, baseline_dir=None): """ - find and return the latest cpl log file in the run directory + Compares model throughput. + + Parameters + ---------- + case : CIME.case.case.Case + Current case object. + baseline_dir : str + Overrides the baseline directory. + + Returns + ------- + below_tolerance : bool + Whether the comparison was below the tolerance. + comment : str + Provides explanation from comparison. """ - coupler_log_path = case.get_value("RUNDIR") + if baseline_dir is None: + baseline_dir = case.get_baseline_dir() - cpllog_name = "drv" if case.get_value("COMP_INTERFACE") == "nuopc" else "cpl" + config = load_coupler_customization(case) - cpllogs = glob.glob(os.path.join(coupler_log_path, "{}*.log.*".format(cpllog_name))) + baseline_file = os.path.join(baseline_dir, "cpl-tput.log") - lastcpllogs = [] + try: + baseline = read_baseline_file(baseline_file) + except FileNotFoundError as e: + comment = f"Could not read baseline throughput file: {e!s}" - if cpllogs: - lastcpllogs.append(max(cpllogs, key=os.path.getctime)) + logger.debug(comment) - basename = os.path.basename(lastcpllogs[0]) + return None, comment - suffix = basename.split(".", 1)[1] + tolerance = case.get_value("TEST_TPUT_TOLERANCE") - for log in cpllogs: - if log in lastcpllogs: - continue + if tolerance is None: + tolerance = 0.1 - if log.endswith(suffix): - lastcpllogs.append(log) + expect( + tolerance > 0.0, + "Bad value for throughput tolerance in test", + ) - return lastcpllogs + try: + below_tolerance, comment = config.compare_baseline_throughput( + case, baseline, tolerance + ) + except AttributeError: + below_tolerance, comment = compare_baseline_throughput( + case, baseline, tolerance + ) + + return below_tolerance, comment def compare_memory(case, baseline_dir=None): """ - Compare current memory usage to baseline. + Compares model highwater memory usage. Parameters ---------- case : CIME.case.case.Case - Case object. - baseline_dir : str or None - Path to the baseline directory. + Current case object. + baseline_dir : str + Overrides the baseline directory. Returns ------- below_tolerance : bool - Whether the current memory usage is below the baseline. - comments : str - Comments about baseline comparison. + Whether the comparison was below the tolerance. + comment : str + Provides explanation from comparison. """ if baseline_dir is None: baseline_dir = case.get_baseline_dir() config = load_coupler_customization(case) - # TODO need better handling - try: - try: - current = config.get_mem_usage(case) - except AttributeError: - current = default_get_mem_usage(case)[-1][1] - except RuntimeError as e: - return None, str(e) + baseline_file = os.path.join(baseline_dir, "cpl-mem.log") try: - baseline = read_baseline_mem(baseline_dir) + baseline = read_baseline_file(baseline_file) except FileNotFoundError as e: comment = f"Could not read baseline memory usage: {e!s}" @@ -79,94 +99,67 @@ def compare_memory(case, baseline_dir=None): return None, comment - if baseline is None: - baseline = 0.0 - tolerance = case.get_value("TEST_MEMLEAK_TOLERANCE") if tolerance is None: tolerance = 0.1 try: - below_tolerance, comments = config.compare_memory_baseline( - current, baseline, tolerance + below_tolerance, comments = config.compare_baseline_memory( + case, baseline, tolerance ) except AttributeError: - below_tolerance, comments = compare_memory_baseline( - current, baseline, tolerance - ) + below_tolerance, comments = compare_baseline_memory(case, baseline, tolerance) return below_tolerance, comments -def compare_memory_baseline(current, baseline, tolerance): - try: - diff = (current - baseline) / baseline - except ZeroDivisionError: - diff = 0.0 - - # Should we check if tolerance is above 0 - below_tolerance = None - comment = "" - - if diff is not None: - below_tolerance = diff < tolerance - - if below_tolerance: - comment = "MEMCOMP: Memory usage highwater has changed by {:.2f}% relative to baseline".format( - diff * 100 - ) - else: - comment = "Error: Memory usage increase >{:d}% from baseline's {:f} to {:f}".format( - int(tolerance * 100), baseline, current - ) - - return below_tolerance, comment - - -def compare_throughput(case, baseline_dir=None): - if baseline_dir is None: - baseline_dir = case.get_baseline_dir() +def write_baseline(case, basegen_dir, throughput=True, memory=True): + """ + Writes the baseline performance files. + Parameters + ---------- + case : CIME.case.case.Case + Current case object. + basegen_dir : str + Path to baseline directory. + throughput : bool + If true, write throughput baseline. + memory : bool + If true, write memory baseline. + """ config = load_coupler_customization(case) - try: - current = config.get_throughput(case) - except AttributeError: - current = default_get_throughput(case) - - try: - baseline = read_baseline_tput(baseline_dir) - except FileNotFoundError as e: - comment = f"Could not read baseline throughput file: {e!s}" + if throughput: + tput = get_throughput(case, config) - logger.debug(comment) + baseline_file = os.path.join(basegen_dir, "cpl-tput.log") - return None, comment + write_baseline_file(baseline_file, tput) - tolerance = case.get_value("TEST_TPUT_TOLERANCE") + if memory: + mem = get_mem_usage(case, config) - if tolerance is None: - tolerance = 0.1 + baseline_file = os.path.join(basegen_dir, "cpl-mem.log") - expect( - tolerance > 0.0, - "Bad value for throughput tolerance in test", - ) + write_baseline_file(baseline_file, mem) - try: - below_tolerance, comment = config.compare_baseline_throughput( - current, baseline, tolerance - ) - except AttributeError: - below_tolerance, comment = compare_baseline_throughput( - current, baseline, tolerance - ) - return below_tolerance, comment +def load_coupler_customization(case): + """ + Loads customizations from the coupler `cime_config` directory. + Parameters + ---------- + case : CIME.case.case.Case + Current case object. -def load_coupler_customization(case): + Returns + ------- + CIME.config.Config + Runtime configuration. + """ comp_root_dir_cpl = case.get_value("COMP_ROOT_DIR_CPL") cpl_customize = os.path.join(comp_root_dir_cpl, "cime_config", "customize") @@ -174,94 +167,92 @@ def load_coupler_customization(case): return Config.load(cpl_customize) -def compare_baseline_throughput(current, baseline, tolerance): - try: - # comparing ypd so bigger is better - diff = (baseline - current) / baseline - except (ValueError, TypeError): - # Should we default the diff to 0.0 as with _compare_current_memory? - comment = f"Could not determine diff with baseline {baseline!r} and current {current!r}" +def get_throughput(case, config): + """ + Gets the model throughput. - logger.debug(comment) + First attempts to use a coupler define method to retrieve the + models throughput. If this is not defined then the default + method of parsing the coupler log is used. - diff = None + Parameters + ---------- + case : CIME.case.case.Case + Current case object. - below_tolerance = None + Returns + ------- + str or None + Model throughput. + """ + try: + tput = config.get_throughput(case) + except AttributeError: + tput = str(get_default_throughput(case)) - if diff is not None: - below_tolerance = diff < tolerance + return tput - if below_tolerance: - comment = "TPUTCOMP: Computation time changed by {:.2f}% relative to baseline".format( - diff * 100 - ) - else: - comment = "Error: TPUTCOMP: Computation time increase > {:d}% from baseline".format( - int(tolerance * 100) - ) - return below_tolerance, comment +def get_mem_usage(case, config): + """ + Gets the model memory usage. + First attempts to use a coupler defined method to retrieve the + models memory usage. If this is not defined then the default + method of parsing the coupler log is used. -def write_baseline(case, basegen_dir, throughput=True, memory=True): - config = load_coupler_customization(case) + Parameters + ---------- + case : CIME.case.case.Case + Current case object. - if throughput: - try: - try: - tput = config.get_throughput(case) - except AttributeError: - tput = str(default_get_throughput(case)) - except RuntimeError: - pass - else: - write_baseline_tput(basegen_dir, tput) + Returns + ------- + str or None + Model memory usage. + """ + try: + mem = config.get_mem_usage(case) + except AttributeError: + mem = get_default_mem_usage(case) - if memory: - try: - try: - mem = config.get_mem_usage(case) - except AttributeError: - mem = str(default_get_mem_usage(case)[-1][1]) - except RuntimeError: - pass - else: - write_baseline_mem(basegen_dir, mem) + mem = str(mem[-1][1]) + + return mem -def default_get_throughput(case): +def write_baseline_file(baseline_file, value): """ + Writes value to `baseline_file`. + Parameters ---------- - cpllog : str - Path to the coupler log. - - Returns - ------- - str - Last recorded highwater memory usage. + baseline_file : str + Path to the baseline file. + value : str + Value to write. """ - cpllog = get_latest_cpl_logs(case) + with open(baseline_file, "w") as fd: + fd.write(value) - try: - tput = get_cpl_throughput(cpllog[0]) - except (FileNotFoundError, IndexError): - tput = None - return tput +def get_default_mem_usage(case, cpllog=None): + """ + Default function to retrieve memory usage from the coupler log. + If the usage is not available from the log then `None` is returned. -def default_get_mem_usage(case, cpllog=None): - """ Parameters ---------- + case : CIME.case.case.Case + Current case object. cpllog : str - Path to the coupler log. + Overrides the default coupler log. Returns ------- - str - Last recorded highwater memory usage. + str or None + Model memory usage or `None`. Raises ------ @@ -278,7 +269,7 @@ def default_get_mem_usage(case, cpllog=None): try: memlist = get_cpl_mem_usage(cpllog[0]) except (FileNotFoundError, IndexError): - memlist = [(None, None)] + memlist = None else: if len(memlist) <= 3: raise RuntimeError( @@ -288,113 +279,59 @@ def default_get_mem_usage(case, cpllog=None): return memlist -def write_baseline_tput(baseline_dir, tput): +def get_default_throughput(case): """ - Writes throughput to baseline file. + Default function to retrieve throughput from the coupler log. - The format is arbitrary, it's the callers responsibilty - to decode the data. + If the throughput is not available from the log then `None` is returned. Parameters ---------- - baseline_dir : str - Path to the baseline directory. - tput : str - Model throughput. - """ - tput_file = os.path.join(baseline_dir, "cpl-tput.log") - - with open(tput_file, "w") as fd: - fd.write(tput) - - -def write_baseline_mem(baseline_dir, mem): - """ - Writes memory usage to baseline file. - - The format is arbitrary, it's the callers responsibilty - to decode the data. + case : CIME.case.case.Case + Current case object. - Parameters - ---------- - baseline_dir : str - Path to the baseline directory. - mem : str - Model memory usage. + Returns + ------- + str or None + Model throughput or `None`. """ - mem_file = os.path.join(baseline_dir, "cpl-mem.log") - - with open(mem_file, "w") as fd: - fd.write(mem) - + cpllog = get_latest_cpl_logs(case) -def read_baseline_tput(baseline_dir): - """ - Reads the raw lines of the throughput baseline file. + try: + tput = get_cpl_throughput(cpllog[0]) + except (FileNotFoundError, IndexError): + tput = None - Parameters - ---------- - baseline_dir : str - Path to the baseline directory. + return tput - Returns - ------- - list - Contents of the throughput baseline file. - Raises - ------ - FileNotFoundError - If baseline file does not exist. +def get_latest_cpl_logs(case): """ - return read_baseline_value(os.path.join(baseline_dir, "cpl-tput.log")) - - -def read_baseline_mem(baseline_dir): + find and return the latest cpl log file in the run directory """ - Read the raw lines of the memory baseline file. + coupler_log_path = case.get_value("RUNDIR") - Parameters - ---------- - baseline_dir : str - Path to the baseline directory. + cpllog_name = "drv" if case.get_value("COMP_INTERFACE") == "nuopc" else "cpl" - Returns - ------- - list - Contents of the memory baseline file. + cpllogs = glob.glob(os.path.join(coupler_log_path, "{}*.log.*".format(cpllog_name))) - Raises - ------ - FileNotFoundError - If baseline file does not exist. - """ - return read_baseline_value(os.path.join(baseline_dir, "cpl-mem.log")) + lastcpllogs = [] + if cpllogs: + lastcpllogs.append(max(cpllogs, key=os.path.getctime)) -def read_baseline_value(baseline_file): - """ - Generic read function, ignores lines prepended by `#`. + basename = os.path.basename(lastcpllogs[0]) - Parameters - ---------- - baseline_file : str - Path to baseline file. + suffix = basename.split(".", 1)[1] - Returns - ------- - list - Lines contained in the baseline file without comments. + for log in cpllogs: + if log in lastcpllogs: + continue - Raises - ------ - FileNotFoundError - If ``baseline_file`` is not found. - """ - with open(baseline_file) as fd: - lines = [x for x in fd.readlines() if not x.startswith("#")] + if log.endswith(suffix): + lastcpllogs.append(log) - return lines + return lastcpllogs def get_cpl_mem_usage(cpllog): @@ -458,3 +395,134 @@ def get_cpl_throughput(cpllog): if m: return float(m.group(1)) return None + + +def read_baseline_file(baseline_file): + """ + Reads value from `baseline_file`. + + Parameters + ---------- + baseline_file : str + Path to the baseline file. + + Returns + ------- + str + Value stored in baseline file without comments. + """ + with open(baseline_file) as fd: + lines = [x for x in fd.readlines() if not x.startswith("#")] + + return lines + + +def compare_baseline_throughput(case, baseline, tolerance): + """ + Default throughput baseline comparison. + + Compares the throughput from the coupler to the baseline value. + + Parameters + ---------- + case : CIME.case.case.Case + Current case object. + baseline : list + Lines contained in the baseline file. + tolerance : float + Allowed tolerance for comparison. + + Returns + ------- + below_tolerance : bool + Whether the comparison was below the tolerance. + comment : str + provides explanation from comparison. + """ + current = get_default_throughput(case) + + try: + # default baseline is stored as single float + baseline = float(baseline[0]) + except IndexError: + comment = "Could not compare throughput to baseline, as basline had no value." + + return None, comment + + # comparing ypd so bigger is better + diff = (baseline - current) / baseline + + below_tolerance = None + + if diff is not None: + below_tolerance = diff < tolerance + + if below_tolerance: + comment = "TPUTCOMP: Computation time changed by {:.2f}% relative to baseline".format( + diff * 100 + ) + else: + comment = "Error: TPUTCOMP: Computation time increase > {:d}% from baseline".format( + int(tolerance * 100) + ) + + return below_tolerance, comment + + +def compare_baseline_memory(case, baseline, tolerance): + """ + Default memory usage baseline comparison. + + Compares the highwater memory usage from the coupler to the baseline value. + + Parameters + ---------- + case : CIME.case.case.Case + Current case object. + baseline : list + Lines contained in the baseline file. + tolerance : float + Allowed tolerance for comparison. + + Returns + ------- + below_tolerance : bool + Whether the comparison was below the tolerance. + comment : str + provides explanation from comparison. + """ + try: + current = get_default_mem_usage(case) + except RuntimeError as e: + return None, str(e) + else: + current = current[-1][1] + + try: + # default baseline is stored as single float + baseline = float(baseline[0]) + except IndexError: + baseline = 0.0 + + try: + diff = (current - baseline) / baseline + except ZeroDivisionError: + diff = 0.0 + + # Should we check if tolerance is above 0 + below_tolerance = None + comment = "" + + if diff is not None: + below_tolerance = diff < tolerance + + if below_tolerance: + comment = "MEMCOMP: Memory usage highwater has changed by {:.2f}% relative to baseline".format( + diff * 100 + ) + else: + comment = "Error: Memory usage increase >{:d}% from baseline's {:f} to {:f}".format( + int(tolerance * 100), baseline, current + ) + + return below_tolerance, comment diff --git a/CIME/tests/test_unit_baselines.py b/CIME/tests/test_unit_baselines.py index 91e7b72d8a3..ae98b2aa79a 100644 --- a/CIME/tests/test_unit_baselines.py +++ b/CIME/tests/test_unit_baselines.py @@ -75,93 +75,75 @@ def test_get_cpl_mem_usage(self, isfile): (10105.0, 1673.89), ] - def test_read_baseline_mem_empty(self): - with mock.patch("builtins.open", mock.mock_open(read_data="")) as mock_file: - baseline = baselines.read_baseline_mem("/tmp/cpl-mem.log") - - assert baseline == [] - - def test_read_baseline_mem_none(self): - with mock.patch("builtins.open", mock.mock_open(read_data="-1")) as mock_file: - baseline = baselines.read_baseline_mem("/tmp/cpl-mem.log") + def test_read_baseline_file_multi_line(self): + with mock.patch( + "builtins.open", mock.mock_open(read_data="1000.0\n2000.0\n") + ) as mock_file: + baseline = baselines.read_baseline_file("/tmp/cpl-mem.log") - assert baseline == ["-1"] + mock_file.assert_called_with("/tmp/cpl-mem.log") + assert baseline == ["1000.0\n", "2000.0\n"] - def test_read_baseline_mem(self): - with mock.patch("builtins.open", mock.mock_open(read_data="200")) as mock_file: - baseline = baselines.read_baseline_mem("/tmp/cpl-mem.log") + def test_read_baseline_file_content(self): + with mock.patch( + "builtins.open", mock.mock_open(read_data="1000.0") + ) as mock_file: + baseline = baselines.read_baseline_file("/tmp/cpl-mem.log") - assert baseline == ["200"] + mock_file.assert_called_with("/tmp/cpl-mem.log") + assert baseline == ["1000.0"] - def test_read_baseline_tput_empty(self): + def test_read_baseline_file(self): with mock.patch("builtins.open", mock.mock_open(read_data="")) as mock_file: - baseline = baselines.read_baseline_tput("/tmp/cpl-tput.log") + baseline = baselines.read_baseline_file("/tmp/cpl-mem.log") + mock_file.assert_called_with("/tmp/cpl-mem.log") assert baseline == [] - def test_read_baseline_tput_none(self): - with mock.patch("builtins.open", mock.mock_open(read_data="-1")) as mock_file: - baseline = baselines.read_baseline_tput("/tmp/cpl-tput.log") - - assert baseline == ["-1"] - - def test_read_baseline_tput(self): - with mock.patch("builtins.open", mock.mock_open(read_data="200")) as mock_file: - baseline = baselines.read_baseline_tput("/tmp/cpl-tput.log") - - assert baseline == ["200"] - - def test_write_baseline_mem_no_value(self): - with mock.patch("builtins.open", mock.mock_open()) as mock_file: - baselines.write_baseline_mem("/tmp", "-1") - - mock_file.assert_called_with("/tmp/cpl-mem.log", "w") - mock_file.return_value.write.assert_called_with("-1") - - def test_write_baseline_mem(self): + def test_write_baseline_file(self): with mock.patch("builtins.open", mock.mock_open()) as mock_file: - baselines.write_baseline_mem("/tmp", "200") - - mock_file.assert_called_with("/tmp/cpl-mem.log", "w") - mock_file.return_value.write.assert_called_with("200") - - def test_write_baseline_tput_no_value(self): - with mock.patch("builtins.open", mock.mock_open()) as mock_file: - baselines.write_baseline_tput("/tmp", "-1") + baselines.write_baseline_file("/tmp/cpl-tput.log", "1000") mock_file.assert_called_with("/tmp/cpl-tput.log", "w") - mock_file.return_value.write.assert_called_with("-1") - - def test_write_baseline_tput(self): - with mock.patch("builtins.open", mock.mock_open()) as mock_file: - baselines.write_baseline_tput("/tmp", "200") - - mock_file.assert_called_with("/tmp/cpl-tput.log", "w") - mock_file.return_value.write.assert_called_with("200") + mock_file.return_value.write.assert_called_with("1000") @mock.patch("CIME.baselines.get_cpl_throughput") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_default_get_throughput(self, get_latest_cpl_logs, get_cpl_throughput): + def test_get_default_throughput(self, get_latest_cpl_logs, get_cpl_throughput): get_cpl_throughput.side_effect = FileNotFoundError() with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) - tput = baselines.default_get_throughput(case) + tput = baselines.get_default_throughput(case) assert tput == None @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_default_get_mem_usage(self, get_latest_cpl_logs, get_cpl_mem_usage): + def test_get_default_mem_usage_override( + self, get_latest_cpl_logs, get_cpl_mem_usage + ): + get_cpl_mem_usage.side_effect = FileNotFoundError() + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + mem = baselines.get_default_mem_usage(case, "/tmp/override") + + assert mem == None + + @mock.patch("CIME.baselines.get_cpl_mem_usage") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_get_default_mem_usage(self, get_latest_cpl_logs, get_cpl_mem_usage): get_cpl_mem_usage.side_effect = FileNotFoundError() with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) - mem = baselines.default_get_mem_usage(case) + mem = baselines.get_default_mem_usage(case) - assert mem == [(None, None)] + assert mem == None @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.get_latest_cpl_logs") @@ -180,15 +162,15 @@ def test_write_baseline(self, get_latest_cpl_logs, get_cpl_mem_usage): case, baseline_root, get_latest_cpl_logs.return_value ) - @mock.patch("CIME.baselines.get_cpl_throughput") - @mock.patch("CIME.baselines.read_baseline_tput") + @mock.patch("CIME.baselines.get_default_throughput") + @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_throughput_no_baseline_file( - self, get_latest_cpl_logs, read_baseline_tput, get_cpl_throughput + self, get_latest_cpl_logs, read_baseline_file, get_default_throughput ): - read_baseline_tput.side_effect = FileNotFoundError + read_baseline_file.side_effect = FileNotFoundError - get_cpl_throughput.return_value = 504 + get_default_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -205,15 +187,15 @@ def test_compare_throughput_no_baseline_file( assert below_tolerance is None assert comment == "Could not read baseline throughput file: " - @mock.patch("CIME.baselines.default_get_throughput") - @mock.patch("CIME.baselines.read_baseline_tput") + @mock.patch("CIME.baselines.get_default_throughput") + @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_throughput_no_baseline( - self, get_latest_cpl_logs, read_baseline_tput, default_get_throughput + self, get_latest_cpl_logs, read_baseline_file, get_default_throughput ): - read_baseline_tput.return_value = None + read_baseline_file.return_value = [] - default_get_throughput.return_value = 504 + get_default_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -230,17 +212,22 @@ def test_compare_throughput_no_baseline( (below_tolerance, comment) = baselines.compare_throughput(case) assert below_tolerance is None - assert comment == "Could not determine diff with baseline None and current 504" + assert ( + comment + == "Could not compare throughput to baseline, as basline had no value." + ) - @mock.patch("CIME.baselines.default_get_throughput") - @mock.patch("CIME.baselines.read_baseline_tput") + @mock.patch("CIME.baselines.get_default_throughput") + @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_throughput_no_tolerance( - self, get_latest_cpl_logs, read_baseline_tput, default_get_throughput + self, get_latest_cpl_logs, read_baseline_file, get_default_throughput ): - read_baseline_tput.return_value = 500 + read_baseline_file.return_value = [ + "500", + ] - default_get_throughput.return_value = 504 + get_default_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -262,15 +249,15 @@ def test_compare_throughput_no_tolerance( == "TPUTCOMP: Computation time changed by -0.80% relative to baseline" ) - @mock.patch("CIME.baselines.default_get_throughput") - @mock.patch("CIME.baselines.read_baseline_tput") + @mock.patch("CIME.baselines.get_default_throughput") + @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_throughput_above_threshold( - self, get_latest_cpl_logs, read_baseline_tput, default_get_throughput + self, get_latest_cpl_logs, read_baseline_file, get_default_throughput ): - read_baseline_tput.return_value = 1000 + read_baseline_file.return_value = ["1000"] - default_get_throughput.return_value = 504 + get_default_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -291,15 +278,15 @@ def test_compare_throughput_above_threshold( comment == "Error: TPUTCOMP: Computation time increase > 5% from baseline" ) - @mock.patch("CIME.baselines.default_get_throughput") - @mock.patch("CIME.baselines.read_baseline_tput") + @mock.patch("CIME.baselines.get_default_throughput") + @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_throughput( - self, get_latest_cpl_logs, read_baseline_tput, default_get_throughput + self, get_latest_cpl_logs, read_baseline_file, get_default_throughput ): - read_baseline_tput.return_value = 500 + read_baseline_file.return_value = ["500"] - default_get_throughput.return_value = 504 + get_default_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -322,12 +309,12 @@ def test_compare_throughput( ) @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.read_baseline_mem") + @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_memory_no_baseline( - self, get_latest_cpl_logs, read_baseline_mem, get_cpl_mem_usage + self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): - read_baseline_mem.return_value = None + read_baseline_file.return_value = [] get_cpl_mem_usage.return_value = [ (1, 1000.0), @@ -357,12 +344,12 @@ def test_compare_memory_no_baseline( ) @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.read_baseline_mem") + @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_memory_not_enough_samples( - self, get_latest_cpl_logs, read_baseline_mem, get_cpl_mem_usage + self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): - read_baseline_mem.return_value = 1000.0 + read_baseline_file.return_value = ["1000.0"] get_cpl_mem_usage.return_value = [ (1, 1000.0), @@ -385,12 +372,12 @@ def test_compare_memory_not_enough_samples( assert comment == "Found 2 memory usage samples, need atleast 4" @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.read_baseline_mem") + @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_memory_no_baseline_file( - self, get_latest_cpl_logs, read_baseline_mem, get_cpl_mem_usage + self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): - read_baseline_mem.side_effect = FileNotFoundError + read_baseline_file.side_effect = FileNotFoundError get_cpl_mem_usage.return_value = [ (1, 1000.0), @@ -415,12 +402,12 @@ def test_compare_memory_no_baseline_file( assert comment == "Could not read baseline memory usage: " @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.read_baseline_mem") + @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_memory_no_tolerance( - self, get_latest_cpl_logs, read_baseline_mem, get_cpl_mem_usage + self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): - read_baseline_mem.return_value = 1000.0 + read_baseline_file.return_value = ["1000.0"] get_cpl_mem_usage.return_value = [ (1, 1000.0), @@ -450,12 +437,12 @@ def test_compare_memory_no_tolerance( ) @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.read_baseline_mem") + @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_memory_above_threshold( - self, get_latest_cpl_logs, read_baseline_mem, get_cpl_mem_usage + self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): - read_baseline_mem.return_value = 1000.0 + read_baseline_file.return_value = ["1000.0"] get_cpl_mem_usage.return_value = [ (1, 2000.0), @@ -485,12 +472,12 @@ def test_compare_memory_above_threshold( ) @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.read_baseline_mem") + @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_memory( - self, get_latest_cpl_logs, read_baseline_mem, get_cpl_mem_usage + self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): - read_baseline_mem.return_value = 1000.0 + read_baseline_file.return_value = ["1000.0"] get_cpl_mem_usage.return_value = [ (1, 1000.0), diff --git a/CIME/tests/test_unit_system_tests.py b/CIME/tests/test_unit_system_tests.py index 9ea088411bd..fc684321cc0 100644 --- a/CIME/tests/test_unit_system_tests.py +++ b/CIME/tests/test_unit_system_tests.py @@ -66,7 +66,186 @@ def create_mock_case(tempdir, idx=None, cpllog_data=None): return case, caseroot, baseline_root, run_dir -class TestCaseSubmit(unittest.TestCase): +class TestUnitSystemTests(unittest.TestCase): + @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") + @mock.patch("CIME.SystemTests.system_tests_common.get_default_mem_usage") + @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") + def test_check_for_memleak_runtime_error( + self, get_latest_cpl_logs, get_default_mem_usage, append_testlog + ): + get_default_mem_usage.side_effect = RuntimeError + + with tempfile.TemporaryDirectory() as tempdir: + caseroot = Path(tempdir) / "caseroot" + caseroot.mkdir(parents=True, exist_ok=False) + + rundir = caseroot / "run" + rundir.mkdir(parents=True, exist_ok=False) + + cpllog = rundir / "cpl.log.gz" + + get_latest_cpl_logs.return_value = [ + str(cpllog), + ] + + case = mock.MagicMock() + case.get_value.side_effect = ( + str(caseroot), + "ERIO.ne30_g16_rx1.A.docker_gnu", + "mct", + 0.01, + ) + + common = SystemTestsCommon(case) + + common._test_status = mock.MagicMock() + + common._check_for_memleak() + + common._test_status.set_status.assert_any_call( + "MEMLEAK", "PASS", comments="insufficient data for memleak test" + ) + + append_testlog.assert_not_called() + + @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") + @mock.patch("CIME.SystemTests.system_tests_common.get_default_mem_usage") + @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") + def test_check_for_memleak_not_enough_samples( + self, get_latest_cpl_logs, get_default_mem_usage, append_testlog + ): + get_default_mem_usage.return_value = [ + (1, 1000.0), + (2, 0), + ] + + with tempfile.TemporaryDirectory() as tempdir: + caseroot = Path(tempdir) / "caseroot" + caseroot.mkdir(parents=True, exist_ok=False) + + rundir = caseroot / "run" + rundir.mkdir(parents=True, exist_ok=False) + + cpllog = rundir / "cpl.log.gz" + + get_latest_cpl_logs.return_value = [ + str(cpllog), + ] + + case = mock.MagicMock() + case.get_value.side_effect = ( + str(caseroot), + "ERIO.ne30_g16_rx1.A.docker_gnu", + "mct", + 0.01, + ) + + common = SystemTestsCommon(case) + + common._test_status = mock.MagicMock() + + common._check_for_memleak() + + common._test_status.set_status.assert_any_call( + "MEMLEAK", "PASS", comments="data for memleak test is insufficient" + ) + + append_testlog.assert_not_called() + + @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") + @mock.patch("CIME.SystemTests.system_tests_common.get_default_mem_usage") + @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") + def test_check_for_memleak_found( + self, get_latest_cpl_logs, get_default_mem_usage, append_testlog + ): + get_default_mem_usage.return_value = [ + (1, 1000.0), + (2, 2000.0), + (3, 3000.0), + (4, 3000.0), + ] + + with tempfile.TemporaryDirectory() as tempdir: + caseroot = Path(tempdir) / "caseroot" + caseroot.mkdir(parents=True, exist_ok=False) + + rundir = caseroot / "run" + rundir.mkdir(parents=True, exist_ok=False) + + cpllog = rundir / "cpl.log.gz" + + get_latest_cpl_logs.return_value = [ + str(cpllog), + ] + + case = mock.MagicMock() + case.get_value.side_effect = ( + str(caseroot), + "ERIO.ne30_g16_rx1.A.docker_gnu", + "mct", + 0.01, + ) + + common = SystemTestsCommon(case) + + common._test_status = mock.MagicMock() + + common._check_for_memleak() + + expected_comment = "memleak detected, memory went from 2000.000000 to 3000.000000 in 2 days" + + common._test_status.set_status.assert_any_call( + "MEMLEAK", "FAIL", comments=expected_comment + ) + + append_testlog.assert_any_call(expected_comment, str(caseroot)) + + @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") + @mock.patch("CIME.SystemTests.system_tests_common.get_default_mem_usage") + @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") + def test_check_for_memleak( + self, get_latest_cpl_logs, get_default_mem_usage, append_testlog + ): + get_default_mem_usage.return_value = [ + (1, 3040.0), + (2, 3002.0), + (3, 3030.0), + (4, 3008.0), + ] + + with tempfile.TemporaryDirectory() as tempdir: + caseroot = Path(tempdir) / "caseroot" + caseroot.mkdir(parents=True, exist_ok=False) + + rundir = caseroot / "run" + rundir.mkdir(parents=True, exist_ok=False) + + cpllog = rundir / "cpl.log.gz" + + get_latest_cpl_logs.return_value = [ + str(cpllog), + ] + + case = mock.MagicMock() + case.get_value.side_effect = ( + str(caseroot), + "ERIO.ne30_g16_rx1.A.docker_gnu", + "mct", + 0.01, + ) + + common = SystemTestsCommon(case) + + common._test_status = mock.MagicMock() + + common._check_for_memleak() + + print(common._test_status.set_status.call_args_list) + + common._test_status.set_status.assert_any_call("MEMLEAK", "PASS") + + append_testlog.assert_not_called() + @mock.patch("CIME.SystemTests.system_tests_common.compare_throughput") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") def test_compare_throughput(self, append_testlog, compare_throughput): From 1c6d7a15057d5f86b1679cee0c9fef9c62cb12f6 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 12 Oct 2023 14:19:38 -0700 Subject: [PATCH 19/57] Fixes handling non performance metrics --- CIME/baselines.py | 32 +++++-- CIME/tests/test_unit_baselines.py | 149 +++++++++++++++++++++++++++--- 2 files changed, 158 insertions(+), 23 deletions(-) diff --git a/CIME/baselines.py b/CIME/baselines.py index 47f74a698fd..1011bfc815b 100644 --- a/CIME/baselines.py +++ b/CIME/baselines.py @@ -132,18 +132,24 @@ def write_baseline(case, basegen_dir, throughput=True, memory=True): config = load_coupler_customization(case) if throughput: - tput = get_throughput(case, config) - - baseline_file = os.path.join(basegen_dir, "cpl-tput.log") + try: + tput = get_throughput(case, config) + except RuntimeError: + pass + else: + baseline_file = os.path.join(basegen_dir, "cpl-tput.log") - write_baseline_file(baseline_file, tput) + write_baseline_file(baseline_file, tput) if memory: - mem = get_mem_usage(case, config) - - baseline_file = os.path.join(basegen_dir, "cpl-mem.log") + try: + mem = get_mem_usage(case, config) + except RuntimeError: + pass + else: + baseline_file = os.path.join(basegen_dir, "cpl-mem.log") - write_baseline_file(baseline_file, mem) + write_baseline_file(baseline_file, mem) def load_coupler_customization(case): @@ -188,7 +194,12 @@ def get_throughput(case, config): try: tput = config.get_throughput(case) except AttributeError: - tput = str(get_default_throughput(case)) + tput = get_default_throughput(case) + + if tput is None: + raise RuntimeError("Could not get default throughput") from None + + tput = str(tput) return tput @@ -216,6 +227,9 @@ def get_mem_usage(case, config): except AttributeError: mem = get_default_mem_usage(case) + if mem is None: + raise RuntimeError("Could not get default memory usage") from None + mem = str(mem[-1][1]) return mem diff --git a/CIME/tests/test_unit_baselines.py b/CIME/tests/test_unit_baselines.py index ae98b2aa79a..a5af4c4a9a2 100644 --- a/CIME/tests/test_unit_baselines.py +++ b/CIME/tests/test_unit_baselines.py @@ -10,12 +10,13 @@ from CIME.tests.test_unit_system_tests import CPLLOG -def create_mock_case(tempdir, get_latest_cpl_logs): +def create_mock_case(tempdir, get_latest_cpl_logs=None): caseroot = Path(tempdir, "0", "caseroot") rundir = caseroot / "run" - get_latest_cpl_logs.return_value = (str(rundir / "cpl.log.gz"),) + if get_latest_cpl_logs is not None: + get_latest_cpl_logs.return_value = (str(rundir / "cpl.log.gz"),) baseline_root = Path(tempdir, "baselines") @@ -27,6 +28,82 @@ def create_mock_case(tempdir, get_latest_cpl_logs): class TestUnitBaseline(unittest.TestCase): + @mock.patch("CIME.baselines.get_default_mem_usage") + def test_get_mem_usage_default_no_value(self, get_default_mem_usage): + get_default_mem_usage.return_value = None + + case = mock.MagicMock() + + config = mock.MagicMock() + + config.get_mem_usage.side_effect = AttributeError + + with self.assertRaises(RuntimeError): + baselines.get_mem_usage(case, config) + + @mock.patch("CIME.baselines.get_default_mem_usage") + def test_get_mem_usage_default(self, get_default_mem_usage): + get_default_mem_usage.return_value = [(1, 1000)] + + case = mock.MagicMock() + + config = mock.MagicMock() + + config.get_mem_usage.side_effect = AttributeError + + mem = baselines.get_mem_usage(case, config) + + assert mem == "1000" + + def test_get_mem_usage(self): + case = mock.MagicMock() + + config = mock.MagicMock() + + config.get_mem_usage.return_value = "1000" + + mem = baselines.get_mem_usage(case, config) + + assert mem == "1000" + + @mock.patch("CIME.baselines.get_default_throughput") + def test_get_throughput_default_no_value(self, get_default_throughput): + get_default_throughput.return_value = None + + case = mock.MagicMock() + + config = mock.MagicMock() + + config.get_throughput.side_effect = AttributeError + + with self.assertRaises(RuntimeError): + baselines.get_throughput(case, config) + + @mock.patch("CIME.baselines.get_default_throughput") + def test_get_throughput_default(self, get_default_throughput): + get_default_throughput.return_value = 100 + + case = mock.MagicMock() + + config = mock.MagicMock() + + config.get_throughput.side_effect = AttributeError + + tput = baselines.get_throughput(case, config) + + assert tput == "100" + + def test_get_throughput(self): + case = mock.MagicMock() + + config = mock.MagicMock() + + config.get_throughput.return_value = "100" + + tput = baselines.get_throughput(case, config) + + assert tput == "100" + def test_get_cpl_throughput_no_file(self): throughput = baselines.get_cpl_throughput("/tmp/cpl.log") @@ -145,23 +222,67 @@ def test_get_default_mem_usage(self, get_latest_cpl_logs, get_cpl_mem_usage): assert mem == None - @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_write_baseline(self, get_latest_cpl_logs, get_cpl_mem_usage): - with tempfile.TemporaryDirectory() as tempdir: - case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + @mock.patch("CIME.baselines.write_baseline_file") + @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.get_throughput") + def test_write_baseline_skip( + self, get_throughput, get_mem_usage, write_baseline_file + ): + get_throughput.return_value = "100" + + get_mem_usage.return_value = "1000" - get_cpl_mem_usage.return_value = [ - (1, 1000.0), - (2, 1001.0), - (3, 1002.0), - (4, 1003.0), - ] + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir) baselines.write_baseline( - case, baseline_root, get_latest_cpl_logs.return_value + case, + baseline_root, + False, + False, ) + get_throughput.assert_not_called() + get_mem_usage.assert_not_called() + write_baseline_file.assert_not_called() + + @mock.patch("CIME.baselines.write_baseline_file") + @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.get_throughput") + def test_write_baseline_runtimeerror( + self, get_throughput, get_mem_usage, write_baseline_file + ): + get_throughput.side_effect = RuntimeError + + get_mem_usage.side_effect = RuntimeError + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir) + + baselines.write_baseline(case, baseline_root) + + get_throughput.assert_called() + get_mem_usage.assert_called() + write_baseline_file.assert_not_called() + + @mock.patch("CIME.baselines.write_baseline_file") + @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.get_throughput") + def test_write_baseline(self, get_throughput, get_mem_usage, write_baseline_file): + get_throughput.return_value = "100" + + get_mem_usage.return_value = "1000" + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir) + + baselines.write_baseline(case, baseline_root) + + get_throughput.assert_called() + get_mem_usage.assert_called() + write_baseline_file.assert_any_call(str(baseline_root / "cpl-tput.log"), "100") + write_baseline_file.assert_any_call(str(baseline_root / "cpl-mem.log"), "1000") + @mock.patch("CIME.baselines.get_default_throughput") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") From 3596b5526580a10017e9a464664afdd3bc20fb6a Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 12 Oct 2023 15:09:59 -0700 Subject: [PATCH 20/57] Updates _check_for_memleak to be customized by the coupler --- CIME/SystemTests/system_tests_common.py | 97 ++++++++++++++----------- CIME/tests/test_unit_system_tests.py | 50 +++++++++++-- 2 files changed, 99 insertions(+), 48 deletions(-) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index 43c08d45c7f..0fd90c5cfa0 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -32,6 +32,7 @@ compare_memory, compare_throughput, write_baseline, + load_coupler_customization, ) import CIME.build as build @@ -631,49 +632,22 @@ def _check_for_memleak(self): Examine memory usage as recorded in the cpl log file and look for unexpected increases. """ + config = load_coupler_customization(self._case) + with self._test_status: - latestcpllogs = get_latest_cpl_logs(self._case) - for cpllog in latestcpllogs: - try: - memlist = get_default_mem_usage(self._case, cpllog) - except RuntimeError: - self._test_status.set_status( - MEMLEAK_PHASE, - TEST_PASS_STATUS, - comments="insufficient data for memleak test", - ) + try: + memleak, comment = config.detect_memory_leak(self._case) + except AttributeError: + memleak, comment = default_detect_memory_leak(self._case) - continue - - finaldate = int(memlist[-1][0]) - originaldate = int( - memlist[1][0] - ) # skip first day mem record, it can be too low while initializing - finalmem = float(memlist[-1][1]) - originalmem = float(memlist[1][1]) - memdiff = -1 - if originalmem > 0: - memdiff = (finalmem - originalmem) / originalmem - tolerance = self._case.get_value("TEST_MEMLEAK_TOLERANCE") - if tolerance is None: - tolerance = 0.1 - expect(tolerance > 0.0, "Bad value for memleak tolerance in test") - if memdiff < 0: - self._test_status.set_status( - MEMLEAK_PHASE, - TEST_PASS_STATUS, - comments="data for memleak test is insufficient", - ) - elif memdiff < tolerance: - self._test_status.set_status(MEMLEAK_PHASE, TEST_PASS_STATUS) - else: - comment = "memleak detected, memory went from {:f} to {:f} in {:d} days".format( - originalmem, finalmem, finaldate - originaldate - ) - append_testlog(comment, self._orig_caseroot) - self._test_status.set_status( - MEMLEAK_PHASE, TEST_FAIL_STATUS, comments=comment - ) + if memleak: + append_testlog(comment, self._orig_caseroot) + + status = TEST_FAIL_STATUS + else: + status = TEST_PASS_STATUS + + self._test_status.set_status(MEMLEAK_PHASE, status, comments=comment) def compare_env_run(self, expected=None): """ @@ -798,6 +772,47 @@ def _generate_baseline(self): write_baseline(self._case, basegen_dir, cpllog) +def default_detect_memory_leak(case): + leak = False + comment = "" + + latestcpllogs = get_latest_cpl_logs(case) + + for cpllog in latestcpllogs: + try: + memlist = get_default_mem_usage(case, cpllog) + except RuntimeError: + return False, "insufficient data for memleak test" + + # last day - second day, skip first day, can be too low while initializing + elapsed_days = int(memlist[-1][0]) - int(memlist[1][0]) + + finalmem, originalmem = float(memlist[-1][1]), float(memlist[1][1]) + + memdiff = -1 if originalmem <= 0 else (finalmem - originalmem) / originalmem + + # default to 0.1 + tolerance = case.get_value("TEST_MEMLEAK_TOLERANCE") or 0.1 + + expect(tolerance > 0.0, "Bad value for memleak tolerance in test") + + if memdiff < 0: + leak = False + comment = "data for memleak test is insufficient" + elif memdiff < tolerance: + leak = False + comment = "" + else: + leak = True + comment = ( + "memleak detected, memory went from {:f} to {:f} in {:d} days".format( + originalmem, finalmem, elapsed_days + ) + ) + + return leak, comment + + class FakeTest(SystemTestsCommon): """ Inheriters of the FakeTest Class are intended to test the code. diff --git a/CIME/tests/test_unit_system_tests.py b/CIME/tests/test_unit_system_tests.py index fc684321cc0..2f9f7d9918b 100644 --- a/CIME/tests/test_unit_system_tests.py +++ b/CIME/tests/test_unit_system_tests.py @@ -67,12 +67,21 @@ def create_mock_case(tempdir, idx=None, cpllog_data=None): class TestUnitSystemTests(unittest.TestCase): + @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") @mock.patch("CIME.SystemTests.system_tests_common.get_default_mem_usage") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak_runtime_error( - self, get_latest_cpl_logs, get_default_mem_usage, append_testlog + self, + get_latest_cpl_logs, + get_default_mem_usage, + append_testlog, + load_coupler_customization, ): + load_coupler_customization.return_value.detect_memory_leak.side_effect = ( + AttributeError + ) + get_default_mem_usage.side_effect = RuntimeError with tempfile.TemporaryDirectory() as tempdir: @@ -108,12 +117,21 @@ def test_check_for_memleak_runtime_error( append_testlog.assert_not_called() + @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") @mock.patch("CIME.SystemTests.system_tests_common.get_default_mem_usage") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak_not_enough_samples( - self, get_latest_cpl_logs, get_default_mem_usage, append_testlog + self, + get_latest_cpl_logs, + get_default_mem_usage, + append_testlog, + load_coupler_customization, ): + load_coupler_customization.return_value.detect_memory_leak.side_effect = ( + AttributeError + ) + get_default_mem_usage.return_value = [ (1, 1000.0), (2, 0), @@ -152,12 +170,21 @@ def test_check_for_memleak_not_enough_samples( append_testlog.assert_not_called() + @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") @mock.patch("CIME.SystemTests.system_tests_common.get_default_mem_usage") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak_found( - self, get_latest_cpl_logs, get_default_mem_usage, append_testlog + self, + get_latest_cpl_logs, + get_default_mem_usage, + append_testlog, + load_coupler_customization, ): + load_coupler_customization.return_value.detect_memory_leak.side_effect = ( + AttributeError + ) + get_default_mem_usage.return_value = [ (1, 1000.0), (2, 2000.0), @@ -200,12 +227,21 @@ def test_check_for_memleak_found( append_testlog.assert_any_call(expected_comment, str(caseroot)) + @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") @mock.patch("CIME.SystemTests.system_tests_common.get_default_mem_usage") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak( - self, get_latest_cpl_logs, get_default_mem_usage, append_testlog + self, + get_latest_cpl_logs, + get_default_mem_usage, + append_testlog, + load_coupler_customization, ): + load_coupler_customization.return_value.detect_memory_leak.side_effect = ( + AttributeError + ) + get_default_mem_usage.return_value = [ (1, 3040.0), (2, 3002.0), @@ -240,9 +276,9 @@ def test_check_for_memleak( common._check_for_memleak() - print(common._test_status.set_status.call_args_list) - - common._test_status.set_status.assert_any_call("MEMLEAK", "PASS") + common._test_status.set_status.assert_any_call( + "MEMLEAK", "PASS", comments="" + ) append_testlog.assert_not_called() From 789002130f2294cbd6d8861667f50e49fef99c45 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 13 Oct 2023 12:41:10 -0700 Subject: [PATCH 21/57] Renames methods --- CIME/SystemTests/system_tests_common.py | 16 +-- CIME/baselines.py | 32 +++--- CIME/bless_test_results.py | 18 ++-- CIME/tests/test_unit_baselines.py | 132 +++++++++++++----------- CIME/tests/test_unit_system_tests.py | 68 ++++++------ 5 files changed, 145 insertions(+), 121 deletions(-) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index 0fd90c5cfa0..7fca1010a60 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -28,10 +28,10 @@ from CIME.locked_files import LOCKED_DIR, lock_file, is_locked from CIME.baselines import ( get_latest_cpl_logs, - get_default_mem_usage, - compare_memory, - compare_throughput, - write_baseline, + _get_mem_usage, + perf_compare_memory_baseline, + perf_compare_throughput_baseline, + perf_write_baseline, load_coupler_customization, ) import CIME.build as build @@ -679,7 +679,7 @@ def _compare_memory(self): Compares current test memory usage to baseline. """ with self._test_status: - below_tolerance, comment = compare_memory(self._case) + below_tolerance, comment = perf_compare_memory_baseline(self._case) if below_tolerance is not None: append_testlog(comment, self._orig_caseroot) @@ -699,7 +699,7 @@ def _compare_throughput(self): Compares current test throughput to baseline. """ with self._test_status: - below_tolerance, comment = compare_throughput(self._case) + below_tolerance, comment = perf_compare_throughput_baseline(self._case) if below_tolerance is not None: append_testlog(comment, self._orig_caseroot) @@ -769,7 +769,7 @@ def _generate_baseline(self): preserve_meta=False, ) - write_baseline(self._case, basegen_dir, cpllog) + perf_write_baseline(self._case, basegen_dir, cpllog) def default_detect_memory_leak(case): @@ -780,7 +780,7 @@ def default_detect_memory_leak(case): for cpllog in latestcpllogs: try: - memlist = get_default_mem_usage(case, cpllog) + memlist = _get_mem_usage(case, cpllog) except RuntimeError: return False, "insufficient data for memleak test" diff --git a/CIME/baselines.py b/CIME/baselines.py index 1011bfc815b..a8c1aecd7ac 100644 --- a/CIME/baselines.py +++ b/CIME/baselines.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) -def compare_throughput(case, baseline_dir=None): +def perf_compare_throughput_baseline(case, baseline_dir=None): """ Compares model throughput. @@ -54,18 +54,18 @@ def compare_throughput(case, baseline_dir=None): ) try: - below_tolerance, comment = config.compare_baseline_throughput( + below_tolerance, comment = config.perf_compare_throughput_baseline( case, baseline, tolerance ) except AttributeError: - below_tolerance, comment = compare_baseline_throughput( + below_tolerance, comment = _perf_compare_throughput_baseline( case, baseline, tolerance ) return below_tolerance, comment -def compare_memory(case, baseline_dir=None): +def perf_compare_memory_baseline(case, baseline_dir=None): """ Compares model highwater memory usage. @@ -105,16 +105,18 @@ def compare_memory(case, baseline_dir=None): tolerance = 0.1 try: - below_tolerance, comments = config.compare_baseline_memory( + below_tolerance, comments = config.perf_compare_memory_baseline( case, baseline, tolerance ) except AttributeError: - below_tolerance, comments = compare_baseline_memory(case, baseline, tolerance) + below_tolerance, comments = _perf_compare_memory_baseline( + case, baseline, tolerance + ) return below_tolerance, comments -def write_baseline(case, basegen_dir, throughput=True, memory=True): +def perf_write_baseline(case, basegen_dir, throughput=True, memory=True): """ Writes the baseline performance files. @@ -194,7 +196,7 @@ def get_throughput(case, config): try: tput = config.get_throughput(case) except AttributeError: - tput = get_default_throughput(case) + tput = _get_throughput(case) if tput is None: raise RuntimeError("Could not get default throughput") from None @@ -225,7 +227,7 @@ def get_mem_usage(case, config): try: mem = config.get_mem_usage(case) except AttributeError: - mem = get_default_mem_usage(case) + mem = _get_mem_usage(case) if mem is None: raise RuntimeError("Could not get default memory usage") from None @@ -250,7 +252,7 @@ def write_baseline_file(baseline_file, value): fd.write(value) -def get_default_mem_usage(case, cpllog=None): +def _get_mem_usage(case, cpllog=None): """ Default function to retrieve memory usage from the coupler log. @@ -293,7 +295,7 @@ def get_default_mem_usage(case, cpllog=None): return memlist -def get_default_throughput(case): +def _get_throughput(case): """ Default function to retrieve throughput from the coupler log. @@ -431,7 +433,7 @@ def read_baseline_file(baseline_file): return lines -def compare_baseline_throughput(case, baseline, tolerance): +def _perf_compare_throughput_baseline(case, baseline, tolerance): """ Default throughput baseline comparison. @@ -453,7 +455,7 @@ def compare_baseline_throughput(case, baseline, tolerance): comment : str provides explanation from comparison. """ - current = get_default_throughput(case) + current = _get_throughput(case) try: # default baseline is stored as single float @@ -483,7 +485,7 @@ def compare_baseline_throughput(case, baseline, tolerance): return below_tolerance, comment -def compare_baseline_memory(case, baseline, tolerance): +def _perf_compare_memory_baseline(case, baseline, tolerance): """ Default memory usage baseline comparison. @@ -506,7 +508,7 @@ def compare_baseline_memory(case, baseline, tolerance): provides explanation from comparison. """ try: - current = get_default_mem_usage(case) + current = _get_mem_usage(case) except RuntimeError as e: return None, str(e) else: diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index d2a88d4b2c8..c2ba6fbb1c9 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -12,9 +12,9 @@ from CIME.case import Case from CIME.test_utils import get_test_status_files from CIME.baselines import ( - compare_throughput, - compare_memory, - write_baseline, + perf_compare_throughput_baseline, + perf_compare_memory_baseline, + perf_write_baseline, ) import os, time @@ -36,7 +36,9 @@ def bless_throughput( baseline_root, baseline_name, case.get_value("CASEBASEID") ) - below_threshold, comment = compare_throughput(case, baseline_dir=baseline_dir) + below_threshold, comment = perf_compare_throughput_baseline( + case, baseline_dir=baseline_dir + ) if below_threshold: logger.info("Diff appears to have been already resolved.") @@ -47,7 +49,7 @@ def bless_throughput( force or input("Update this diff (y/n)? ").upper() in ["Y", "YES"] ): try: - write_baseline(case, baseline_dir, memory=False) + perf_write_baseline(case, baseline_dir, memory=False) except Exception as e: success = False @@ -71,7 +73,9 @@ def bless_memory( baseline_root, baseline_name, case.get_value("CASEBASEID") ) - below_threshold, comment = compare_memory(case, baseline_dir=baseline_dir) + below_threshold, comment = perf_compare_memory_baseline( + case, baseline_dir=baseline_dir + ) if below_threshold: logger.info("Diff appears to have been already resolved.") @@ -82,7 +86,7 @@ def bless_memory( force or input("Update this diff (y/n)? ").upper() in ["Y", "YES"] ): try: - write_baseline(case, baseline_dir, throughput=False) + perf_write_baseline(case, baseline_dir, throughput=False) except Exception as e: success = False diff --git a/CIME/tests/test_unit_baselines.py b/CIME/tests/test_unit_baselines.py index a5af4c4a9a2..0b357e51253 100644 --- a/CIME/tests/test_unit_baselines.py +++ b/CIME/tests/test_unit_baselines.py @@ -28,9 +28,9 @@ def create_mock_case(tempdir, get_latest_cpl_logs=None): class TestUnitBaseline(unittest.TestCase): - @mock.patch("CIME.baselines.get_default_mem_usage") - def test_get_mem_usage_default_no_value(self, get_default_mem_usage): - get_default_mem_usage.return_value = None + @mock.patch("CIME.baselines._get_mem_usage") + def test_get_mem_usage_default_no_value(self, _get_mem_usage): + _get_mem_usage.return_value = None case = mock.MagicMock() @@ -41,9 +41,9 @@ def test_get_mem_usage_default_no_value(self, get_default_mem_usage): with self.assertRaises(RuntimeError): baselines.get_mem_usage(case, config) - @mock.patch("CIME.baselines.get_default_mem_usage") - def test_get_mem_usage_default(self, get_default_mem_usage): - get_default_mem_usage.return_value = [(1, 1000)] + @mock.patch("CIME.baselines._get_mem_usage") + def test_get_mem_usage_default(self, _get_mem_usage): + _get_mem_usage.return_value = [(1, 1000)] case = mock.MagicMock() @@ -66,9 +66,9 @@ def test_get_mem_usage(self): assert mem == "1000" - @mock.patch("CIME.baselines.get_default_throughput") - def test_get_throughput_default_no_value(self, get_default_throughput): - get_default_throughput.return_value = None + @mock.patch("CIME.baselines._get_throughput") + def test_get_throughput_default_no_value(self, _get_throughput): + _get_throughput.return_value = None case = mock.MagicMock() @@ -79,9 +79,9 @@ def test_get_throughput_default_no_value(self, get_default_throughput): with self.assertRaises(RuntimeError): baselines.get_throughput(case, config) - @mock.patch("CIME.baselines.get_default_throughput") - def test_get_throughput_default(self, get_default_throughput): - get_default_throughput.return_value = 100 + @mock.patch("CIME.baselines._get_throughput") + def test_get_throughput_default(self, _get_throughput): + _get_throughput.return_value = 100 case = mock.MagicMock() @@ -186,39 +186,37 @@ def test_write_baseline_file(self): @mock.patch("CIME.baselines.get_cpl_throughput") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_get_default_throughput(self, get_latest_cpl_logs, get_cpl_throughput): + def test__get_throughput(self, get_latest_cpl_logs, get_cpl_throughput): get_cpl_throughput.side_effect = FileNotFoundError() with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) - tput = baselines.get_default_throughput(case) + tput = baselines._get_throughput(case) assert tput == None @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_get_default_mem_usage_override( - self, get_latest_cpl_logs, get_cpl_mem_usage - ): + def test__get_mem_usage_override(self, get_latest_cpl_logs, get_cpl_mem_usage): get_cpl_mem_usage.side_effect = FileNotFoundError() with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) - mem = baselines.get_default_mem_usage(case, "/tmp/override") + mem = baselines._get_mem_usage(case, "/tmp/override") assert mem == None @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_get_default_mem_usage(self, get_latest_cpl_logs, get_cpl_mem_usage): + def test__get_mem_usage(self, get_latest_cpl_logs, get_cpl_mem_usage): get_cpl_mem_usage.side_effect = FileNotFoundError() with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) - mem = baselines.get_default_mem_usage(case) + mem = baselines._get_mem_usage(case) assert mem == None @@ -235,7 +233,7 @@ def test_write_baseline_skip( with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir) - baselines.write_baseline( + baselines.perf_write_baseline( case, baseline_root, False, @@ -259,7 +257,7 @@ def test_write_baseline_runtimeerror( with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir) - baselines.write_baseline(case, baseline_root) + baselines.perf_write_baseline(case, baseline_root) get_throughput.assert_called() get_mem_usage.assert_called() @@ -268,7 +266,9 @@ def test_write_baseline_runtimeerror( @mock.patch("CIME.baselines.write_baseline_file") @mock.patch("CIME.baselines.get_mem_usage") @mock.patch("CIME.baselines.get_throughput") - def test_write_baseline(self, get_throughput, get_mem_usage, write_baseline_file): + def test_perf_write_baseline( + self, get_throughput, get_mem_usage, write_baseline_file + ): get_throughput.return_value = "100" get_mem_usage.return_value = "1000" @@ -276,22 +276,22 @@ def test_write_baseline(self, get_throughput, get_mem_usage, write_baseline_file with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir) - baselines.write_baseline(case, baseline_root) + baselines.perf_write_baseline(case, baseline_root) get_throughput.assert_called() get_mem_usage.assert_called() write_baseline_file.assert_any_call(str(baseline_root / "cpl-tput.log"), "100") write_baseline_file.assert_any_call(str(baseline_root / "cpl-mem.log"), "1000") - @mock.patch("CIME.baselines.get_default_throughput") + @mock.patch("CIME.baselines._get_throughput") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_throughput_no_baseline_file( - self, get_latest_cpl_logs, read_baseline_file, get_default_throughput + def test_perf_compare_throughput_baseline_no_baseline_file( + self, get_latest_cpl_logs, read_baseline_file, _get_throughput ): read_baseline_file.side_effect = FileNotFoundError - get_default_throughput.return_value = 504 + _get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -303,20 +303,22 @@ def test_compare_throughput_no_baseline_file( 0.05, ) - (below_tolerance, comment) = baselines.compare_throughput(case) + (below_tolerance, comment) = baselines.perf_compare_throughput_baseline( + case + ) assert below_tolerance is None assert comment == "Could not read baseline throughput file: " - @mock.patch("CIME.baselines.get_default_throughput") + @mock.patch("CIME.baselines._get_throughput") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_throughput_no_baseline( - self, get_latest_cpl_logs, read_baseline_file, get_default_throughput + def test_perf_compare_throughput_baseline_no_baseline( + self, get_latest_cpl_logs, read_baseline_file, _get_throughput ): read_baseline_file.return_value = [] - get_default_throughput.return_value = 504 + _get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -330,7 +332,9 @@ def test_compare_throughput_no_baseline( 0.05, ) - (below_tolerance, comment) = baselines.compare_throughput(case) + (below_tolerance, comment) = baselines.perf_compare_throughput_baseline( + case + ) assert below_tolerance is None assert ( @@ -338,17 +342,17 @@ def test_compare_throughput_no_baseline( == "Could not compare throughput to baseline, as basline had no value." ) - @mock.patch("CIME.baselines.get_default_throughput") + @mock.patch("CIME.baselines._get_throughput") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_throughput_no_tolerance( - self, get_latest_cpl_logs, read_baseline_file, get_default_throughput + def test_perf_compare_throughput_baseline_no_tolerance( + self, get_latest_cpl_logs, read_baseline_file, _get_throughput ): read_baseline_file.return_value = [ "500", ] - get_default_throughput.return_value = 504 + _get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -362,7 +366,9 @@ def test_compare_throughput_no_tolerance( None, ) - (below_tolerance, comment) = baselines.compare_throughput(case) + (below_tolerance, comment) = baselines.perf_compare_throughput_baseline( + case + ) assert below_tolerance assert ( @@ -370,15 +376,15 @@ def test_compare_throughput_no_tolerance( == "TPUTCOMP: Computation time changed by -0.80% relative to baseline" ) - @mock.patch("CIME.baselines.get_default_throughput") + @mock.patch("CIME.baselines._get_throughput") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_throughput_above_threshold( - self, get_latest_cpl_logs, read_baseline_file, get_default_throughput + def test_perf_compare_throughput_baseline_above_threshold( + self, get_latest_cpl_logs, read_baseline_file, _get_throughput ): read_baseline_file.return_value = ["1000"] - get_default_throughput.return_value = 504 + _get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -392,22 +398,24 @@ def test_compare_throughput_above_threshold( 0.05, ) - (below_tolerance, comment) = baselines.compare_throughput(case) + (below_tolerance, comment) = baselines.perf_compare_throughput_baseline( + case + ) assert not below_tolerance assert ( comment == "Error: TPUTCOMP: Computation time increase > 5% from baseline" ) - @mock.patch("CIME.baselines.get_default_throughput") + @mock.patch("CIME.baselines._get_throughput") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_throughput( - self, get_latest_cpl_logs, read_baseline_file, get_default_throughput + def test_perf_compare_throughput_baseline( + self, get_latest_cpl_logs, read_baseline_file, _get_throughput ): read_baseline_file.return_value = ["500"] - get_default_throughput.return_value = 504 + _get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -421,7 +429,9 @@ def test_compare_throughput( 0.05, ) - (below_tolerance, comment) = baselines.compare_throughput(case) + (below_tolerance, comment) = baselines.perf_compare_throughput_baseline( + case + ) assert below_tolerance assert ( @@ -432,7 +442,7 @@ def test_compare_throughput( @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_memory_no_baseline( + def test_perf_compare_memory_baseline_no_baseline( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): read_baseline_file.return_value = [] @@ -456,7 +466,7 @@ def test_compare_memory_no_baseline( 0.05, ) - (below_tolerance, comment) = baselines.compare_memory(case) + (below_tolerance, comment) = baselines.perf_compare_memory_baseline(case) assert below_tolerance assert ( @@ -467,7 +477,7 @@ def test_compare_memory_no_baseline( @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_memory_not_enough_samples( + def test_perf_compare_memory_baseline_not_enough_samples( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): read_baseline_file.return_value = ["1000.0"] @@ -487,7 +497,7 @@ def test_compare_memory_not_enough_samples( 0.05, ) - (below_tolerance, comment) = baselines.compare_memory(case) + (below_tolerance, comment) = baselines.perf_compare_memory_baseline(case) assert below_tolerance is None assert comment == "Found 2 memory usage samples, need atleast 4" @@ -495,7 +505,7 @@ def test_compare_memory_not_enough_samples( @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_memory_no_baseline_file( + def test_perf_compare_memory_baseline_no_baseline_file( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): read_baseline_file.side_effect = FileNotFoundError @@ -517,7 +527,7 @@ def test_compare_memory_no_baseline_file( 0.05, ) - (below_tolerance, comment) = baselines.compare_memory(case) + (below_tolerance, comment) = baselines.perf_compare_memory_baseline(case) assert below_tolerance is None assert comment == "Could not read baseline memory usage: " @@ -525,7 +535,7 @@ def test_compare_memory_no_baseline_file( @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_memory_no_tolerance( + def test_perf_compare_memory_baseline_no_tolerance( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): read_baseline_file.return_value = ["1000.0"] @@ -549,7 +559,7 @@ def test_compare_memory_no_tolerance( None, ) - (below_tolerance, comment) = baselines.compare_memory(case) + (below_tolerance, comment) = baselines.perf_compare_memory_baseline(case) assert below_tolerance assert ( @@ -560,7 +570,7 @@ def test_compare_memory_no_tolerance( @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_memory_above_threshold( + def test_perf_compare_memory_baseline_above_threshold( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): read_baseline_file.return_value = ["1000.0"] @@ -584,7 +594,7 @@ def test_compare_memory_above_threshold( 0.05, ) - (below_tolerance, comment) = baselines.compare_memory(case) + (below_tolerance, comment) = baselines.perf_compare_memory_baseline(case) assert not below_tolerance assert ( @@ -595,7 +605,7 @@ def test_compare_memory_above_threshold( @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_memory( + def test_perf_compare_memory_baseline( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): read_baseline_file.return_value = ["1000.0"] @@ -619,7 +629,7 @@ def test_compare_memory( 0.05, ) - (below_tolerance, comment) = baselines.compare_memory(case) + (below_tolerance, comment) = baselines.perf_compare_memory_baseline(case) assert below_tolerance assert ( diff --git a/CIME/tests/test_unit_system_tests.py b/CIME/tests/test_unit_system_tests.py index 2f9f7d9918b..f61fffee7cf 100644 --- a/CIME/tests/test_unit_system_tests.py +++ b/CIME/tests/test_unit_system_tests.py @@ -69,12 +69,12 @@ def create_mock_case(tempdir, idx=None, cpllog_data=None): class TestUnitSystemTests(unittest.TestCase): @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - @mock.patch("CIME.SystemTests.system_tests_common.get_default_mem_usage") + @mock.patch("CIME.SystemTests.system_tests_common._get_mem_usage") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak_runtime_error( self, get_latest_cpl_logs, - get_default_mem_usage, + _get_mem_usage, append_testlog, load_coupler_customization, ): @@ -82,7 +82,7 @@ def test_check_for_memleak_runtime_error( AttributeError ) - get_default_mem_usage.side_effect = RuntimeError + _get_mem_usage.side_effect = RuntimeError with tempfile.TemporaryDirectory() as tempdir: caseroot = Path(tempdir) / "caseroot" @@ -119,12 +119,12 @@ def test_check_for_memleak_runtime_error( @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - @mock.patch("CIME.SystemTests.system_tests_common.get_default_mem_usage") + @mock.patch("CIME.SystemTests.system_tests_common._get_mem_usage") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak_not_enough_samples( self, get_latest_cpl_logs, - get_default_mem_usage, + _get_mem_usage, append_testlog, load_coupler_customization, ): @@ -132,7 +132,7 @@ def test_check_for_memleak_not_enough_samples( AttributeError ) - get_default_mem_usage.return_value = [ + _get_mem_usage.return_value = [ (1, 1000.0), (2, 0), ] @@ -172,12 +172,12 @@ def test_check_for_memleak_not_enough_samples( @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - @mock.patch("CIME.SystemTests.system_tests_common.get_default_mem_usage") + @mock.patch("CIME.SystemTests.system_tests_common._get_mem_usage") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak_found( self, get_latest_cpl_logs, - get_default_mem_usage, + _get_mem_usage, append_testlog, load_coupler_customization, ): @@ -185,7 +185,7 @@ def test_check_for_memleak_found( AttributeError ) - get_default_mem_usage.return_value = [ + _get_mem_usage.return_value = [ (1, 1000.0), (2, 2000.0), (3, 3000.0), @@ -229,12 +229,12 @@ def test_check_for_memleak_found( @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - @mock.patch("CIME.SystemTests.system_tests_common.get_default_mem_usage") + @mock.patch("CIME.SystemTests.system_tests_common._get_mem_usage") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak( self, get_latest_cpl_logs, - get_default_mem_usage, + _get_mem_usage, append_testlog, load_coupler_customization, ): @@ -242,7 +242,7 @@ def test_check_for_memleak( AttributeError ) - get_default_mem_usage.return_value = [ + _get_mem_usage.return_value = [ (1, 3040.0), (2, 3002.0), (3, 3030.0), @@ -282,10 +282,10 @@ def test_check_for_memleak( append_testlog.assert_not_called() - @mock.patch("CIME.SystemTests.system_tests_common.compare_throughput") + @mock.patch("CIME.SystemTests.system_tests_common.perf_compare_throughput_baseline") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - def test_compare_throughput(self, append_testlog, compare_throughput): - compare_throughput.return_value = ( + def test_compare_throughput(self, append_testlog, perf_compare_throughput_baseline): + perf_compare_throughput_baseline.return_value = ( True, "TPUTCOMP: Computation time changed by 2.00% relative to baseline", ) @@ -312,10 +312,12 @@ def test_compare_throughput(self, append_testlog, compare_throughput): str(caseroot), ) - @mock.patch("CIME.SystemTests.system_tests_common.compare_throughput") + @mock.patch("CIME.SystemTests.system_tests_common.perf_compare_throughput_baseline") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - def test_compare_throughput_error_diff(self, append_testlog, compare_throughput): - compare_throughput.return_value = (None, "Error diff value") + def test_compare_throughput_error_diff( + self, append_testlog, perf_compare_throughput_baseline + ): + perf_compare_throughput_baseline.return_value = (None, "Error diff value") with tempfile.TemporaryDirectory() as tempdir: caseroot = Path(tempdir) / "caseroot" @@ -336,10 +338,12 @@ def test_compare_throughput_error_diff(self, append_testlog, compare_throughput) append_testlog.assert_not_called() - @mock.patch("CIME.SystemTests.system_tests_common.compare_throughput") + @mock.patch("CIME.SystemTests.system_tests_common.perf_compare_throughput_baseline") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - def test_compare_throughput_fail(self, append_testlog, compare_throughput): - compare_throughput.return_value = ( + def test_compare_throughput_fail( + self, append_testlog, perf_compare_throughput_baseline + ): + perf_compare_throughput_baseline.return_value = ( False, "Error: TPUTCOMP: Computation time increase > 5% from baseline", ) @@ -366,10 +370,10 @@ def test_compare_throughput_fail(self, append_testlog, compare_throughput): str(caseroot), ) - @mock.patch("CIME.SystemTests.system_tests_common.compare_memory") + @mock.patch("CIME.SystemTests.system_tests_common.perf_compare_memory_baseline") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - def test_compare_memory(self, append_testlog, compare_memory): - compare_memory.return_value = ( + def test_compare_memory(self, append_testlog, perf_compare_memory_baseline): + perf_compare_memory_baseline.return_value = ( True, "MEMCOMP: Memory usage highwater has changed by 2.00% relative to baseline", ) @@ -396,10 +400,12 @@ def test_compare_memory(self, append_testlog, compare_memory): str(caseroot), ) - @mock.patch("CIME.SystemTests.system_tests_common.compare_memory") + @mock.patch("CIME.SystemTests.system_tests_common.perf_compare_memory_baseline") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - def test_compare_memory_erorr_diff(self, append_testlog, compare_memory): - compare_memory.return_value = (None, "Error diff value") + def test_compare_memory_erorr_diff( + self, append_testlog, perf_compare_memory_baseline + ): + perf_compare_memory_baseline.return_value = (None, "Error diff value") with tempfile.TemporaryDirectory() as tempdir: caseroot = Path(tempdir) / "caseroot" @@ -420,10 +426,12 @@ def test_compare_memory_erorr_diff(self, append_testlog, compare_memory): append_testlog.assert_not_called() - @mock.patch("CIME.SystemTests.system_tests_common.compare_memory") + @mock.patch("CIME.SystemTests.system_tests_common.perf_compare_memory_baseline") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - def test_compare_memory_erorr_fail(self, append_testlog, compare_memory): - compare_memory.return_value = ( + def test_compare_memory_erorr_fail( + self, append_testlog, perf_compare_memory_baseline + ): + perf_compare_memory_baseline.return_value = ( False, "Error: Memory usage increase >5% from baseline's 1000.000000 to 1002.000000", ) From d76f407081b14599c4a5d28c8ec4c9b1d29f8a7f Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 13 Oct 2023 12:51:07 -0700 Subject: [PATCH 22/57] Fixes more naming --- CIME/SystemTests/system_tests_common.py | 4 +- CIME/baselines.py | 24 ++--- CIME/tests/test_unit_baselines.py | 136 ++++++++++++------------ CIME/tests/test_unit_system_tests.py | 24 ++--- 4 files changed, 94 insertions(+), 94 deletions(-) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index 7fca1010a60..7edfb5d30ac 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -28,7 +28,7 @@ from CIME.locked_files import LOCKED_DIR, lock_file, is_locked from CIME.baselines import ( get_latest_cpl_logs, - _get_mem_usage, + _perf_get_memory, perf_compare_memory_baseline, perf_compare_throughput_baseline, perf_write_baseline, @@ -780,7 +780,7 @@ def default_detect_memory_leak(case): for cpllog in latestcpllogs: try: - memlist = _get_mem_usage(case, cpllog) + memlist = _perf_get_memory(case, cpllog) except RuntimeError: return False, "insufficient data for memleak test" diff --git a/CIME/baselines.py b/CIME/baselines.py index a8c1aecd7ac..40a11a13db9 100644 --- a/CIME/baselines.py +++ b/CIME/baselines.py @@ -135,7 +135,7 @@ def perf_write_baseline(case, basegen_dir, throughput=True, memory=True): if throughput: try: - tput = get_throughput(case, config) + tput = perf_get_throughput(case, config) except RuntimeError: pass else: @@ -145,7 +145,7 @@ def perf_write_baseline(case, basegen_dir, throughput=True, memory=True): if memory: try: - mem = get_mem_usage(case, config) + mem = perf_get_memory(case, config) except RuntimeError: pass else: @@ -175,7 +175,7 @@ def load_coupler_customization(case): return Config.load(cpl_customize) -def get_throughput(case, config): +def perf_get_throughput(case, config): """ Gets the model throughput. @@ -194,9 +194,9 @@ def get_throughput(case, config): Model throughput. """ try: - tput = config.get_throughput(case) + tput = config.perf_get_throughput(case) except AttributeError: - tput = _get_throughput(case) + tput = _perf_get_throughput(case) if tput is None: raise RuntimeError("Could not get default throughput") from None @@ -206,7 +206,7 @@ def get_throughput(case, config): return tput -def get_mem_usage(case, config): +def perf_get_memory(case, config): """ Gets the model memory usage. @@ -225,9 +225,9 @@ def get_mem_usage(case, config): Model memory usage. """ try: - mem = config.get_mem_usage(case) + mem = config.perf_get_memory(case) except AttributeError: - mem = _get_mem_usage(case) + mem = _perf_get_memory(case) if mem is None: raise RuntimeError("Could not get default memory usage") from None @@ -252,7 +252,7 @@ def write_baseline_file(baseline_file, value): fd.write(value) -def _get_mem_usage(case, cpllog=None): +def _perf_get_memory(case, cpllog=None): """ Default function to retrieve memory usage from the coupler log. @@ -295,7 +295,7 @@ def _get_mem_usage(case, cpllog=None): return memlist -def _get_throughput(case): +def _perf_get_throughput(case): """ Default function to retrieve throughput from the coupler log. @@ -455,7 +455,7 @@ def _perf_compare_throughput_baseline(case, baseline, tolerance): comment : str provides explanation from comparison. """ - current = _get_throughput(case) + current = _perf_get_throughput(case) try: # default baseline is stored as single float @@ -508,7 +508,7 @@ def _perf_compare_memory_baseline(case, baseline, tolerance): provides explanation from comparison. """ try: - current = _get_mem_usage(case) + current = _perf_get_memory(case) except RuntimeError as e: return None, str(e) else: diff --git a/CIME/tests/test_unit_baselines.py b/CIME/tests/test_unit_baselines.py index 0b357e51253..dac5bc9ef10 100644 --- a/CIME/tests/test_unit_baselines.py +++ b/CIME/tests/test_unit_baselines.py @@ -28,79 +28,79 @@ def create_mock_case(tempdir, get_latest_cpl_logs=None): class TestUnitBaseline(unittest.TestCase): - @mock.patch("CIME.baselines._get_mem_usage") - def test_get_mem_usage_default_no_value(self, _get_mem_usage): - _get_mem_usage.return_value = None + @mock.patch("CIME.baselines._perf_get_memory") + def test_perf_get_memory_default_no_value(self, _perf_get_memory): + _perf_get_memory.return_value = None case = mock.MagicMock() config = mock.MagicMock() - config.get_mem_usage.side_effect = AttributeError + config.perf_get_memory.side_effect = AttributeError with self.assertRaises(RuntimeError): - baselines.get_mem_usage(case, config) + baselines.perf_get_memory(case, config) - @mock.patch("CIME.baselines._get_mem_usage") - def test_get_mem_usage_default(self, _get_mem_usage): - _get_mem_usage.return_value = [(1, 1000)] + @mock.patch("CIME.baselines._perf_get_memory") + def test_perf_get_memory_default(self, _perf_get_memory): + _perf_get_memory.return_value = [(1, 1000)] case = mock.MagicMock() config = mock.MagicMock() - config.get_mem_usage.side_effect = AttributeError + config.perf_get_memory.side_effect = AttributeError - mem = baselines.get_mem_usage(case, config) + mem = baselines.perf_get_memory(case, config) assert mem == "1000" - def test_get_mem_usage(self): + def test_perf_get_memory(self): case = mock.MagicMock() config = mock.MagicMock() - config.get_mem_usage.return_value = "1000" + config.perf_get_memory.return_value = "1000" - mem = baselines.get_mem_usage(case, config) + mem = baselines.perf_get_memory(case, config) assert mem == "1000" - @mock.patch("CIME.baselines._get_throughput") - def test_get_throughput_default_no_value(self, _get_throughput): - _get_throughput.return_value = None + @mock.patch("CIME.baselines._perf_get_throughput") + def test_perf_get_throughput_default_no_value(self, _perf_get_throughput): + _perf_get_throughput.return_value = None case = mock.MagicMock() config = mock.MagicMock() - config.get_throughput.side_effect = AttributeError + config.perf_get_throughput.side_effect = AttributeError with self.assertRaises(RuntimeError): - baselines.get_throughput(case, config) + baselines.perf_get_throughput(case, config) - @mock.patch("CIME.baselines._get_throughput") - def test_get_throughput_default(self, _get_throughput): - _get_throughput.return_value = 100 + @mock.patch("CIME.baselines._perf_get_throughput") + def test_perf_get_throughput_default(self, _perf_get_throughput): + _perf_get_throughput.return_value = 100 case = mock.MagicMock() config = mock.MagicMock() - config.get_throughput.side_effect = AttributeError + config.perf_get_throughput.side_effect = AttributeError - tput = baselines.get_throughput(case, config) + tput = baselines.perf_get_throughput(case, config) assert tput == "100" - def test_get_throughput(self): + def test_perf_get_throughput(self): case = mock.MagicMock() config = mock.MagicMock() - config.get_throughput.return_value = "100" + config.perf_get_throughput.return_value = "100" - tput = baselines.get_throughput(case, config) + tput = baselines.perf_get_throughput(case, config) assert tput == "100" @@ -186,49 +186,49 @@ def test_write_baseline_file(self): @mock.patch("CIME.baselines.get_cpl_throughput") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test__get_throughput(self, get_latest_cpl_logs, get_cpl_throughput): + def test__perf_get_throughput(self, get_latest_cpl_logs, get_cpl_throughput): get_cpl_throughput.side_effect = FileNotFoundError() with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) - tput = baselines._get_throughput(case) + tput = baselines._perf_get_throughput(case) assert tput == None @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test__get_mem_usage_override(self, get_latest_cpl_logs, get_cpl_mem_usage): + def test__perf_get_memory_override(self, get_latest_cpl_logs, get_cpl_mem_usage): get_cpl_mem_usage.side_effect = FileNotFoundError() with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) - mem = baselines._get_mem_usage(case, "/tmp/override") + mem = baselines._perf_get_memory(case, "/tmp/override") assert mem == None @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test__get_mem_usage(self, get_latest_cpl_logs, get_cpl_mem_usage): + def test__perf_get_memory(self, get_latest_cpl_logs, get_cpl_mem_usage): get_cpl_mem_usage.side_effect = FileNotFoundError() with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) - mem = baselines._get_mem_usage(case) + mem = baselines._perf_get_memory(case) assert mem == None @mock.patch("CIME.baselines.write_baseline_file") - @mock.patch("CIME.baselines.get_mem_usage") - @mock.patch("CIME.baselines.get_throughput") + @mock.patch("CIME.baselines.perf_get_memory") + @mock.patch("CIME.baselines.perf_get_throughput") def test_write_baseline_skip( - self, get_throughput, get_mem_usage, write_baseline_file + self, perf_get_throughput, perf_get_memory, write_baseline_file ): - get_throughput.return_value = "100" + perf_get_throughput.return_value = "100" - get_mem_usage.return_value = "1000" + perf_get_memory.return_value = "1000" with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir) @@ -240,58 +240,58 @@ def test_write_baseline_skip( False, ) - get_throughput.assert_not_called() - get_mem_usage.assert_not_called() + perf_get_throughput.assert_not_called() + perf_get_memory.assert_not_called() write_baseline_file.assert_not_called() @mock.patch("CIME.baselines.write_baseline_file") - @mock.patch("CIME.baselines.get_mem_usage") - @mock.patch("CIME.baselines.get_throughput") + @mock.patch("CIME.baselines.perf_get_memory") + @mock.patch("CIME.baselines.perf_get_throughput") def test_write_baseline_runtimeerror( - self, get_throughput, get_mem_usage, write_baseline_file + self, perf_get_throughput, perf_get_memory, write_baseline_file ): - get_throughput.side_effect = RuntimeError + perf_get_throughput.side_effect = RuntimeError - get_mem_usage.side_effect = RuntimeError + perf_get_memory.side_effect = RuntimeError with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir) baselines.perf_write_baseline(case, baseline_root) - get_throughput.assert_called() - get_mem_usage.assert_called() + perf_get_throughput.assert_called() + perf_get_memory.assert_called() write_baseline_file.assert_not_called() @mock.patch("CIME.baselines.write_baseline_file") - @mock.patch("CIME.baselines.get_mem_usage") - @mock.patch("CIME.baselines.get_throughput") + @mock.patch("CIME.baselines.perf_get_memory") + @mock.patch("CIME.baselines.perf_get_throughput") def test_perf_write_baseline( - self, get_throughput, get_mem_usage, write_baseline_file + self, perf_get_throughput, perf_get_memory, write_baseline_file ): - get_throughput.return_value = "100" + perf_get_throughput.return_value = "100" - get_mem_usage.return_value = "1000" + perf_get_memory.return_value = "1000" with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir) baselines.perf_write_baseline(case, baseline_root) - get_throughput.assert_called() - get_mem_usage.assert_called() + perf_get_throughput.assert_called() + perf_get_memory.assert_called() write_baseline_file.assert_any_call(str(baseline_root / "cpl-tput.log"), "100") write_baseline_file.assert_any_call(str(baseline_root / "cpl-mem.log"), "1000") - @mock.patch("CIME.baselines._get_throughput") + @mock.patch("CIME.baselines._perf_get_throughput") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_perf_compare_throughput_baseline_no_baseline_file( - self, get_latest_cpl_logs, read_baseline_file, _get_throughput + self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): read_baseline_file.side_effect = FileNotFoundError - _get_throughput.return_value = 504 + _perf_get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -310,15 +310,15 @@ def test_perf_compare_throughput_baseline_no_baseline_file( assert below_tolerance is None assert comment == "Could not read baseline throughput file: " - @mock.patch("CIME.baselines._get_throughput") + @mock.patch("CIME.baselines._perf_get_throughput") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_perf_compare_throughput_baseline_no_baseline( - self, get_latest_cpl_logs, read_baseline_file, _get_throughput + self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): read_baseline_file.return_value = [] - _get_throughput.return_value = 504 + _perf_get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -342,17 +342,17 @@ def test_perf_compare_throughput_baseline_no_baseline( == "Could not compare throughput to baseline, as basline had no value." ) - @mock.patch("CIME.baselines._get_throughput") + @mock.patch("CIME.baselines._perf_get_throughput") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_perf_compare_throughput_baseline_no_tolerance( - self, get_latest_cpl_logs, read_baseline_file, _get_throughput + self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): read_baseline_file.return_value = [ "500", ] - _get_throughput.return_value = 504 + _perf_get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -376,15 +376,15 @@ def test_perf_compare_throughput_baseline_no_tolerance( == "TPUTCOMP: Computation time changed by -0.80% relative to baseline" ) - @mock.patch("CIME.baselines._get_throughput") + @mock.patch("CIME.baselines._perf_get_throughput") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_perf_compare_throughput_baseline_above_threshold( - self, get_latest_cpl_logs, read_baseline_file, _get_throughput + self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): read_baseline_file.return_value = ["1000"] - _get_throughput.return_value = 504 + _perf_get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -407,15 +407,15 @@ def test_perf_compare_throughput_baseline_above_threshold( comment == "Error: TPUTCOMP: Computation time increase > 5% from baseline" ) - @mock.patch("CIME.baselines._get_throughput") + @mock.patch("CIME.baselines._perf_get_throughput") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_perf_compare_throughput_baseline( - self, get_latest_cpl_logs, read_baseline_file, _get_throughput + self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): read_baseline_file.return_value = ["500"] - _get_throughput.return_value = 504 + _perf_get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) diff --git a/CIME/tests/test_unit_system_tests.py b/CIME/tests/test_unit_system_tests.py index f61fffee7cf..4a206fb454e 100644 --- a/CIME/tests/test_unit_system_tests.py +++ b/CIME/tests/test_unit_system_tests.py @@ -69,12 +69,12 @@ def create_mock_case(tempdir, idx=None, cpllog_data=None): class TestUnitSystemTests(unittest.TestCase): @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - @mock.patch("CIME.SystemTests.system_tests_common._get_mem_usage") + @mock.patch("CIME.SystemTests.system_tests_common._perf_get_memory") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak_runtime_error( self, get_latest_cpl_logs, - _get_mem_usage, + _perf_get_memory, append_testlog, load_coupler_customization, ): @@ -82,7 +82,7 @@ def test_check_for_memleak_runtime_error( AttributeError ) - _get_mem_usage.side_effect = RuntimeError + _perf_get_memory.side_effect = RuntimeError with tempfile.TemporaryDirectory() as tempdir: caseroot = Path(tempdir) / "caseroot" @@ -119,12 +119,12 @@ def test_check_for_memleak_runtime_error( @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - @mock.patch("CIME.SystemTests.system_tests_common._get_mem_usage") + @mock.patch("CIME.SystemTests.system_tests_common._perf_get_memory") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak_not_enough_samples( self, get_latest_cpl_logs, - _get_mem_usage, + _perf_get_memory, append_testlog, load_coupler_customization, ): @@ -132,7 +132,7 @@ def test_check_for_memleak_not_enough_samples( AttributeError ) - _get_mem_usage.return_value = [ + _perf_get_memory.return_value = [ (1, 1000.0), (2, 0), ] @@ -172,12 +172,12 @@ def test_check_for_memleak_not_enough_samples( @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - @mock.patch("CIME.SystemTests.system_tests_common._get_mem_usage") + @mock.patch("CIME.SystemTests.system_tests_common._perf_get_memory") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak_found( self, get_latest_cpl_logs, - _get_mem_usage, + _perf_get_memory, append_testlog, load_coupler_customization, ): @@ -185,7 +185,7 @@ def test_check_for_memleak_found( AttributeError ) - _get_mem_usage.return_value = [ + _perf_get_memory.return_value = [ (1, 1000.0), (2, 2000.0), (3, 3000.0), @@ -229,12 +229,12 @@ def test_check_for_memleak_found( @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - @mock.patch("CIME.SystemTests.system_tests_common._get_mem_usage") + @mock.patch("CIME.SystemTests.system_tests_common._perf_get_memory") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak( self, get_latest_cpl_logs, - _get_mem_usage, + _perf_get_memory, append_testlog, load_coupler_customization, ): @@ -242,7 +242,7 @@ def test_check_for_memleak( AttributeError ) - _get_mem_usage.return_value = [ + _perf_get_memory.return_value = [ (1, 3040.0), (2, 3002.0), (3, 3030.0), From 13e0e8b65d9281c8ee4400484fda03696dae2348 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 13 Oct 2023 13:10:58 -0700 Subject: [PATCH 23/57] Fixes striping newlines when reading baseline files --- CIME/baselines.py | 2 +- CIME/tests/test_unit_baselines.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CIME/baselines.py b/CIME/baselines.py index 40a11a13db9..2a3fe7369c6 100644 --- a/CIME/baselines.py +++ b/CIME/baselines.py @@ -428,7 +428,7 @@ def read_baseline_file(baseline_file): Value stored in baseline file without comments. """ with open(baseline_file) as fd: - lines = [x for x in fd.readlines() if not x.startswith("#")] + lines = [x.strip() for x in fd.readlines() if not x.startswith("#")] return lines diff --git a/CIME/tests/test_unit_baselines.py b/CIME/tests/test_unit_baselines.py index dac5bc9ef10..8382339bdc7 100644 --- a/CIME/tests/test_unit_baselines.py +++ b/CIME/tests/test_unit_baselines.py @@ -159,7 +159,7 @@ def test_read_baseline_file_multi_line(self): baseline = baselines.read_baseline_file("/tmp/cpl-mem.log") mock_file.assert_called_with("/tmp/cpl-mem.log") - assert baseline == ["1000.0\n", "2000.0\n"] + assert baseline == ["1000.0", "2000.0"] def test_read_baseline_file_content(self): with mock.patch( From 7780b1fee43cb1892067bd917b2e786192f03d45 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 13 Oct 2023 13:11:42 -0700 Subject: [PATCH 24/57] Fixes passing correct paths to memory/throughput bless functions --- CIME/bless_test_results.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index c2ba6fbb1c9..5802d685b71 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -397,8 +397,8 @@ def bless_test_results( success, reason = bless_throughput( case, test_name, - baseline_root, - baseline_name, + baseline_root_resolved, + baseline_name_resolved, report_only, force, ) @@ -410,8 +410,8 @@ def bless_test_results( success, reason = bless_memory( case, test_name, - baseline_root, - baseline_name, + baseline_root_resolved, + baseline_name_resolved, report_only, force, ) From 52ca0a1c791e14c3756d6e6d32f52628a37f2f72 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 13 Oct 2023 13:18:57 -0700 Subject: [PATCH 25/57] Renames memory leak functions --- CIME/SystemTests/system_tests_common.py | 6 +++--- CIME/tests/test_unit_system_tests.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index 7edfb5d30ac..e63c4c6d502 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -636,9 +636,9 @@ def _check_for_memleak(self): with self._test_status: try: - memleak, comment = config.detect_memory_leak(self._case) + memleak, comment = config.perf_check_for_memory_leak(self._case) except AttributeError: - memleak, comment = default_detect_memory_leak(self._case) + memleak, comment = perf_check_for_memory_leak(self._case) if memleak: append_testlog(comment, self._orig_caseroot) @@ -772,7 +772,7 @@ def _generate_baseline(self): perf_write_baseline(self._case, basegen_dir, cpllog) -def default_detect_memory_leak(case): +def perf_check_for_memory_leak(case): leak = False comment = "" diff --git a/CIME/tests/test_unit_system_tests.py b/CIME/tests/test_unit_system_tests.py index 4a206fb454e..99486178f36 100644 --- a/CIME/tests/test_unit_system_tests.py +++ b/CIME/tests/test_unit_system_tests.py @@ -78,7 +78,7 @@ def test_check_for_memleak_runtime_error( append_testlog, load_coupler_customization, ): - load_coupler_customization.return_value.detect_memory_leak.side_effect = ( + load_coupler_customization.return_value.perf_check_for_memory_leak.side_effect = ( AttributeError ) @@ -128,7 +128,7 @@ def test_check_for_memleak_not_enough_samples( append_testlog, load_coupler_customization, ): - load_coupler_customization.return_value.detect_memory_leak.side_effect = ( + load_coupler_customization.return_value.perf_check_for_memory_leak.side_effect = ( AttributeError ) @@ -181,7 +181,7 @@ def test_check_for_memleak_found( append_testlog, load_coupler_customization, ): - load_coupler_customization.return_value.detect_memory_leak.side_effect = ( + load_coupler_customization.return_value.perf_check_for_memory_leak.side_effect = ( AttributeError ) @@ -238,7 +238,7 @@ def test_check_for_memleak( append_testlog, load_coupler_customization, ): - load_coupler_customization.return_value.detect_memory_leak.side_effect = ( + load_coupler_customization.return_value.perf_check_for_memory_leak.side_effect = ( AttributeError ) From 69f7e367bc8e20e170bcce3b5f65df2f7082e4f0 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 13 Oct 2023 13:36:39 -0700 Subject: [PATCH 26/57] Fixes black formatting --- CIME/SystemTests/system_tests_common.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index e63c4c6d502..233de1e2660 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -634,11 +634,18 @@ def _check_for_memleak(self): """ config = load_coupler_customization(self._case) + # default to 0.1 + tolerance = self._case.get_value("TEST_MEMLEAK_TOLERANCE") or 0.1 + + expect(tolerance > 0.0, "Bad value for memleak tolerance in test") + with self._test_status: try: - memleak, comment = config.perf_check_for_memory_leak(self._case) + memleak, comment = config.perf_check_for_memory_leak( + self._case, tolerance + ) except AttributeError: - memleak, comment = perf_check_for_memory_leak(self._case) + memleak, comment = perf_check_for_memory_leak(self._case, tolerance) if memleak: append_testlog(comment, self._orig_caseroot) @@ -772,7 +779,7 @@ def _generate_baseline(self): perf_write_baseline(self._case, basegen_dir, cpllog) -def perf_check_for_memory_leak(case): +def perf_check_for_memory_leak(case, tolerance): leak = False comment = "" @@ -791,11 +798,6 @@ def perf_check_for_memory_leak(case): memdiff = -1 if originalmem <= 0 else (finalmem - originalmem) / originalmem - # default to 0.1 - tolerance = case.get_value("TEST_MEMLEAK_TOLERANCE") or 0.1 - - expect(tolerance > 0.0, "Bad value for memleak tolerance in test") - if memdiff < 0: leak = False comment = "data for memleak test is insufficient" From ccb3b4ea59e854fb26d31499492f170bd9766d02 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 13 Oct 2023 15:51:38 -0700 Subject: [PATCH 27/57] Fixes baseline module organization --- CIME/SystemTests/system_tests_common.py | 2 +- CIME/baselines/__init__.py | 0 .../performance.py} | 0 CIME/bless_test_results.py | 2 +- ....py => test_unit_baselines_performance.py} | 178 +++++++++--------- 5 files changed, 91 insertions(+), 91 deletions(-) create mode 100644 CIME/baselines/__init__.py rename CIME/{baselines.py => baselines/performance.py} (100%) rename CIME/tests/{test_unit_baselines.py => test_unit_baselines_performance.py} (74%) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index 233de1e2660..a05616882df 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -26,7 +26,7 @@ from CIME.config import Config from CIME.provenance import save_test_time, get_test_success from CIME.locked_files import LOCKED_DIR, lock_file, is_locked -from CIME.baselines import ( +from CIME.baselines.performance import ( get_latest_cpl_logs, _perf_get_memory, perf_compare_memory_baseline, diff --git a/CIME/baselines/__init__.py b/CIME/baselines/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/CIME/baselines.py b/CIME/baselines/performance.py similarity index 100% rename from CIME/baselines.py rename to CIME/baselines/performance.py diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index 5802d685b71..0f09b91eb0b 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -11,7 +11,7 @@ from CIME.hist_utils import generate_baseline, compare_baseline from CIME.case import Case from CIME.test_utils import get_test_status_files -from CIME.baselines import ( +from CIME.baselines.performance import ( perf_compare_throughput_baseline, perf_compare_memory_baseline, perf_write_baseline, diff --git a/CIME/tests/test_unit_baselines.py b/CIME/tests/test_unit_baselines_performance.py similarity index 74% rename from CIME/tests/test_unit_baselines.py rename to CIME/tests/test_unit_baselines_performance.py index 8382339bdc7..a6eebbb2bbb 100644 --- a/CIME/tests/test_unit_baselines.py +++ b/CIME/tests/test_unit_baselines_performance.py @@ -6,7 +6,7 @@ from unittest import mock from pathlib import Path -from CIME import baselines +from CIME.baselines import performance from CIME.tests.test_unit_system_tests import CPLLOG @@ -27,8 +27,8 @@ def create_mock_case(tempdir, get_latest_cpl_logs=None): return case, caseroot, rundir, baseline_root -class TestUnitBaseline(unittest.TestCase): - @mock.patch("CIME.baselines._perf_get_memory") +class TestUnitBaselinesPerformance(unittest.TestCase): + @mock.patch("CIME.baselines.performance._perf_get_memory") def test_perf_get_memory_default_no_value(self, _perf_get_memory): _perf_get_memory.return_value = None @@ -39,9 +39,9 @@ def test_perf_get_memory_default_no_value(self, _perf_get_memory): config.perf_get_memory.side_effect = AttributeError with self.assertRaises(RuntimeError): - baselines.perf_get_memory(case, config) + performance.perf_get_memory(case, config) - @mock.patch("CIME.baselines._perf_get_memory") + @mock.patch("CIME.baselines.performance._perf_get_memory") def test_perf_get_memory_default(self, _perf_get_memory): _perf_get_memory.return_value = [(1, 1000)] @@ -51,7 +51,7 @@ def test_perf_get_memory_default(self, _perf_get_memory): config.perf_get_memory.side_effect = AttributeError - mem = baselines.perf_get_memory(case, config) + mem = performance.perf_get_memory(case, config) assert mem == "1000" @@ -62,11 +62,11 @@ def test_perf_get_memory(self): config.perf_get_memory.return_value = "1000" - mem = baselines.perf_get_memory(case, config) + mem = performance.perf_get_memory(case, config) assert mem == "1000" - @mock.patch("CIME.baselines._perf_get_throughput") + @mock.patch("CIME.baselines.performance._perf_get_throughput") def test_perf_get_throughput_default_no_value(self, _perf_get_throughput): _perf_get_throughput.return_value = None @@ -77,9 +77,9 @@ def test_perf_get_throughput_default_no_value(self, _perf_get_throughput): config.perf_get_throughput.side_effect = AttributeError with self.assertRaises(RuntimeError): - baselines.perf_get_throughput(case, config) + performance.perf_get_throughput(case, config) - @mock.patch("CIME.baselines._perf_get_throughput") + @mock.patch("CIME.baselines.performance._perf_get_throughput") def test_perf_get_throughput_default(self, _perf_get_throughput): _perf_get_throughput.return_value = 100 @@ -89,7 +89,7 @@ def test_perf_get_throughput_default(self, _perf_get_throughput): config.perf_get_throughput.side_effect = AttributeError - tput = baselines.perf_get_throughput(case, config) + tput = performance.perf_get_throughput(case, config) assert tput == "100" @@ -100,12 +100,12 @@ def test_perf_get_throughput(self): config.perf_get_throughput.return_value = "100" - tput = baselines.perf_get_throughput(case, config) + tput = performance.perf_get_throughput(case, config) assert tput == "100" def test_get_cpl_throughput_no_file(self): - throughput = baselines.get_cpl_throughput("/tmp/cpl.log") + throughput = performance.get_cpl_throughput("/tmp/cpl.log") assert throughput is None @@ -116,7 +116,7 @@ def test_get_cpl_throughput(self): with gzip.open(cpl_log_path, "w") as fd: fd.write(CPLLOG.encode("utf-8")) - throughput = baselines.get_cpl_throughput(str(cpl_log_path)) + throughput = performance.get_cpl_throughput(str(cpl_log_path)) assert throughput == 719.635 @@ -127,7 +127,7 @@ def test_get_cpl_mem_usage_gz(self): with gzip.open(cpl_log_path, "w") as fd: fd.write(CPLLOG.encode("utf-8")) - mem_usage = baselines.get_cpl_mem_usage(str(cpl_log_path)) + mem_usage = performance.get_cpl_mem_usage(str(cpl_log_path)) assert mem_usage == [ (10102.0, 1673.89), @@ -136,14 +136,14 @@ def test_get_cpl_mem_usage_gz(self): (10105.0, 1673.89), ] - @mock.patch("CIME.baselines.os.path.isfile") + @mock.patch("CIME.baselines.performance.os.path.isfile") def test_get_cpl_mem_usage(self, isfile): isfile.return_value = True with mock.patch( "builtins.open", mock.mock_open(read_data=CPLLOG.encode("utf-8")) ) as mock_file: - mem_usage = baselines.get_cpl_mem_usage("/tmp/cpl.log") + mem_usage = performance.get_cpl_mem_usage("/tmp/cpl.log") assert mem_usage == [ (10102.0, 1673.89), @@ -156,7 +156,7 @@ def test_read_baseline_file_multi_line(self): with mock.patch( "builtins.open", mock.mock_open(read_data="1000.0\n2000.0\n") ) as mock_file: - baseline = baselines.read_baseline_file("/tmp/cpl-mem.log") + baseline = performance.read_baseline_file("/tmp/cpl-mem.log") mock_file.assert_called_with("/tmp/cpl-mem.log") assert baseline == ["1000.0", "2000.0"] @@ -165,64 +165,64 @@ def test_read_baseline_file_content(self): with mock.patch( "builtins.open", mock.mock_open(read_data="1000.0") ) as mock_file: - baseline = baselines.read_baseline_file("/tmp/cpl-mem.log") + baseline = performance.read_baseline_file("/tmp/cpl-mem.log") mock_file.assert_called_with("/tmp/cpl-mem.log") assert baseline == ["1000.0"] def test_read_baseline_file(self): with mock.patch("builtins.open", mock.mock_open(read_data="")) as mock_file: - baseline = baselines.read_baseline_file("/tmp/cpl-mem.log") + baseline = performance.read_baseline_file("/tmp/cpl-mem.log") mock_file.assert_called_with("/tmp/cpl-mem.log") assert baseline == [] def test_write_baseline_file(self): with mock.patch("builtins.open", mock.mock_open()) as mock_file: - baselines.write_baseline_file("/tmp/cpl-tput.log", "1000") + performance.write_baseline_file("/tmp/cpl-tput.log", "1000") mock_file.assert_called_with("/tmp/cpl-tput.log", "w") mock_file.return_value.write.assert_called_with("1000") - @mock.patch("CIME.baselines.get_cpl_throughput") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance.get_cpl_throughput") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test__perf_get_throughput(self, get_latest_cpl_logs, get_cpl_throughput): get_cpl_throughput.side_effect = FileNotFoundError() with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) - tput = baselines._perf_get_throughput(case) + tput = performance._perf_get_throughput(case) assert tput == None - @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test__perf_get_memory_override(self, get_latest_cpl_logs, get_cpl_mem_usage): get_cpl_mem_usage.side_effect = FileNotFoundError() with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) - mem = baselines._perf_get_memory(case, "/tmp/override") + mem = performance._perf_get_memory(case, "/tmp/override") assert mem == None - @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test__perf_get_memory(self, get_latest_cpl_logs, get_cpl_mem_usage): get_cpl_mem_usage.side_effect = FileNotFoundError() with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) - mem = baselines._perf_get_memory(case) + mem = performance._perf_get_memory(case) assert mem == None - @mock.patch("CIME.baselines.write_baseline_file") - @mock.patch("CIME.baselines.perf_get_memory") - @mock.patch("CIME.baselines.perf_get_throughput") + @mock.patch("CIME.baselines.performance.write_baseline_file") + @mock.patch("CIME.baselines.performance.perf_get_memory") + @mock.patch("CIME.baselines.performance.perf_get_throughput") def test_write_baseline_skip( self, perf_get_throughput, perf_get_memory, write_baseline_file ): @@ -233,7 +233,7 @@ def test_write_baseline_skip( with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir) - baselines.perf_write_baseline( + performance.perf_write_baseline( case, baseline_root, False, @@ -244,9 +244,9 @@ def test_write_baseline_skip( perf_get_memory.assert_not_called() write_baseline_file.assert_not_called() - @mock.patch("CIME.baselines.write_baseline_file") - @mock.patch("CIME.baselines.perf_get_memory") - @mock.patch("CIME.baselines.perf_get_throughput") + @mock.patch("CIME.baselines.performance.write_baseline_file") + @mock.patch("CIME.baselines.performance.perf_get_memory") + @mock.patch("CIME.baselines.performance.perf_get_throughput") def test_write_baseline_runtimeerror( self, perf_get_throughput, perf_get_memory, write_baseline_file ): @@ -257,15 +257,15 @@ def test_write_baseline_runtimeerror( with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir) - baselines.perf_write_baseline(case, baseline_root) + performance.perf_write_baseline(case, baseline_root) perf_get_throughput.assert_called() perf_get_memory.assert_called() write_baseline_file.assert_not_called() - @mock.patch("CIME.baselines.write_baseline_file") - @mock.patch("CIME.baselines.perf_get_memory") - @mock.patch("CIME.baselines.perf_get_throughput") + @mock.patch("CIME.baselines.performance.write_baseline_file") + @mock.patch("CIME.baselines.performance.perf_get_memory") + @mock.patch("CIME.baselines.performance.perf_get_throughput") def test_perf_write_baseline( self, perf_get_throughput, perf_get_memory, write_baseline_file ): @@ -276,16 +276,16 @@ def test_perf_write_baseline( with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir) - baselines.perf_write_baseline(case, baseline_root) + performance.perf_write_baseline(case, baseline_root) perf_get_throughput.assert_called() perf_get_memory.assert_called() write_baseline_file.assert_any_call(str(baseline_root / "cpl-tput.log"), "100") write_baseline_file.assert_any_call(str(baseline_root / "cpl-mem.log"), "1000") - @mock.patch("CIME.baselines._perf_get_throughput") - @mock.patch("CIME.baselines.read_baseline_file") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance._perf_get_throughput") + @mock.patch("CIME.baselines.performance.read_baseline_file") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test_perf_compare_throughput_baseline_no_baseline_file( self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): @@ -303,16 +303,16 @@ def test_perf_compare_throughput_baseline_no_baseline_file( 0.05, ) - (below_tolerance, comment) = baselines.perf_compare_throughput_baseline( + (below_tolerance, comment) = performance.perf_compare_throughput_baseline( case ) assert below_tolerance is None assert comment == "Could not read baseline throughput file: " - @mock.patch("CIME.baselines._perf_get_throughput") - @mock.patch("CIME.baselines.read_baseline_file") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance._perf_get_throughput") + @mock.patch("CIME.baselines.performance.read_baseline_file") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test_perf_compare_throughput_baseline_no_baseline( self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): @@ -332,7 +332,7 @@ def test_perf_compare_throughput_baseline_no_baseline( 0.05, ) - (below_tolerance, comment) = baselines.perf_compare_throughput_baseline( + (below_tolerance, comment) = performance.perf_compare_throughput_baseline( case ) @@ -342,9 +342,9 @@ def test_perf_compare_throughput_baseline_no_baseline( == "Could not compare throughput to baseline, as basline had no value." ) - @mock.patch("CIME.baselines._perf_get_throughput") - @mock.patch("CIME.baselines.read_baseline_file") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance._perf_get_throughput") + @mock.patch("CIME.baselines.performance.read_baseline_file") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test_perf_compare_throughput_baseline_no_tolerance( self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): @@ -366,7 +366,7 @@ def test_perf_compare_throughput_baseline_no_tolerance( None, ) - (below_tolerance, comment) = baselines.perf_compare_throughput_baseline( + (below_tolerance, comment) = performance.perf_compare_throughput_baseline( case ) @@ -376,9 +376,9 @@ def test_perf_compare_throughput_baseline_no_tolerance( == "TPUTCOMP: Computation time changed by -0.80% relative to baseline" ) - @mock.patch("CIME.baselines._perf_get_throughput") - @mock.patch("CIME.baselines.read_baseline_file") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance._perf_get_throughput") + @mock.patch("CIME.baselines.performance.read_baseline_file") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test_perf_compare_throughput_baseline_above_threshold( self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): @@ -398,7 +398,7 @@ def test_perf_compare_throughput_baseline_above_threshold( 0.05, ) - (below_tolerance, comment) = baselines.perf_compare_throughput_baseline( + (below_tolerance, comment) = performance.perf_compare_throughput_baseline( case ) @@ -407,9 +407,9 @@ def test_perf_compare_throughput_baseline_above_threshold( comment == "Error: TPUTCOMP: Computation time increase > 5% from baseline" ) - @mock.patch("CIME.baselines._perf_get_throughput") - @mock.patch("CIME.baselines.read_baseline_file") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance._perf_get_throughput") + @mock.patch("CIME.baselines.performance.read_baseline_file") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test_perf_compare_throughput_baseline( self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): @@ -429,7 +429,7 @@ def test_perf_compare_throughput_baseline( 0.05, ) - (below_tolerance, comment) = baselines.perf_compare_throughput_baseline( + (below_tolerance, comment) = performance.perf_compare_throughput_baseline( case ) @@ -439,9 +439,9 @@ def test_perf_compare_throughput_baseline( == "TPUTCOMP: Computation time changed by -0.80% relative to baseline" ) - @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.read_baseline_file") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") + @mock.patch("CIME.baselines.performance.read_baseline_file") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test_perf_compare_memory_baseline_no_baseline( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): @@ -466,7 +466,7 @@ def test_perf_compare_memory_baseline_no_baseline( 0.05, ) - (below_tolerance, comment) = baselines.perf_compare_memory_baseline(case) + (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) assert below_tolerance assert ( @@ -474,9 +474,9 @@ def test_perf_compare_memory_baseline_no_baseline( == "MEMCOMP: Memory usage highwater has changed by 0.00% relative to baseline" ) - @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.read_baseline_file") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") + @mock.patch("CIME.baselines.performance.read_baseline_file") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test_perf_compare_memory_baseline_not_enough_samples( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): @@ -497,14 +497,14 @@ def test_perf_compare_memory_baseline_not_enough_samples( 0.05, ) - (below_tolerance, comment) = baselines.perf_compare_memory_baseline(case) + (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) assert below_tolerance is None assert comment == "Found 2 memory usage samples, need atleast 4" - @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.read_baseline_file") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") + @mock.patch("CIME.baselines.performance.read_baseline_file") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test_perf_compare_memory_baseline_no_baseline_file( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): @@ -527,14 +527,14 @@ def test_perf_compare_memory_baseline_no_baseline_file( 0.05, ) - (below_tolerance, comment) = baselines.perf_compare_memory_baseline(case) + (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) assert below_tolerance is None assert comment == "Could not read baseline memory usage: " - @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.read_baseline_file") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") + @mock.patch("CIME.baselines.performance.read_baseline_file") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test_perf_compare_memory_baseline_no_tolerance( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): @@ -559,7 +559,7 @@ def test_perf_compare_memory_baseline_no_tolerance( None, ) - (below_tolerance, comment) = baselines.perf_compare_memory_baseline(case) + (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) assert below_tolerance assert ( @@ -567,9 +567,9 @@ def test_perf_compare_memory_baseline_no_tolerance( == "MEMCOMP: Memory usage highwater has changed by 0.30% relative to baseline" ) - @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.read_baseline_file") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") + @mock.patch("CIME.baselines.performance.read_baseline_file") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test_perf_compare_memory_baseline_above_threshold( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): @@ -594,7 +594,7 @@ def test_perf_compare_memory_baseline_above_threshold( 0.05, ) - (below_tolerance, comment) = baselines.perf_compare_memory_baseline(case) + (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) assert not below_tolerance assert ( @@ -602,9 +602,9 @@ def test_perf_compare_memory_baseline_above_threshold( == "Error: Memory usage increase >5% from baseline's 1000.000000 to 2003.000000" ) - @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.read_baseline_file") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") + @mock.patch("CIME.baselines.performance.read_baseline_file") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test_perf_compare_memory_baseline( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): @@ -629,7 +629,7 @@ def test_perf_compare_memory_baseline( 0.05, ) - (below_tolerance, comment) = baselines.perf_compare_memory_baseline(case) + (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) assert below_tolerance assert ( @@ -654,7 +654,7 @@ def test_get_latest_cpl_logs_found_multiple(self): "mct", ) - latest_cpl_logs = baselines.get_latest_cpl_logs(case) + latest_cpl_logs = performance.get_latest_cpl_logs(case) assert len(latest_cpl_logs) == 2 assert sorted(latest_cpl_logs) == sorted( @@ -675,7 +675,7 @@ def test_get_latest_cpl_logs_found_single(self): "mct", ) - latest_cpl_logs = baselines.get_latest_cpl_logs(case) + latest_cpl_logs = performance.get_latest_cpl_logs(case) assert len(latest_cpl_logs) == 1 assert latest_cpl_logs[0] == str(cpl_log_path) @@ -687,6 +687,6 @@ def test_get_latest_cpl_logs(self): "mct", ) - latest_cpl_logs = baselines.get_latest_cpl_logs(case) + latest_cpl_logs = performance.get_latest_cpl_logs(case) assert len(latest_cpl_logs) == 0 From b856aa19213ec50e6945b10269c167340f7b76e5 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 13 Oct 2023 18:04:49 -0700 Subject: [PATCH 28/57] Fixes returning raw string from baseline --- CIME/baselines/performance.py | 12 +++++---- CIME/tests/test_unit_baselines_performance.py | 27 +++++++++---------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/CIME/baselines/performance.py b/CIME/baselines/performance.py index 2a3fe7369c6..b2ede20dce5 100644 --- a/CIME/baselines/performance.py +++ b/CIME/baselines/performance.py @@ -417,6 +417,8 @@ def read_baseline_file(baseline_file): """ Reads value from `baseline_file`. + Strips comments and returns the raw content to be decoded. + Parameters ---------- baseline_file : str @@ -430,7 +432,7 @@ def read_baseline_file(baseline_file): with open(baseline_file) as fd: lines = [x.strip() for x in fd.readlines() if not x.startswith("#")] - return lines + return "\n".join(lines) def _perf_compare_throughput_baseline(case, baseline, tolerance): @@ -459,8 +461,8 @@ def _perf_compare_throughput_baseline(case, baseline, tolerance): try: # default baseline is stored as single float - baseline = float(baseline[0]) - except IndexError: + baseline = float(baseline) + except ValueError: comment = "Could not compare throughput to baseline, as basline had no value." return None, comment @@ -516,8 +518,8 @@ def _perf_compare_memory_baseline(case, baseline, tolerance): try: # default baseline is stored as single float - baseline = float(baseline[0]) - except IndexError: + baseline = float(baseline) + except ValueError: baseline = 0.0 try: diff --git a/CIME/tests/test_unit_baselines_performance.py b/CIME/tests/test_unit_baselines_performance.py index a6eebbb2bbb..c73c2c15bd3 100644 --- a/CIME/tests/test_unit_baselines_performance.py +++ b/CIME/tests/test_unit_baselines_performance.py @@ -154,12 +154,13 @@ def test_get_cpl_mem_usage(self, isfile): def test_read_baseline_file_multi_line(self): with mock.patch( - "builtins.open", mock.mock_open(read_data="1000.0\n2000.0\n") + "builtins.open", + mock.mock_open(read_data="#comment about data\n1000.0\n2000.0\n"), ) as mock_file: baseline = performance.read_baseline_file("/tmp/cpl-mem.log") mock_file.assert_called_with("/tmp/cpl-mem.log") - assert baseline == ["1000.0", "2000.0"] + assert baseline == "1000.0\n2000.0" def test_read_baseline_file_content(self): with mock.patch( @@ -168,14 +169,14 @@ def test_read_baseline_file_content(self): baseline = performance.read_baseline_file("/tmp/cpl-mem.log") mock_file.assert_called_with("/tmp/cpl-mem.log") - assert baseline == ["1000.0"] + assert baseline == "1000.0" def test_read_baseline_file(self): with mock.patch("builtins.open", mock.mock_open(read_data="")) as mock_file: baseline = performance.read_baseline_file("/tmp/cpl-mem.log") mock_file.assert_called_with("/tmp/cpl-mem.log") - assert baseline == [] + assert baseline == "" def test_write_baseline_file(self): with mock.patch("builtins.open", mock.mock_open()) as mock_file: @@ -316,7 +317,7 @@ def test_perf_compare_throughput_baseline_no_baseline_file( def test_perf_compare_throughput_baseline_no_baseline( self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): - read_baseline_file.return_value = [] + read_baseline_file.return_value = "" _perf_get_throughput.return_value = 504 @@ -348,9 +349,7 @@ def test_perf_compare_throughput_baseline_no_baseline( def test_perf_compare_throughput_baseline_no_tolerance( self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): - read_baseline_file.return_value = [ - "500", - ] + read_baseline_file.return_value = "500" _perf_get_throughput.return_value = 504 @@ -382,7 +381,7 @@ def test_perf_compare_throughput_baseline_no_tolerance( def test_perf_compare_throughput_baseline_above_threshold( self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): - read_baseline_file.return_value = ["1000"] + read_baseline_file.return_value = "1000" _perf_get_throughput.return_value = 504 @@ -413,7 +412,7 @@ def test_perf_compare_throughput_baseline_above_threshold( def test_perf_compare_throughput_baseline( self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): - read_baseline_file.return_value = ["500"] + read_baseline_file.return_value = "500" _perf_get_throughput.return_value = 504 @@ -445,7 +444,7 @@ def test_perf_compare_throughput_baseline( def test_perf_compare_memory_baseline_no_baseline( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): - read_baseline_file.return_value = [] + read_baseline_file.return_value = "" get_cpl_mem_usage.return_value = [ (1, 1000.0), @@ -538,7 +537,7 @@ def test_perf_compare_memory_baseline_no_baseline_file( def test_perf_compare_memory_baseline_no_tolerance( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): - read_baseline_file.return_value = ["1000.0"] + read_baseline_file.return_value = "1000.0" get_cpl_mem_usage.return_value = [ (1, 1000.0), @@ -573,7 +572,7 @@ def test_perf_compare_memory_baseline_no_tolerance( def test_perf_compare_memory_baseline_above_threshold( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): - read_baseline_file.return_value = ["1000.0"] + read_baseline_file.return_value = "1000.0" get_cpl_mem_usage.return_value = [ (1, 2000.0), @@ -608,7 +607,7 @@ def test_perf_compare_memory_baseline_above_threshold( def test_perf_compare_memory_baseline( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): - read_baseline_file.return_value = ["1000.0"] + read_baseline_file.return_value = "1000.0" get_cpl_mem_usage.return_value = [ (1, 1000.0), From 192eb3236fb377325cf1c5e1524166bb84943c2a Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 13 Oct 2023 18:05:37 -0700 Subject: [PATCH 29/57] Updates baseline documentation --- doc/source/users_guide/testing.rst | 151 ++++++++++++++++++++++++++--- 1 file changed, 135 insertions(+), 16 deletions(-) diff --git a/doc/source/users_guide/testing.rst b/doc/source/users_guide/testing.rst index 8ea7c29467c..40868d2bbdd 100644 --- a/doc/source/users_guide/testing.rst +++ b/doc/source/users_guide/testing.rst @@ -371,29 +371,148 @@ Interpreting test output is pretty easy, looking at an example:: You can see that `create_test <../Tools_user/create_test.html>`_ informs the user of the case directory and of the progress and duration of the various test phases. -=================== -Managing baselines -=================== -.. _`Managing baselines`: +========= +Baselines +========= +.. _`Baselines`: -A big part of testing is managing your baselines (sometimes called gold results). We have provided -tools to help the user do this without having to repeat full runs of test cases with `create_test <../Tools_user/create_test.html>`_ . +A big part of testing is managing your baselines (sometimes called gold results). We have provided tools to help the user do this without having to repeat full runs of test cases with `create_test <../Tools_user/create_test.html>`_ . -bless_test_results: Takes a batch of cases of tests that have already been run and copy their -results to a baseline area. +------------------- +Creating a baseline +------------------- +.. _`Creating a baseline`: -compare_test_results: Takes a batch of cases of tests that have already been run and compare their -results to a baseline area. +A baseline can be generated by passing ``-g`` to `create_test <../Tools_user/create_test.html>`_. There are additional options to control generating baselines.:: -Take a batch of results for the jenkins user for the testid 'mytest' and copy the results to -baselines for 'master':: + ./scripts/create_test -b master -g SMS.ne30_f19_g16_rx1.A - ./bless_test_results -r /home/jenkins/e3sm/scratch/jenkins/ -t mytest -b master +-------------------- +Comparing a baseline +-------------------- +.. _`Comparing a baseline`: -Take a batch of results for the jenkins user for the testid 'mytest' and compare the results to -baselines for 'master':: +Comparing the output of a test to a baseline is achieved by passing ``-c`` to `create_test <../Tools_user/create_test.html>`_.:: + + ./scripts/create_test -b master -c SMS.ne30_f19_g16_rx1.A + +------------------ +Managing baselines +------------------ +.. _`Managing baselines`: - ./compare_test_results -r /home/jenkins/e3sm/scratch/jenkins/ -t mytest -b master +Once a baseline has been generated it can be managed using the `bless_test_results <../Tools_user/bless_test_results.html>`_ tool. The tool provides the ability to bless different features of the baseline. The currently supported features are namelist files, history files, and performance metrics. The performance metrics are separated into throughput and memory usage. + +The following command can be used to compare a test to a baseline and bless an update to the history file.:: + + ./CIME/Tools/bless_test_results -b master --hist-only SMS.ne30_f19_g16_rx1.A + +The `compare_test_results <../Tools_user/compare_test_results.html>_` tool can be used to quickly compare tests to baselines and report any `diffs`.:: + + ./CIME/Tools/compare_test_results -b master SMS.ne30_f19_g16_rx1.A + +--------------------- +Performance baselines +--------------------- +.. _`Performance baselines`: +By default performance baselines are generated by parsing the coupler log and comparing the throughput in SYPD (Simulated Years Per Day) and the memory usage high water. + +This can be customized by creating a python module under ``$DRIVER_ROOT/cime_config/customize``. There are four hooks that can be used to customize the generation and comparison. + +- perf_get_throughput +- perf_get_memory +- perf_compare_throughput_baseline +- perf_compare_memory_baseline + +.. + TODO need to add api docs and link +The following pseudo code is an example of this customization.:: + + # $DRIVER/cime_config/customize/perf_baseline.py + + def perf_get_throughput(case): + """ + Parameters + ---------- + case : CIME.case.case.Case + Current case object. + + Returns + ------- + str + Storing throughput value. + """ + current = analyze_throughput(...) + + return json.dumps(current) + + def perf_get_memory(case): + """ + Parameters + ---------- + case : CIME.case.case.Case + Current case object. + + Returns + ------- + str + Storing memory value. + """ + current = analyze_memory(case) + + return json.dumps(current) + + def perf_compare_throughput_baseline(case, baseline, tolerance): + """ + Parameters + ---------- + case : CIME.case.case.Case + Current case object. + baseline : str + Baseline throughput value. + tolerance : float + Allowed difference tolerance. + + Returns + ------- + bool + Whether throughput diff is below tolerance. + str + Comments about the results. + """ + current = analyze_throughput(case) + + baseline = json.loads(baseline) + + diff, comments = generate_diff(...) + + return diff, comments + + def perf_compare_memory_baseline(case, baseline, tolerance): + """ + Parameters + ---------- + case : CIME.case.case.Case + Current case object. + baseline : str + Baseline memory value. + tolerance : float + Allowed difference tolerance. + + Returns + ------- + bool + Whether memory diff is below tolerance. + str + Comments about the results. + """ + current = analyze_memory(case) + + baseline = json.loads(baseline) + + diff, comments = generate_diff(...) + + return diff, comments ============= Adding tests From 3071dc359d03c7d24ba1d65581fd49f062eb4420 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 13 Oct 2023 18:40:06 -0700 Subject: [PATCH 30/57] Adds missing logging --- CIME/baselines/performance.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/CIME/baselines/performance.py b/CIME/baselines/performance.py index b2ede20dce5..f8f1fda77a1 100644 --- a/CIME/baselines/performance.py +++ b/CIME/baselines/performance.py @@ -136,8 +136,8 @@ def perf_write_baseline(case, basegen_dir, throughput=True, memory=True): if throughput: try: tput = perf_get_throughput(case, config) - except RuntimeError: - pass + except RuntimeError as e: + logger.debug("Could not get throughput: {0!s}".format(e)) else: baseline_file = os.path.join(basegen_dir, "cpl-tput.log") @@ -146,8 +146,8 @@ def perf_write_baseline(case, basegen_dir, throughput=True, memory=True): if memory: try: mem = perf_get_memory(case, config) - except RuntimeError: - pass + except RuntimeError as e: + logger.info("Could not get memory usage: {0!s}".format(e)) else: baseline_file = os.path.join(basegen_dir, "cpl-mem.log") @@ -286,6 +286,8 @@ def _perf_get_memory(case, cpllog=None): memlist = get_cpl_mem_usage(cpllog[0]) except (FileNotFoundError, IndexError): memlist = None + + logger.debug("Could not parse memory usage from coupler log") else: if len(memlist) <= 3: raise RuntimeError( @@ -318,6 +320,8 @@ def _perf_get_throughput(case): except (FileNotFoundError, IndexError): tput = None + logger.debug("Could not parse throughput from coupler log") + return tput From 7e307036e05904c448b523726b6274a946e4dae2 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 13 Oct 2023 18:56:01 -0700 Subject: [PATCH 31/57] Fixes commenting preview that was never pushed --- .github/workflows/docs.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 098cd4c8c01..70899438082 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -91,7 +91,9 @@ jobs: user_name: 'github-actions[bot]' user_email: 'github-actions[bot]@users.noreply.github.com' - name: Comment about previewing documentation - if: ${{ github.event_name == 'pull_request' }} + if: | + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository uses: actions/github-script@v6 with: script: | From b5ea392f79205b20c8ecf794fe67d23311a4e225 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Mon, 16 Oct 2023 14:36:50 -0600 Subject: [PATCH 32/57] Remove deprecated 'distutils' package from ParamGen. --- CIME/ParamGen/paramgen.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/CIME/ParamGen/paramgen.py b/CIME/ParamGen/paramgen.py index 4fa6221f06e..188b954bd31 100644 --- a/CIME/ParamGen/paramgen.py +++ b/CIME/ParamGen/paramgen.py @@ -4,6 +4,7 @@ from copy import deepcopy import logging import subprocess +import shutil try: from paramgen_utils import is_logical_expr, is_formula, has_unexpanded_var @@ -136,9 +137,7 @@ def from_xml_nml(cls, input_path, match="last", no_duplicates=False): """ # First check whether the given xml file conforms to the entry_id_pg.xsd schema - from distutils.spawn import find_executable - - xmllint = find_executable("xmllint") + xmllint = shutil.which("xmllint") if xmllint is None: logger.warning("Couldn't find xmllint. Skipping schema check") else: From 56ea782a6dcb37481fa29d431d7d671345f30311 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Tue, 17 Oct 2023 09:49:10 -0600 Subject: [PATCH 33/57] Jenkins_generic_job: Add --save-timing --- CIME/Tools/jenkins_generic_job | 9 +++++++++ CIME/jenkins_generic_job.py | 12 ++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/CIME/Tools/jenkins_generic_job b/CIME/Tools/jenkins_generic_job index 66dbbdc6e31..0c1a05a128e 100755 --- a/CIME/Tools/jenkins_generic_job +++ b/CIME/Tools/jenkins_generic_job @@ -180,6 +180,12 @@ OR help="Do not fail if there are memleaks", ) + parser.add_argument( + "--save-timing", + action="store_true", + help="Tell create_test to save timings of tests", + ) + parser.add_argument( "--pes-file", help="Full pathname of an optional pes specification file. The file" @@ -259,6 +265,7 @@ OR args.check_throughput, args.check_memory, args.ignore_memleak, + args.save_timing, args.pes_file, args.jenkins_id, args.queue, @@ -289,6 +296,7 @@ def _main_func(description): check_throughput, check_memory, ignore_memleak, + save_timing, pes_file, jenkins_id, queue, @@ -317,6 +325,7 @@ def _main_func(description): check_throughput, check_memory, ignore_memleak, + save_timing, pes_file, jenkins_id, queue, diff --git a/CIME/jenkins_generic_job.py b/CIME/jenkins_generic_job.py index 57432e822e6..e89e7ec2e9c 100644 --- a/CIME/jenkins_generic_job.py +++ b/CIME/jenkins_generic_job.py @@ -279,6 +279,7 @@ def jenkins_generic_job( check_throughput, check_memory, ignore_memleak, + save_timing, pes_file, jenkins_id, queue, @@ -361,16 +362,19 @@ def jenkins_generic_job( create_test_args.append("-j {:d}".format(parallel_jobs)) if walltime is not None: - create_test_args.append(" --walltime " + walltime) + create_test_args.append("--walltime " + walltime) if baseline_root is not None: - create_test_args.append(" --baseline-root " + baseline_root) + create_test_args.append("--baseline-root " + baseline_root) if pes_file is not None: - create_test_args.append(" --pesfile " + pes_file) + create_test_args.append("--pesfile " + pes_file) if queue is not None: - create_test_args.append(" --queue " + queue) + create_test_args.append("--queue " + queue) + + if save_timing: + create_test_args.append("--save-timing") create_test_cmd = "./create_test " + " ".join(create_test_args) From 00f957cce1d3c4e29cfadb9e99ffb9e76a2e175a Mon Sep 17 00:00:00 2001 From: James Foucar Date: Tue, 17 Oct 2023 10:58:42 -0600 Subject: [PATCH 34/57] Add test --- CIME/get_tests.py | 4 ++++ CIME/tests/test_sys_jenkins_generic_job.py | 28 ++++++++++++++++------ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/CIME/get_tests.py b/CIME/get_tests.py index f6762e385cc..e4d7d3ea4a8 100644 --- a/CIME/get_tests.py +++ b/CIME/get_tests.py @@ -100,6 +100,10 @@ "SMS_P16.T42_T42.S", ), }, + "cime_test_timing": { + "time": "0:10:00", + "tests": ("SMS_P1.T42_T42.S",), + }, "cime_test_repeat": { "tests": ( "TESTRUNPASS_P1.f19_g16_rx1.A", diff --git a/CIME/tests/test_sys_jenkins_generic_job.py b/CIME/tests/test_sys_jenkins_generic_job.py index 7fb2a83b740..30b31c5c8d6 100644 --- a/CIME/tests/test_sys_jenkins_generic_job.py +++ b/CIME/tests/test_sys_jenkins_generic_job.py @@ -61,18 +61,17 @@ def threaded_test(self, expect_works, extra_args, build_name=None): self._thread_error = str(e) def assert_num_leftovers(self, suite): - num_tests_in_tiny = len(get_tests.get_test_suite(suite)) + num_tests_in_suite = len(get_tests.get_test_suite(suite)) - jenkins_dirs = glob.glob( - "%s/*%s*/" % (self._jenkins_root, self._baseline_name.capitalize()) - ) # case dirs + case_glob = "%s/*%s*/" % (self._jenkins_root, self._baseline_name.capitalize()) + jenkins_dirs = glob.glob(case_glob) # Case dirs # scratch_dirs = glob.glob("%s/*%s*/" % (self._testroot, test_id)) # blr/run dirs self.assertEqual( - num_tests_in_tiny, + num_tests_in_suite, len(jenkins_dirs), - msg="Wrong number of leftover directories in %s, expected %d, see %s" - % (self._jenkins_root, num_tests_in_tiny, jenkins_dirs), + msg="Wrong number of leftover directories in %s, expected %d, see %s. Glob checked %s" + % (self._jenkins_root, num_tests_in_suite, jenkins_dirs, case_glob), ) # JGF: Can't test this at the moment due to root change flag given to jenkins_generic_job @@ -97,6 +96,21 @@ def test_jenkins_generic_job(self): ) # jenkins_generic_job should have automatically cleaned up leftovers from prior run self.assert_dashboard_has_build(build_name) + def test_jenkins_generic_job_save_timing(self): + self.simple_test( + True, "-t cime_test_timing --save-timing -b %s" % self._baseline_name + ) + self.assert_num_leftovers("cime_test_timing") + + jenkins_dirs = glob.glob( + "%s/*%s*/" % (self._jenkins_root, self._baseline_name.capitalize()) + ) # case dirs + case = jenkins_dirs[0] + result = self.run_cmd_assert_result( + "./xmlquery --value SAVE_TIMING", from_dir=case + ) + self.assertEqual(result, "TRUE") + def test_jenkins_generic_job_kill(self): build_name = "jenkins_generic_job_kill_%s" % utils.get_timestamp() run_thread = threading.Thread( From 89c5b5564778c3ddaf31448d806e2d35881f0d91 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 19 Oct 2023 12:29:33 -0700 Subject: [PATCH 35/57] Removes requiring an *-only option --- CIME/Tools/bless_test_results | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CIME/Tools/bless_test_results b/CIME/Tools/bless_test_results index cb6bd2f972a..82c92c6cfad 100755 --- a/CIME/Tools/bless_test_results +++ b/CIME/Tools/bless_test_results @@ -104,7 +104,7 @@ OR def create_bless_options(parser): bless_group = parser.add_argument_group("Bless options") - mutual_bless_group = bless_group.add_mutually_exclusive_group(required=True) + mutual_bless_group = bless_group.add_mutually_exclusive_group() mutual_bless_group.add_argument( "-n", "--namelists-only", action="store_true", help="Only analyze namelists." From 01b7547481d5d9738703a6dd20a5296d3d45beb4 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 19 Oct 2023 17:59:03 -0700 Subject: [PATCH 36/57] Fixes default bless behavior --- CIME/bless_test_results.py | 104 +++++++++++++++++++++---------------- 1 file changed, 60 insertions(+), 44 deletions(-) diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index 0f09b91eb0b..181d449e59f 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -183,7 +183,6 @@ def bless_history(test_name, case, baseline_name, baseline_root, report_only, fo return True, None -############################################################################### def bless_test_results( baseline_name, baseline_root, @@ -203,7 +202,8 @@ def bless_test_results( new_test_id=None, **_, # Capture all for extra ): - ############################################################################### + bless_all = not (namelists_only | hist_only | tput_only | mem_only) + test_status_files = get_test_status_files(test_root, compiler, test_id=test_id) # auto-adjust test-id if multiple rounds of tests were matched @@ -255,18 +255,15 @@ def bless_test_results( if bless_tests in [[], None] or CIME.utils.match_any( test_name, bless_tests_counts ): - ts_kwargs = dict(ignore_namelists=True, ignore_memleak=True) - - if tput_only: - ts_kwargs["check_throughput"] = True - - if mem_only: - ts_kwargs["check_memory"] = True - - overall_result, phase = ts.get_overall_test_status(**ts_kwargs) + overall_result, phase = ts.get_overall_test_status( + ignore_namelists=True, + ignore_memleak=True, + check_throughput=True, + check_memory=True, + ) # See if we need to bless namelist - if not hist_only: + if namelists_only or bless_all: if no_skip_pass: nl_bless = True else: @@ -274,41 +271,26 @@ def bless_test_results( else: nl_bless = False - # See if we need to bless baselines - if not namelists_only and not build_only: - run_result = ts.get_status(RUN_PHASE) - if run_result is None: - broken_blesses.append((test_name, "no run phase")) - logger.warning( - "Test '{}' did not make it to run phase".format(test_name) - ) - hist_bless = False - elif run_result != TEST_PASS_STATUS: - broken_blesses.append((test_name, "run phase did not pass")) - logger.warning( - "Test '{}' run phase did not pass, not safe to bless, test status = {}".format( - test_name, ts.phase_statuses_dump() - ) - ) - hist_bless = False - elif overall_result == TEST_FAIL_STATUS: - broken_blesses.append((test_name, "test did not pass")) - logger.warning( - "Test '{}' did not pass due to phase {}, not safe to bless, test status = {}".format( - test_name, phase, ts.phase_statuses_dump() - ) - ) - hist_bless = False + hist_bless, tput_bless, mem_bless = [False] * 3 - elif no_skip_pass: - hist_bless = True - else: - hist_bless = ts.get_status(BASELINE_PHASE) != TEST_PASS_STATUS - else: - hist_bless = False + # Skip if test is build only i.e. testopts contains "B" + if not build_only: + allowed = bless_allowed( + test_name, ts, broken_blesses, overall_result, no_skip_pass, phase + ) + + # See if we need to bless baselines + if hist_only or bless_all: + hist_bless = allowed + + if tput_only or bless_all: + tput_bless = allowed + + if mem_only or bless_all: + mem_bless = allowed # Now, do the bless - if not nl_bless and not hist_bless and not tput_only and not mem_only: + if not nl_bless and not hist_bless and not tput_bless and not mem_bless: logger.info( "Nothing to bless for test: {}, overall status: {}".format( test_name, overall_result @@ -441,3 +423,37 @@ def bless_test_results( success = False return success + + +def bless_allowed(test_name, ts, broken_blesses, overall_result, no_skip_pass, phase): + allow_bless = False + + run_result = ts.get_status(RUN_PHASE) + + if run_result is None: + broken_blesses.append((test_name, "no run phase")) + logger.warning("Test '{}' did not make it to run phase".format(test_name)) + allow_bless = False + elif run_result != TEST_PASS_STATUS: + broken_blesses.append((test_name, "run phase did not pass")) + logger.warning( + "Test '{}' run phase did not pass, not safe to bless, test status = {}".format( + test_name, ts.phase_statuses_dump() + ) + ) + allow_bless = False + elif overall_result == TEST_FAIL_STATUS: + broken_blesses.append((test_name, "test did not pass")) + logger.warning( + "Test '{}' did not pass due to phase {}, not safe to bless, test status = {}".format( + test_name, phase, ts.phase_statuses_dump() + ) + ) + allow_bless = False + + elif no_skip_pass: + allow_bless = True + else: + allow_bless = ts.get_status(BASELINE_PHASE) != TEST_PASS_STATUS + + return allow_bless From df9c7202089b23dc21b30f46b46f9487e7fe8d07 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 19 Oct 2023 18:52:06 -0700 Subject: [PATCH 37/57] Adds feature to exclude tests --- CIME/Tools/bless_test_results | 2 + CIME/bless_test_results.py | 267 ++++++++++++++++++---------------- 2 files changed, 141 insertions(+), 128 deletions(-) diff --git a/CIME/Tools/bless_test_results b/CIME/Tools/bless_test_results index 82c92c6cfad..d2bc707d3c2 100755 --- a/CIME/Tools/bless_test_results +++ b/CIME/Tools/bless_test_results @@ -90,6 +90,8 @@ OR "\ncan follow either the config_pes.xml or the env_mach_pes.xml format.", ) + parser.add_argument("--exclude", nargs="*", help="Exclude tests") + parser.add_argument( "bless_tests", nargs="*", diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index 181d449e59f..a22e7ed9519 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -5,6 +5,7 @@ get_scripts_root, EnvironmentContext, parse_test_name, + match_any, ) from CIME.config import Config from CIME.test_status import * @@ -200,6 +201,7 @@ def bless_test_results( no_skip_pass=False, new_test_root=None, new_test_id=None, + exclude=None, **_, # Capture all for extra ): bless_all = not (namelists_only | hist_only | tput_only | mem_only) @@ -220,10 +222,14 @@ def bless_test_results( most_recent = sorted(timestamps)[-1] logger.info("Matched test batch is {}".format(most_recent)) - bless_tests_counts = None + bless_tests_counts = [] if bless_tests: bless_tests_counts = dict([(bless_test, 0) for bless_test in bless_tests]) + # compile excludes into single regex + if exclude is not None: + exclude = re.compile("|".join([f"({x})" for x in exclude])) + broken_blesses = [] for test_status_file in test_status_files: if not most_recent in test_status_file: @@ -239,7 +245,7 @@ def bless_test_results( if test_name is None: case_dir = os.path.basename(test_dir) test_name = CIME.utils.normalize_case_id(case_dir) - if not bless_tests or CIME.utils.match_any(test_name, bless_tests_counts): + if not bless_tests or match_any(test_name, bless_tests_counts): broken_blesses.append( ( "unknown", @@ -252,154 +258,159 @@ def bless_test_results( else: continue - if bless_tests in [[], None] or CIME.utils.match_any( - test_name, bless_tests_counts - ): - overall_result, phase = ts.get_overall_test_status( - ignore_namelists=True, - ignore_memleak=True, - check_throughput=True, - check_memory=True, - ) + # Must pass tests to continue + has_no_tests = bless_tests in [[], None] + match_test_name = match_any(test_name, bless_tests_counts) + excluded = exclude.match(test_name) - # See if we need to bless namelist - if namelists_only or bless_all: - if no_skip_pass: - nl_bless = True - else: - nl_bless = ts.get_status(NAMELIST_PHASE) != TEST_PASS_STATUS + if (not has_no_tests and not match_test_name) or excluded: + logger.info("Skipping {!r}".format(test_name)) + + continue + + overall_result, phase = ts.get_overall_test_status( + ignore_namelists=True, + ignore_memleak=True, + check_throughput=True, + check_memory=True, + ) + + # See if we need to bless namelist + if namelists_only or bless_all: + if no_skip_pass: + nl_bless = True else: - nl_bless = False + nl_bless = ts.get_status(NAMELIST_PHASE) != TEST_PASS_STATUS + else: + nl_bless = False - hist_bless, tput_bless, mem_bless = [False] * 3 + hist_bless, tput_bless, mem_bless = [False] * 3 - # Skip if test is build only i.e. testopts contains "B" - if not build_only: - allowed = bless_allowed( - test_name, ts, broken_blesses, overall_result, no_skip_pass, phase - ) + # Skip if test is build only i.e. testopts contains "B" + if not build_only: + allowed = bless_allowed( + test_name, ts, broken_blesses, overall_result, no_skip_pass, phase + ) - # See if we need to bless baselines - if hist_only or bless_all: - hist_bless = allowed + # See if we need to bless baselines + if hist_only or bless_all: + hist_bless = allowed - if tput_only or bless_all: - tput_bless = allowed + if tput_only or bless_all: + tput_bless = allowed - if mem_only or bless_all: - mem_bless = allowed + if mem_only or bless_all: + mem_bless = allowed - # Now, do the bless - if not nl_bless and not hist_bless and not tput_bless and not mem_bless: - logger.info( - "Nothing to bless for test: {}, overall status: {}".format( - test_name, overall_result - ) + # Now, do the bless + if not nl_bless and not hist_bless and not tput_bless and not mem_bless: + logger.info( + "Nothing to bless for test: {}, overall status: {}".format( + test_name, overall_result ) - else: - logger.info( - "###############################################################################" - ) - logger.info( - "Blessing results for test: {}, most recent result: {}".format( - test_name, overall_result - ) - ) - logger.info("Case dir: {}".format(test_dir)) - logger.info( - "###############################################################################" + ) + else: + logger.info( + "###############################################################################" + ) + logger.info( + "Blessing results for test: {}, most recent result: {}".format( + test_name, overall_result ) - if not force: - time.sleep(2) - - with Case(test_dir) as case: - # Resolve baseline_name and baseline_root - if baseline_name is None: - baseline_name_resolved = case.get_value("BASELINE_NAME_CMP") - if not baseline_name_resolved: - baseline_name_resolved = CIME.utils.get_current_branch( - repo=CIME.utils.get_cime_root() - ) - else: - baseline_name_resolved = baseline_name - - if baseline_root is None: - baseline_root_resolved = case.get_value("BASELINE_ROOT") - else: - baseline_root_resolved = baseline_root - - if baseline_name_resolved is None: - broken_blesses.append( - (test_name, "Could not determine baseline name") + ) + logger.info("Case dir: {}".format(test_dir)) + logger.info( + "###############################################################################" + ) + if not force: + time.sleep(2) + + with Case(test_dir) as case: + # Resolve baseline_name and baseline_root + if baseline_name is None: + baseline_name_resolved = case.get_value("BASELINE_NAME_CMP") + if not baseline_name_resolved: + baseline_name_resolved = CIME.utils.get_current_branch( + repo=CIME.utils.get_cime_root() ) - continue + else: + baseline_name_resolved = baseline_name - if baseline_root_resolved is None: - broken_blesses.append( - (test_name, "Could not determine baseline root") - ) - continue + if baseline_root is None: + baseline_root_resolved = case.get_value("BASELINE_ROOT") + else: + baseline_root_resolved = baseline_root - # Bless namelists - if nl_bless: - success, reason = bless_namelists( + if baseline_name_resolved is None: + broken_blesses.append( + (test_name, "Could not determine baseline name") + ) + continue + + if baseline_root_resolved is None: + broken_blesses.append( + (test_name, "Could not determine baseline root") + ) + continue + + # Bless namelists + if nl_bless: + success, reason = bless_namelists( + test_name, + report_only, + force, + pesfile, + baseline_name_resolved, + baseline_root_resolved, + new_test_root=new_test_root, + new_test_id=new_test_id, + ) + if not success: + broken_blesses.append((test_name, reason)) + + # Bless hist files + if hist_bless: + if "HOMME" in test_name: + success = False + reason = "HOMME tests cannot be blessed with bless_for_tests" + else: + success, reason = bless_history( test_name, - report_only, - force, - pesfile, - baseline_name_resolved, - baseline_root_resolved, - new_test_root=new_test_root, - new_test_id=new_test_id, - ) - if not success: - broken_blesses.append((test_name, reason)) - - # Bless hist files - if hist_bless: - if "HOMME" in test_name: - success = False - reason = ( - "HOMME tests cannot be blessed with bless_for_tests" - ) - else: - success, reason = bless_history( - test_name, - case, - baseline_name_resolved, - baseline_root_resolved, - report_only, - force, - ) - - if not success: - broken_blesses.append((test_name, reason)) - - if tput_only: - success, reason = bless_throughput( case, - test_name, - baseline_root_resolved, baseline_name_resolved, + baseline_root_resolved, report_only, force, ) - if not success: - broken_blesses.append((test_name, reason)) + if not success: + broken_blesses.append((test_name, reason)) + + if tput_only: + success, reason = bless_throughput( + case, + test_name, + baseline_root_resolved, + baseline_name_resolved, + report_only, + force, + ) - if mem_only: - success, reason = bless_memory( - case, - test_name, - baseline_root_resolved, - baseline_name_resolved, - report_only, - force, - ) + if not success: + broken_blesses.append((test_name, reason)) + + if mem_only: + success, reason = bless_memory( + case, + test_name, + baseline_root_resolved, + baseline_name_resolved, + report_only, + force, + ) - if not success: - broken_blesses.append((test_name, reason)) + if not success: + broken_blesses.append((test_name, reason)) # Emit a warning if items in bless_tests did not match anything if bless_tests: From 54369953e0439fce62960999063442d3364f06ff Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 20 Oct 2023 08:40:51 -0600 Subject: [PATCH 38/57] reduce the number of times env_batch is rewriten --- CIME/case/case_submit.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CIME/case/case_submit.py b/CIME/case/case_submit.py index 71484c9dea0..4ddb26e3056 100644 --- a/CIME/case/case_submit.py +++ b/CIME/case/case_submit.py @@ -94,8 +94,10 @@ def _submit( batch_system = "none" else: batch_system = env_batch.get_batch_system_type() - unlock_file(os.path.basename(env_batch.filename), caseroot=caseroot) - case.set_value("BATCH_SYSTEM", batch_system) + + if batch_system != case.get_value("BATCH_SYSTEM"): + unlock_file(os.path.basename(env_batch.filename), caseroot=caseroot) + case.set_value("BATCH_SYSTEM", batch_system) env_batch_has_changed = False if not external_workflow: From 78f15ddf3bb522ee39b34216ff027443d72f910d Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Mon, 23 Oct 2023 09:36:26 -0700 Subject: [PATCH 39/57] Fixes none value for exclude regex --- CIME/bless_test_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index a22e7ed9519..396a9a0e1c6 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -261,7 +261,7 @@ def bless_test_results( # Must pass tests to continue has_no_tests = bless_tests in [[], None] match_test_name = match_any(test_name, bless_tests_counts) - excluded = exclude.match(test_name) + excluded = exclude.match(test_name) if exclude else False if (not has_no_tests and not match_test_name) or excluded: logger.info("Skipping {!r}".format(test_name)) From 9ffded57f351d1570c6dd5721d9fa904c0198c1e Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Mon, 23 Oct 2023 09:45:34 -0700 Subject: [PATCH 40/57] Fixes function name --- CIME/bless_test_results.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index 396a9a0e1c6..59488fc9450 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -288,19 +288,19 @@ def bless_test_results( # Skip if test is build only i.e. testopts contains "B" if not build_only: - allowed = bless_allowed( + bless_needed = is_bless_needed( test_name, ts, broken_blesses, overall_result, no_skip_pass, phase ) # See if we need to bless baselines if hist_only or bless_all: - hist_bless = allowed + hist_bless = bless_needed if tput_only or bless_all: - tput_bless = allowed + tput_bless = bless_needed if mem_only or bless_all: - mem_bless = allowed + mem_bless = bless_needed # Now, do the bless if not nl_bless and not hist_bless and not tput_bless and not mem_bless: @@ -436,15 +436,15 @@ def bless_test_results( return success -def bless_allowed(test_name, ts, broken_blesses, overall_result, no_skip_pass, phase): - allow_bless = False +def is_bless_needed(test_name, ts, broken_blesses, overall_result, no_skip_pass, phase): + needed = False run_result = ts.get_status(RUN_PHASE) if run_result is None: broken_blesses.append((test_name, "no run phase")) logger.warning("Test '{}' did not make it to run phase".format(test_name)) - allow_bless = False + needed = False elif run_result != TEST_PASS_STATUS: broken_blesses.append((test_name, "run phase did not pass")) logger.warning( @@ -452,7 +452,7 @@ def bless_allowed(test_name, ts, broken_blesses, overall_result, no_skip_pass, p test_name, ts.phase_statuses_dump() ) ) - allow_bless = False + needed = False elif overall_result == TEST_FAIL_STATUS: broken_blesses.append((test_name, "test did not pass")) logger.warning( @@ -460,11 +460,11 @@ def bless_allowed(test_name, ts, broken_blesses, overall_result, no_skip_pass, p test_name, phase, ts.phase_statuses_dump() ) ) - allow_bless = False + needed = False elif no_skip_pass: - allow_bless = True + needed = True else: - allow_bless = ts.get_status(BASELINE_PHASE) != TEST_PASS_STATUS + needed = ts.get_status(BASELINE_PHASE) != TEST_PASS_STATUS - return allow_bless + return needed From 7050f24b5b73dce804f1dcdf4184ff9cce3a0f98 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Mon, 23 Oct 2023 16:51:20 -0700 Subject: [PATCH 41/57] Updates cmake macros --- docker/config_machines.xml | 4 ++-- docker/docker.cmake | 14 ++++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/docker/config_machines.xml b/docker/config_machines.xml index 46bac2f5618..ea60a2cfbb3 100644 --- a/docker/config_machines.xml +++ b/docker/config_machines.xml @@ -37,8 +37,8 @@ 1 1 - /opt/conda - /opt/conda + /opt/conda + /opt/conda diff --git a/docker/docker.cmake b/docker/docker.cmake index 2f9771e6f4c..c60655fd6be 100644 --- a/docker/docker.cmake +++ b/docker/docker.cmake @@ -1,9 +1,15 @@ string(APPEND CXXFLAGS " -std=c++14") string(APPEND CXX_LIBS " -lstdc++") -string(APPEND FFLAGS " -I/opt/conda/include") + +# DEBUGGING variables +# get_cmake_property(_variableNames VARIABLES) +# foreach (_variableName ${_variableNames}) +# message("${_variableName}=${${_variableName}}") +# endforeach() +# message( FATAL_ERROR "EXIT") + # required for grid generation tests that use make +if (CMAKE_SOURCE_DIR MATCHES "^.*TestGridGeneration.*$") +string(APPEND FFLAGS " -I/opt/conda/include") string(APPEND SLIBS " -L/opt/conda/lib -lnetcdf -lnetcdff") -set(MPI_PATH "/opt/conda") -if (CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER_EQUAL 10) - string(APPEND FFLAGS " -fallow-argument-mismatch -fallow-invalid-boz ") endif() From eadb638256bfb2d0673be4d84874d59836de157d Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Tue, 24 Oct 2023 12:15:37 -0700 Subject: [PATCH 42/57] Fixes pes_file argument --- CIME/bless_test_results.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index 59488fc9450..209d5a23931 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -101,7 +101,7 @@ def bless_namelists( test_name, report_only, force, - pesfile, + pes_file, baseline_name, baseline_root, new_test_root=None, @@ -131,8 +131,8 @@ def bless_namelists( if new_test_id is not None: create_test_gen_args += " -t {}".format(new_test_id) - if pesfile is not None: - create_test_gen_args += " --pesfile {}".format(pesfile) + if pes_file is not None: + create_test_gen_args += " --pes_file {}".format(pes_file) stat, out, _ = run_cmd( "{}/create_test {} --namelists-only {} --baseline-root {} -o".format( @@ -196,7 +196,7 @@ def bless_test_results( mem_only=False, report_only=False, force=False, - pesfile=None, + pes_file=None, bless_tests=None, no_skip_pass=False, new_test_root=None, @@ -359,7 +359,7 @@ def bless_test_results( test_name, report_only, force, - pesfile, + pes_file, baseline_name_resolved, baseline_root_resolved, new_test_root=new_test_root, From 50f0ada5b1f662c05006f736db01c04d9e6c6cc0 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Tue, 24 Oct 2023 13:04:48 -0700 Subject: [PATCH 43/57] Fixes create_test pesfile arg --- CIME/bless_test_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index 209d5a23931..f280b40474b 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -132,7 +132,7 @@ def bless_namelists( create_test_gen_args += " -t {}".format(new_test_id) if pes_file is not None: - create_test_gen_args += " --pes_file {}".format(pes_file) + create_test_gen_args += " --pesfile {}".format(pes_file) stat, out, _ = run_cmd( "{}/create_test {} --namelists-only {} --baseline-root {} -o".format( From f50f5f1e91071060dad55de2856391da044c2c00 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Tue, 24 Oct 2023 13:17:32 -0700 Subject: [PATCH 44/57] Fixes duplicate -g args --- CIME/bless_test_results.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index f280b40474b..9034a6594d1 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -119,11 +119,12 @@ def bless_namelists( ): config = Config.instance() - create_test_gen_args = " -g {} ".format( - baseline_name + create_test_gen_args = ( + " -g {} ".format(baseline_name) if config.create_test_flag_mode == "cesm" else " -g -b {} ".format(baseline_name) ) + if new_test_root is not None: create_test_gen_args += " --test-root={0} --output-root={0} ".format( new_test_root From 3b878e559fb2cb6874205d74012dc958b4009a01 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Tue, 24 Oct 2023 13:43:48 -0700 Subject: [PATCH 45/57] Adds missing checks to THROUGHPUT and MEMCOMP phases --- CIME/bless_test_results.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index 9034a6594d1..0dd125a4111 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -300,9 +300,15 @@ def bless_test_results( if tput_only or bless_all: tput_bless = bless_needed + if not tput_bless: + tput_bless = ts.get_status(THROUGHPUT_PHASE) != TEST_PASS_STATUS + if mem_only or bless_all: mem_bless = bless_needed + if not mem_bless: + mem_bless = ts.get_status(MEMCOMP_PHASE) != TEST_PASS_STATUS + # Now, do the bless if not nl_bless and not hist_bless and not tput_bless and not mem_bless: logger.info( From ae788e8b3b39a3054a94318b20048a467abd041b Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Tue, 24 Oct 2023 14:31:45 -0700 Subject: [PATCH 46/57] Fixes memory/throughput check --- CIME/SystemTests/system_tests_common.py | 71 ++++++++++++------- CIME/baselines/performance.py | 18 +---- CIME/bless_test_results.py | 50 ++++++++++--- CIME/tests/test_unit_baselines_performance.py | 14 ++-- 4 files changed, 91 insertions(+), 62 deletions(-) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index a05616882df..a9c61b28c24 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -686,40 +686,59 @@ def _compare_memory(self): Compares current test memory usage to baseline. """ with self._test_status: - below_tolerance, comment = perf_compare_memory_baseline(self._case) - - if below_tolerance is not None: - append_testlog(comment, self._orig_caseroot) + try: + below_tolerance, comment = perf_compare_memory_baseline(self._case) + except Exception as e: + logger.info("Failed to compare memory usage baseline: {!s}".format(e)) - if ( - below_tolerance - and self._test_status.get_status(MEMCOMP_PHASE) is None - ): - self._test_status.set_status(MEMCOMP_PHASE, TEST_PASS_STATUS) - elif self._test_status.get_status(MEMCOMP_PHASE) != TEST_FAIL_STATUS: - self._test_status.set_status( - MEMCOMP_PHASE, TEST_FAIL_STATUS, comments=comment - ) + self._test_status.set_status( + MEMCOMP_PHASE, TEST_FAIL_STATUS, comments=str(e) + ) + else: + if below_tolerance is not None: + append_testlog(comment, self._orig_caseroot) + + if ( + below_tolerance + and self._test_status.get_status(MEMCOMP_PHASE) is None + ): + self._test_status.set_status(MEMCOMP_PHASE, TEST_PASS_STATUS) + elif ( + self._test_status.get_status(MEMCOMP_PHASE) != TEST_FAIL_STATUS + ): + self._test_status.set_status( + MEMCOMP_PHASE, TEST_FAIL_STATUS, comments=comment + ) def _compare_throughput(self): """ Compares current test throughput to baseline. """ with self._test_status: - below_tolerance, comment = perf_compare_throughput_baseline(self._case) - - if below_tolerance is not None: - append_testlog(comment, self._orig_caseroot) + try: + below_tolerance, comment = perf_compare_throughput_baseline(self._case) + except Exception as e: + logger.info("Failed to compare throughput baseline: {!s}".format(e)) - if ( - below_tolerance - and self._test_status.get_status(THROUGHPUT_PHASE) is None - ): - self._test_status.set_status(THROUGHPUT_PHASE, TEST_PASS_STATUS) - elif self._test_status.get_status(THROUGHPUT_PHASE) != TEST_FAIL_STATUS: - self._test_status.set_status( - THROUGHPUT_PHASE, TEST_FAIL_STATUS, comments=comment - ) + self._test_status.set_status( + THROUGHPUT_PHASE, TEST_FAIL_STATUS, comments=str(e) + ) + else: + if below_tolerance is not None: + append_testlog(comment, self._orig_caseroot) + + if ( + below_tolerance + and self._test_status.get_status(THROUGHPUT_PHASE) is None + ): + self._test_status.set_status(THROUGHPUT_PHASE, TEST_PASS_STATUS) + elif ( + self._test_status.get_status(THROUGHPUT_PHASE) + != TEST_FAIL_STATUS + ): + self._test_status.set_status( + THROUGHPUT_PHASE, TEST_FAIL_STATUS, comments=comment + ) def _compare_baseline(self): """ diff --git a/CIME/baselines/performance.py b/CIME/baselines/performance.py index f8f1fda77a1..8aa76083a82 100644 --- a/CIME/baselines/performance.py +++ b/CIME/baselines/performance.py @@ -34,14 +34,7 @@ def perf_compare_throughput_baseline(case, baseline_dir=None): baseline_file = os.path.join(baseline_dir, "cpl-tput.log") - try: - baseline = read_baseline_file(baseline_file) - except FileNotFoundError as e: - comment = f"Could not read baseline throughput file: {e!s}" - - logger.debug(comment) - - return None, comment + baseline = read_baseline_file(baseline_file) tolerance = case.get_value("TEST_TPUT_TOLERANCE") @@ -90,14 +83,7 @@ def perf_compare_memory_baseline(case, baseline_dir=None): baseline_file = os.path.join(baseline_dir, "cpl-mem.log") - try: - baseline = read_baseline_file(baseline_file) - except FileNotFoundError as e: - comment = f"Could not read baseline memory usage: {e!s}" - - logger.debug(comment) - - return None, comment + baseline = read_baseline_file(baseline_file) tolerance = case.get_value("TEST_MEMLEAK_TOLERANCE") diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index 0dd125a4111..4461090c0a6 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -37,12 +37,27 @@ def bless_throughput( baseline_root, baseline_name, case.get_value("CASEBASEID") ) - below_threshold, comment = perf_compare_throughput_baseline( - case, baseline_dir=baseline_dir - ) + try: + below_threshold, comment = perf_compare_throughput_baseline( + case, baseline_dir=baseline_dir + ) + except FileNotFoundError as e: + success = False + + comment = f"Could not read throughput file: {e!s}" + except Exception as e: + success = False + + comment = f"Error comparing throughput baseline: {e!s}" + + # fail early + if not success: + logger.info(comment) + + return success, comment if below_threshold: - logger.info("Diff appears to have been already resolved.") + logger.info("Throughput diff appears to have been already resolved.") else: logger.info(comment) @@ -74,12 +89,27 @@ def bless_memory( baseline_root, baseline_name, case.get_value("CASEBASEID") ) - below_threshold, comment = perf_compare_memory_baseline( - case, baseline_dir=baseline_dir - ) + try: + below_threshold, comment = perf_compare_memory_baseline( + case, baseline_dir=baseline_dir + ) + except FileNotFoundError as e: + success = False + + comment = f"Could not read memory usage file: {e!s}" + except Exception as e: + success = False + + comment = f"Error comparing memory baseline: {e!s}" + + # fail early + if not success: + logger.info(comment) + + return success, comment if below_threshold: - logger.info("Diff appears to have been already resolved.") + logger.info("Memory usage diff appears to have been already resolved.") else: logger.info(comment) @@ -393,7 +423,7 @@ def bless_test_results( if not success: broken_blesses.append((test_name, reason)) - if tput_only: + if tput_bless: success, reason = bless_throughput( case, test_name, @@ -406,7 +436,7 @@ def bless_test_results( if not success: broken_blesses.append((test_name, reason)) - if mem_only: + if mem_bless: success, reason = bless_memory( case, test_name, diff --git a/CIME/tests/test_unit_baselines_performance.py b/CIME/tests/test_unit_baselines_performance.py index c73c2c15bd3..1422f3412b8 100644 --- a/CIME/tests/test_unit_baselines_performance.py +++ b/CIME/tests/test_unit_baselines_performance.py @@ -304,12 +304,8 @@ def test_perf_compare_throughput_baseline_no_baseline_file( 0.05, ) - (below_tolerance, comment) = performance.perf_compare_throughput_baseline( - case - ) - - assert below_tolerance is None - assert comment == "Could not read baseline throughput file: " + with self.assertRaises(FileNotFoundError): + performance.perf_compare_throughput_baseline(case) @mock.patch("CIME.baselines.performance._perf_get_throughput") @mock.patch("CIME.baselines.performance.read_baseline_file") @@ -526,10 +522,8 @@ def test_perf_compare_memory_baseline_no_baseline_file( 0.05, ) - (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) - - assert below_tolerance is None - assert comment == "Could not read baseline memory usage: " + with self.assertRaises(FileNotFoundError): + performance.perf_compare_memory_baseline(case) @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") @mock.patch("CIME.baselines.performance.read_baseline_file") From b1d3d88dc11190433180786a07a94af82c119b6e Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Tue, 24 Oct 2023 21:33:50 -0700 Subject: [PATCH 47/57] Adds unittests to cover bless_test_results module --- CIME/bless_test_results.py | 3 +- CIME/tests/test_unit_bless_test_results.py | 958 +++++++++++++++++++++ 2 files changed, 959 insertions(+), 2 deletions(-) create mode 100644 CIME/tests/test_unit_bless_test_results.py diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index 4461090c0a6..8817f1f22a4 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -179,9 +179,7 @@ def bless_namelists( return True, None -############################################################################### def bless_history(test_name, case, baseline_name, baseline_root, report_only, force): - ############################################################################### real_user = case.get_value("REALUSER") with EnvironmentContext(USER=real_user): @@ -273,6 +271,7 @@ def bless_test_results( testopts = parse_test_name(test_name)[1] testopts = [] if testopts is None else testopts build_only = "B" in testopts + # TODO test_name will never be None otherwise `parse_test_name` would raise an error if test_name is None: case_dir = os.path.basename(test_dir) test_name = CIME.utils.normalize_case_id(case_dir) diff --git a/CIME/tests/test_unit_bless_test_results.py b/CIME/tests/test_unit_bless_test_results.py new file mode 100644 index 00000000000..b0a549d3e1b --- /dev/null +++ b/CIME/tests/test_unit_bless_test_results.py @@ -0,0 +1,958 @@ +import re +import unittest +import tempfile +from unittest import mock +from pathlib import Path + +from CIME.bless_test_results import ( + bless_test_results, + bless_throughput, + bless_memory, + bless_history, + bless_namelists, + is_bless_needed, +) + + +class TestUnitBlessTestResults(unittest.TestCase): + @mock.patch("CIME.bless_test_results.generate_baseline") + @mock.patch("CIME.bless_test_results.compare_baseline") + def test_bless_history_fail(self, compare_baseline, generate_baseline): + generate_baseline.return_value = (False, "") + + compare_baseline.return_value = (False, "") + + case = mock.MagicMock() + case.get_value.side_effect = [ + "USER", + "SMS.f19_g16.S", + "/tmp/run", + ] + + success, comment = bless_history( + "SMS.f19_g16.S", case, "master", "/tmp/baselines", False, True + ) + + assert not success + assert comment == "Generate baseline failed: " + + @mock.patch("CIME.bless_test_results.generate_baseline") + @mock.patch("CIME.bless_test_results.compare_baseline") + def test_bless_history_force(self, compare_baseline, generate_baseline): + generate_baseline.return_value = (True, "") + + compare_baseline.return_value = (False, "") + + case = mock.MagicMock() + case.get_value.side_effect = [ + "USER", + "SMS.f19_g16.S", + "/tmp/run", + ] + + success, comment = bless_history( + "SMS.f19_g16.S", case, "master", "/tmp/baselines", False, True + ) + + assert success + assert comment is None + + @mock.patch("CIME.bless_test_results.compare_baseline") + def test_bless_history(self, compare_baseline): + compare_baseline.return_value = (True, "") + + case = mock.MagicMock() + case.get_value.side_effect = [ + "USER", + "SMS.f19_g16.S", + "/tmp/run", + ] + + success, comment = bless_history( + "SMS.f19_g16.S", case, "master", "/tmp/baselines", True, False + ) + + assert success + assert comment is None + + def test_bless_namelists_report_only(self): + success, comment = bless_namelists( + "SMS.f19_g16.S", + True, + False, + None, + "master", + "/tmp/baselines", + ) + + assert success + assert comment is None + + @mock.patch("CIME.bless_test_results.get_scripts_root") + @mock.patch("CIME.bless_test_results.run_cmd") + def test_bless_namelists_pes_file(self, run_cmd, get_scripts_root): + get_scripts_root.return_value = "/tmp/cime" + + run_cmd.return_value = [1, None, None] + + success, comment = bless_namelists( + "SMS.f19_g16.S", + False, + True, + "/tmp/pes/new_layout.xml", + "master", + "/tmp/baselines", + ) + + assert not success + assert comment == "Namelist regen failed: 'None'" + + call = run_cmd.call_args_list[0] + + assert re.match( + r"/tmp/cime/create_test SMS.f19_g16.S --namelists-only -g (?:-b )?master --pesfile /tmp/pes/new_layout.xml --baseline-root /tmp/baselines -o", + call[0][0], + ) + + @mock.patch("CIME.bless_test_results.get_scripts_root") + @mock.patch("CIME.bless_test_results.run_cmd") + def test_bless_namelists_new_test_id(self, run_cmd, get_scripts_root): + get_scripts_root.return_value = "/tmp/cime" + + run_cmd.return_value = [1, None, None] + + success, comment = bless_namelists( + "SMS.f19_g16.S", + False, + True, + None, + "master", + "/tmp/baselines", + new_test_root="/tmp/other-test-root", + new_test_id="hello", + ) + + assert not success + assert comment == "Namelist regen failed: 'None'" + + call = run_cmd.call_args_list[0] + + assert re.match( + r"/tmp/cime/create_test SMS.f19_g16.S --namelists-only -g (?:-b )?master --test-root=/tmp/other-test-root --output-root=/tmp/other-test-root -t hello --baseline-root /tmp/baselines -o", + call[0][0], + ) + + @mock.patch("CIME.bless_test_results.get_scripts_root") + @mock.patch("CIME.bless_test_results.run_cmd") + def test_bless_namelists_new_test_root(self, run_cmd, get_scripts_root): + get_scripts_root.return_value = "/tmp/cime" + + run_cmd.return_value = [1, None, None] + + success, comment = bless_namelists( + "SMS.f19_g16.S", + False, + True, + None, + "master", + "/tmp/baselines", + new_test_root="/tmp/other-test-root", + ) + + assert not success + assert comment == "Namelist regen failed: 'None'" + + call = run_cmd.call_args_list[0] + + assert re.match( + r"/tmp/cime/create_test SMS.f19_g16.S --namelists-only -g (?:-b )?master --test-root=/tmp/other-test-root --output-root=/tmp/other-test-root --baseline-root /tmp/baselines -o", + call[0][0], + ) + + @mock.patch("CIME.bless_test_results.get_scripts_root") + @mock.patch("CIME.bless_test_results.run_cmd") + def test_bless_namelists_fail(self, run_cmd, get_scripts_root): + get_scripts_root.return_value = "/tmp/cime" + + run_cmd.return_value = [1, None, None] + + success, comment = bless_namelists( + "SMS.f19_g16.S", + False, + True, + None, + "master", + "/tmp/baselines", + ) + + assert not success + assert comment == "Namelist regen failed: 'None'" + + call = run_cmd.call_args_list[0] + + assert re.match( + r"/tmp/cime/create_test SMS.f19_g16.S --namelists-only -g (?:-b )?master --baseline-root /tmp/baselines -o", + call[0][0], + ) + + @mock.patch("CIME.bless_test_results.get_scripts_root") + @mock.patch("CIME.bless_test_results.run_cmd") + def test_bless_namelists_force(self, run_cmd, get_scripts_root): + get_scripts_root.return_value = "/tmp/cime" + + run_cmd.return_value = [0, None, None] + + success, comment = bless_namelists( + "SMS.f19_g16.S", + False, + True, + None, + "master", + "/tmp/baselines", + ) + + assert success + assert comment is None + + call = run_cmd.call_args_list[0] + + assert re.match( + r"/tmp/cime/create_test SMS.f19_g16.S --namelists-only -g (?:-b )?master --baseline-root /tmp/baselines -o", + call[0][0], + ) + + @mock.patch("CIME.bless_test_results.perf_write_baseline") + @mock.patch("CIME.bless_test_results.perf_compare_memory_baseline") + def test_bless_memory_force_error( + self, perf_compare_memory_baseline, perf_write_baseline + ): + perf_write_baseline.side_effect = Exception + + perf_compare_memory_baseline.return_value = (False, "") + + case = mock.MagicMock() + + success, comment = bless_memory( + case, "SMS.f19_g16.S", "/tmp/baselines", "master", False, True + ) + + assert not success + assert ( + comment + == "Failed to write baseline memory usage for test 'SMS.f19_g16.S': " + ) + perf_write_baseline.assert_called() + + @mock.patch("CIME.bless_test_results.perf_write_baseline") + @mock.patch("CIME.bless_test_results.perf_compare_memory_baseline") + def test_bless_memory_force( + self, perf_compare_memory_baseline, perf_write_baseline + ): + perf_compare_memory_baseline.return_value = (False, "") + + case = mock.MagicMock() + + success, comment = bless_memory( + case, "SMS.f19_g16.S", "/tmp/baselines", "master", False, True + ) + + assert success + assert comment is None + perf_write_baseline.assert_called() + + @mock.patch("CIME.bless_test_results.perf_compare_memory_baseline") + def test_bless_memory_report_only(self, perf_compare_memory_baseline): + perf_compare_memory_baseline.return_value = (True, "") + + case = mock.MagicMock() + + success, comment = bless_memory( + case, "SMS.f19_g16.S", "/tmp/baselines", "master", True, False + ) + + assert success + assert comment is None + + @mock.patch("CIME.bless_test_results.perf_compare_memory_baseline") + def test_bless_memory_general_error(self, perf_compare_memory_baseline): + perf_compare_memory_baseline.side_effect = Exception + + case = mock.MagicMock() + + success, comment = bless_memory( + case, "SMS.f19_g16.S", "/tmp/baselines", "master", False, False + ) + + assert not success + assert comment == "Error comparing memory baseline: " + + @mock.patch("CIME.bless_test_results.perf_compare_memory_baseline") + def test_bless_memory_file_not_found_error(self, perf_compare_memory_baseline): + perf_compare_memory_baseline.side_effect = FileNotFoundError + + case = mock.MagicMock() + + success, comment = bless_memory( + case, "SMS.f19_g16.S", "/tmp/baselines", "master", False, False + ) + + assert not success + assert comment == "Could not read memory usage file: " + + @mock.patch("CIME.bless_test_results.perf_compare_memory_baseline") + def test_bless_memory(self, perf_compare_memory_baseline): + perf_compare_memory_baseline.return_value = (True, "") + + case = mock.MagicMock() + + success, comment = bless_memory( + case, "SMS.f19_g16.S", "/tmp/baselines", "master", False, False + ) + + assert success + + @mock.patch("CIME.bless_test_results.perf_write_baseline") + @mock.patch("CIME.bless_test_results.perf_compare_throughput_baseline") + def test_bless_throughput_force_error( + self, perf_compare_throughput_baseline, perf_write_baseline + ): + perf_write_baseline.side_effect = Exception + + perf_compare_throughput_baseline.return_value = (False, "") + + case = mock.MagicMock() + + success, comment = bless_throughput( + case, "SMS.f19_g16.S", "/tmp/baselines", "master", False, True + ) + + assert not success + assert comment == "Failed to write baseline throughput for 'SMS.f19_g16.S': " + perf_write_baseline.assert_called() + + @mock.patch("CIME.bless_test_results.perf_write_baseline") + @mock.patch("CIME.bless_test_results.perf_compare_throughput_baseline") + def test_bless_throughput_force( + self, perf_compare_throughput_baseline, perf_write_baseline + ): + perf_compare_throughput_baseline.return_value = (False, "") + + case = mock.MagicMock() + + success, comment = bless_throughput( + case, "SMS.f19_g16.S", "/tmp/baselines", "master", False, True + ) + + assert success + assert comment is None + perf_write_baseline.assert_called() + + @mock.patch("CIME.bless_test_results.perf_compare_throughput_baseline") + def test_bless_throughput_report_only(self, perf_compare_throughput_baseline): + perf_compare_throughput_baseline.return_value = (True, "") + + case = mock.MagicMock() + + success, comment = bless_throughput( + case, "SMS.f19_g16.S", "/tmp/baselines", "master", True, False + ) + + assert success + assert comment is None + + @mock.patch("CIME.bless_test_results.perf_compare_throughput_baseline") + def test_bless_throughput_general_error(self, perf_compare_throughput_baseline): + perf_compare_throughput_baseline.side_effect = Exception + + case = mock.MagicMock() + + success, comment = bless_throughput( + case, "SMS.f19_g16.S", "/tmp/baselines", "master", False, False + ) + + assert not success + assert comment == "Error comparing throughput baseline: " + + @mock.patch("CIME.bless_test_results.perf_compare_throughput_baseline") + def test_bless_throughput_file_not_found_error( + self, perf_compare_throughput_baseline + ): + perf_compare_throughput_baseline.side_effect = FileNotFoundError + + case = mock.MagicMock() + + success, comment = bless_throughput( + case, "SMS.f19_g16.S", "/tmp/baselines", "master", False, False + ) + + assert not success + assert comment == "Could not read throughput file: " + + @mock.patch("CIME.bless_test_results.perf_compare_throughput_baseline") + def test_bless_throughput(self, perf_compare_throughput_baseline): + perf_compare_throughput_baseline.return_value = (True, "") + + case = mock.MagicMock() + + success, comment = bless_throughput( + case, "SMS.f19_g16.S", "/tmp/baselines", "master", False, False + ) + + assert success + + @mock.patch("CIME.bless_test_results.bless_memory") + @mock.patch("CIME.bless_test_results.Case") + @mock.patch("CIME.bless_test_results.TestStatus") + @mock.patch("CIME.bless_test_results.get_test_status_files") + def test_bless_memory_only( + self, + get_test_status_files, + TestStatus, + Case, + bless_memory, + ): + get_test_status_files.return_value = [ + "/tmp/cases/SMS.f19_g16.S.docker_gnu/TestStatus", + ] + + ts = TestStatus.return_value + ts.get_name.return_value = "SMS.f19_g16.S.docker_gnu" + ts.get_overall_test_status.return_value = ("PASS", "RUN") + ts.get_status.side_effect = ["PASS", "FAIL"] + + case = Case.return_value.__enter__.return_value + + bless_memory.return_value = (True, "") + + success = bless_test_results( + "master", + "/tmp/baseline", + "/tmp/cases", + "gnu", + force=True, + mem_only=True, + ) + + assert success + bless_memory.assert_called() + + @mock.patch("CIME.bless_test_results.bless_throughput") + @mock.patch("CIME.bless_test_results.Case") + @mock.patch("CIME.bless_test_results.TestStatus") + @mock.patch("CIME.bless_test_results.get_test_status_files") + def test_bless_throughput_only( + self, + get_test_status_files, + TestStatus, + Case, + bless_throughput, + ): + get_test_status_files.return_value = [ + "/tmp/cases/SMS.f19_g16.S.docker_gnu/TestStatus", + ] + + ts = TestStatus.return_value + ts.get_name.return_value = "SMS.f19_g16.S.docker_gnu" + ts.get_overall_test_status.return_value = ("PASS", "RUN") + ts.get_status.side_effect = ["PASS", "FAIL"] + + case = Case.return_value.__enter__.return_value + + bless_throughput.return_value = (True, "") + + success = bless_test_results( + "master", + "/tmp/baseline", + "/tmp/cases", + "gnu", + force=True, + tput_only=True, + ) + + assert success + bless_throughput.assert_called() + + @mock.patch("CIME.bless_test_results.bless_namelists") + @mock.patch("CIME.bless_test_results.Case") + @mock.patch("CIME.bless_test_results.TestStatus") + @mock.patch("CIME.bless_test_results.get_test_status_files") + def test_bless_namelists_only( + self, + get_test_status_files, + TestStatus, + Case, + bless_namelists, + ): + get_test_status_files.return_value = [ + "/tmp/cases/SMS.f19_g16.S.docker_gnu/TestStatus", + ] + + ts = TestStatus.return_value + ts.get_name.return_value = "SMS.f19_g16.S.docker_gnu" + ts.get_overall_test_status.return_value = ("PASS", "RUN") + ts.get_status.side_effect = ["FAIL", "PASS", "PASS"] + + case = Case.return_value.__enter__.return_value + + bless_namelists.return_value = (True, "") + + success = bless_test_results( + "master", + "/tmp/baseline", + "/tmp/cases", + "gnu", + force=True, + namelists_only=True, + ) + + assert success + bless_namelists.assert_called() + + @mock.patch("CIME.bless_test_results.bless_history") + @mock.patch("CIME.bless_test_results.Case") + @mock.patch("CIME.bless_test_results.TestStatus") + @mock.patch("CIME.bless_test_results.get_test_status_files") + def test_bless_hist_only( + self, + get_test_status_files, + TestStatus, + Case, + bless_history, + ): + get_test_status_files.return_value = [ + "/tmp/cases/SMS.f19_g16.S.docker_gnu/TestStatus", + ] + + ts = TestStatus.return_value + ts.get_name.return_value = "SMS.f19_g16.S.docker_gnu" + ts.get_overall_test_status.return_value = ("PASS", "RUN") + ts.get_status.side_effect = ["PASS", "FAIL"] + + case = Case.return_value.__enter__.return_value + + bless_history.return_value = (True, "") + + success = bless_test_results( + "master", + "/tmp/baseline", + "/tmp/cases", + "gnu", + force=True, + hist_only=True, + ) + + assert success + bless_history.assert_called() + + @mock.patch("CIME.bless_test_results.Case") + @mock.patch("CIME.bless_test_results.TestStatus") + @mock.patch("CIME.bless_test_results.get_test_status_files") + def test_specific(self, get_test_status_files, TestStatus, Case): + get_test_status_files.return_value = [ + "/tmp/cases/SMS.f19_g16.S.docker_gnu.12345/TestStatus", + "/tmp/cases/PET.f19_g16.S.docker-gnu.12345/TestStatus", + ] + + ts = TestStatus.return_value + ts.get_name.return_value = "SMS.f19_g16.S.docker_gnu" + ts.get_overall_test_status.return_value = ("PASS", "RUN") + ts.get_status.side_effect = ["PASS"] * 10 + + case = Case.return_value.__enter__.return_value + + success = bless_test_results( + "master", + "/tmp/baseline", + "/tmp/cases", + "gnu", + force=True, + bless_tests=["SMS"], + ) + + assert success + + @mock.patch("CIME.bless_test_results.bless_memory") + @mock.patch("CIME.bless_test_results.bless_throughput") + @mock.patch("CIME.bless_test_results.bless_history") + @mock.patch("CIME.bless_test_results.bless_namelists") + @mock.patch("CIME.bless_test_results.Case") + @mock.patch("CIME.bless_test_results.TestStatus") + @mock.patch("CIME.bless_test_results.get_test_status_files") + def test_bless_tests_results_homme( + self, + get_test_status_files, + TestStatus, + Case, + bless_namelists, + bless_history, + bless_throughput, + bless_memory, + ): + bless_memory.return_value = (False, "") + + bless_throughput.return_value = (False, "") + + bless_history.return_value = (False, "") + + bless_namelists.return_value = (False, "") + + get_test_status_files.return_value = [ + "/tmp/cases/SMS.f19_g16.S.docker_gnu.12345/TestStatus", + "/tmp/cases/PET.f19_g16.S.docker-gnu.12345/TestStatus", + ] + + ts = TestStatus.return_value + ts.get_name.return_value = "SMS.f19_g16.HOMME.docker_gnu" + ts.get_overall_test_status.return_value = ("PASS", "RUN") + ts.get_status.side_effect = ["PASS", "PASS", "PASS", "PASS", "PASS"] + + case = Case.return_value.__enter__.return_value + + success = bless_test_results( + "master", + "/tmp/baseline", + "/tmp/cases", + "gnu", + force=True, + no_skip_pass=True, + ) + + assert not success + + @mock.patch("CIME.bless_test_results.bless_memory") + @mock.patch("CIME.bless_test_results.bless_throughput") + @mock.patch("CIME.bless_test_results.bless_history") + @mock.patch("CIME.bless_test_results.bless_namelists") + @mock.patch("CIME.bless_test_results.Case") + @mock.patch("CIME.bless_test_results.TestStatus") + @mock.patch("CIME.bless_test_results.get_test_status_files") + def test_bless_tests_results_fail( + self, + get_test_status_files, + TestStatus, + Case, + bless_namelists, + bless_history, + bless_throughput, + bless_memory, + ): + bless_memory.return_value = (False, "") + + bless_throughput.return_value = (False, "") + + bless_history.return_value = (False, "") + + bless_namelists.return_value = (False, "") + + get_test_status_files.return_value = [ + "/tmp/cases/SMS.f19_g16.S.docker_gnu.12345/TestStatus", + "/tmp/cases/PET.f19_g16.S.docker-gnu.12345/TestStatus", + ] + + ts = TestStatus.return_value + ts.get_name.return_value = "SMS.f19_g16.S.docker_gnu" + ts.get_overall_test_status.return_value = ("PASS", "RUN") + ts.get_status.side_effect = ["PASS", "PASS", "PASS", "PASS", "PASS"] + + case = Case.return_value.__enter__.return_value + + success = bless_test_results( + "master", + "/tmp/baseline", + "/tmp/cases", + "gnu", + force=True, + no_skip_pass=True, + ) + + assert not success + + @mock.patch("CIME.bless_test_results.bless_memory") + @mock.patch("CIME.bless_test_results.bless_throughput") + @mock.patch("CIME.bless_test_results.bless_history") + @mock.patch("CIME.bless_test_results.bless_namelists") + @mock.patch("CIME.bless_test_results.Case") + @mock.patch("CIME.bless_test_results.TestStatus") + @mock.patch("CIME.bless_test_results.get_test_status_files") + def test_no_skip_pass( + self, + get_test_status_files, + TestStatus, + Case, + bless_namelists, + bless_history, + bless_throughput, + bless_memory, + ): + bless_memory.return_value = (True, "") + + bless_throughput.return_value = (True, "") + + bless_history.return_value = (True, "") + + bless_namelists.return_value = (True, "") + + get_test_status_files.return_value = [ + "/tmp/cases/SMS.f19_g16.S.docker_gnu.12345/TestStatus", + "/tmp/cases/PET.f19_g16.S.docker-gnu.12345/TestStatus", + ] + + ts = TestStatus.return_value + ts.get_name.return_value = "SMS.f19_g16.S.docker_gnu" + ts.get_overall_test_status.return_value = ("PASS", "RUN") + ts.get_status.side_effect = ["PASS", "PASS", "PASS", "PASS", "PASS"] + + case = Case.return_value.__enter__.return_value + + success = bless_test_results( + "master", + "/tmp/baseline", + "/tmp/cases", + "gnu", + force=True, + no_skip_pass=True, + ) + + assert success + + @mock.patch("CIME.bless_test_results.Case") + @mock.patch("CIME.bless_test_results.TestStatus") + @mock.patch("CIME.bless_test_results.get_test_status_files") + def test_baseline_root_none(self, get_test_status_files, TestStatus, Case): + get_test_status_files.return_value = [ + "/tmp/cases/SMS.f19_g16.S.docker_gnu.12345/TestStatus", + "/tmp/cases/PET.f19_g16.S.docker-gnu.12345/TestStatus", + ] + + ts = TestStatus.return_value + ts.get_name.return_value = "SMS.f19_g16.S.docker_gnu" + ts.get_overall_test_status.return_value = ("PASS", "RUN") + ts.get_status.side_effect = ["FAIL"] + ["PASS"] * 9 + + case = Case.return_value.__enter__.return_value + case.get_value.side_effect = [None, None] + + success = bless_test_results( + "master", + None, + "/tmp/cases", + "gnu", + force=True, + ) + + assert not success + + @mock.patch("CIME.bless_test_results.Case") + @mock.patch("CIME.bless_test_results.TestStatus") + @mock.patch("CIME.bless_test_results.get_test_status_files") + def test_baseline_name_none(self, get_test_status_files, TestStatus, Case): + get_test_status_files.return_value = [ + "/tmp/cases/SMS.f19_g16.S.docker_gnu.12345/TestStatus", + "/tmp/cases/PET.f19_g16.S.docker-gnu.12345/TestStatus", + ] + + ts = TestStatus.return_value + ts.get_name.return_value = "SMS.f19_g16.S.docker_gnu" + ts.get_overall_test_status.return_value = ("PASS", "RUN") + ts.get_status.side_effect = ["FAIL"] + ["PASS"] * 9 + + case = Case.return_value.__enter__.return_value + case.get_value.side_effect = [None, None] + + success = bless_test_results( + None, + "/tmp/baselines", + "/tmp/cases", + "gnu", + force=True, + ) + + assert not success + + @mock.patch("CIME.bless_test_results.Case") + @mock.patch("CIME.bless_test_results.TestStatus") + @mock.patch("CIME.bless_test_results.get_test_status_files") + def test_exclude(self, get_test_status_files, TestStatus, Case): + get_test_status_files.return_value = [ + "/tmp/cases/SMS.f19_g16.S.docker_gnu.12345/TestStatus", + "/tmp/cases/PET.f19_g16.S.docker-gnu.12345/TestStatus", + ] + + ts = TestStatus.return_value + ts.get_name.return_value = "SMS.f19_g16.S.docker_gnu" + ts.get_overall_test_status.return_value = ("PASS", "RUN") + ts.get_status.side_effect = ["PASS", "PASS", "PASS", "PASS", "PASS"] + + case = Case.return_value.__enter__.return_value + + success = bless_test_results( + "master", + "/tmp/baseline", + "/tmp/cases", + "gnu", + force=True, + exclude="SMS", + ) + + assert success + + @mock.patch("CIME.bless_test_results.Case") + @mock.patch("CIME.bless_test_results.TestStatus") + @mock.patch("CIME.bless_test_results.get_test_status_files") + def test_multiple_files(self, get_test_status_files, TestStatus, Case): + get_test_status_files.return_value = [ + "/tmp/cases/SMS.f19_g16.S.docker_gnu.12345/TestStatus", + "/tmp/cases/SMS.f19_g16.S.docker-gnu.23456/TestStatus", + ] + + ts = TestStatus.return_value + ts.get_name.return_value = "SMS.f19_g16.S.docker_gnu" + ts.get_overall_test_status.return_value = ("PASS", "RUN") + ts.get_status.side_effect = ["PASS", "PASS", "PASS", "PASS", "PASS"] + + case = Case.return_value.__enter__.return_value + + success = bless_test_results( + "master", + "/tmp/baseline", + "/tmp/cases", + "gnu", + force=True, + ) + + assert success + + @mock.patch("CIME.bless_test_results.Case") + @mock.patch("CIME.bless_test_results.TestStatus") + @mock.patch("CIME.bless_test_results.get_test_status_files") + def test_bless_tests_no_match(self, get_test_status_files, TestStatus, Case): + get_test_status_files.return_value = [ + "/tmp/cases/SMS.f19_g16.S.docker_gnu/TestStatus", + "/tmp/cases/PET.f19_g16.S.docker_gnu/TestStatus", + ] + + ts = TestStatus.return_value + ts.get_name.return_value = "SMS.f19_g16.S.docker_gnu" + ts.get_overall_test_status.return_value = ("PASS", "RUN") + ts.get_status.side_effect = ["PASS"] * 10 + + case = Case.return_value.__enter__.return_value + + success = bless_test_results( + "master", + "/tmp/baseline", + "/tmp/cases", + "gnu", + force=True, + bless_tests=["SEQ"], + ) + + assert success + + @mock.patch("CIME.bless_test_results.Case") + @mock.patch("CIME.bless_test_results.TestStatus") + @mock.patch("CIME.bless_test_results.get_test_status_files") + def test_bless_all(self, get_test_status_files, TestStatus, Case): + get_test_status_files.return_value = [ + "/tmp/cases/SMS.f19_g16.S.docker_gnu/TestStatus", + ] + + ts = TestStatus.return_value + ts.get_name.return_value = "SMS.f19_g16.S.docker_gnu" + ts.get_overall_test_status.return_value = ("PASS", "RUN") + ts.get_status.side_effect = ["PASS", "PASS", "PASS", "PASS", "PASS"] + + case = Case.return_value.__enter__.return_value + + success = bless_test_results( + "master", + "/tmp/baseline", + "/tmp/cases", + "gnu", + force=True, + ) + + assert success + + def test_is_bless_needed_no_skip_fail(self): + ts = mock.MagicMock() + ts.get_status.side_effect = [ + "PASS", + ] + + broken_blesses = [] + + needed = is_bless_needed( + "SMS.f19_g16.A", ts, broken_blesses, "PASS", True, "RUN" + ) + + assert needed + assert broken_blesses == [] + + def test_is_bless_needed_overall_fail(self): + ts = mock.MagicMock() + ts.get_status.side_effect = [ + "PASS", + ] + + broken_blesses = [] + + needed = is_bless_needed( + "SMS.f19_g16.A", ts, broken_blesses, "FAIL", False, "RUN" + ) + + assert not needed + assert broken_blesses == [("SMS.f19_g16.A", "test did not pass")] + + def test_is_bless_needed_baseline_fail(self): + ts = mock.MagicMock() + ts.get_status.side_effect = ["PASS", "FAIL"] + + broken_blesses = [] + + needed = is_bless_needed( + "SMS.f19_g16.A", ts, broken_blesses, "PASS", False, "RUN" + ) + + assert needed + assert broken_blesses == [] + + def test_is_bless_needed_run_phase_fail(self): + ts = mock.MagicMock() + ts.get_status.side_effect = [ + "FAIL", + ] + + broken_blesses = [] + + needed = is_bless_needed( + "SMS.f19_g16.A", ts, broken_blesses, "PASS", False, "RUN" + ) + + assert not needed + assert broken_blesses == [("SMS.f19_g16.A", "run phase did not pass")] + + def test_is_bless_needed_no_run_phase(self): + ts = mock.MagicMock() + ts.get_status.side_effect = [None] + + broken_blesses = [] + + needed = is_bless_needed( + "SMS.f19_g16.A", ts, broken_blesses, "PASS", False, "RUN" + ) + + assert not needed + assert broken_blesses == [("SMS.f19_g16.A", "no run phase")] + + def test_is_bless_needed(self): + ts = mock.MagicMock() + ts.get_status.side_effect = ["PASS", "PASS"] + + broken_blesses = [] + + needed = is_bless_needed( + "SMS.f19_g16.A", ts, broken_blesses, "PASS", False, "RUN" + ) + + assert not needed From 89fa0525a10b3acc669bafae270a83bdeab46a6b Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Wed, 25 Oct 2023 02:07:11 -0700 Subject: [PATCH 48/57] Fixes failing test --- CIME/tests/test_unit_bless_test_results.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/CIME/tests/test_unit_bless_test_results.py b/CIME/tests/test_unit_bless_test_results.py index b0a549d3e1b..69e6de56629 100644 --- a/CIME/tests/test_unit_bless_test_results.py +++ b/CIME/tests/test_unit_bless_test_results.py @@ -742,13 +742,17 @@ def test_baseline_root_none(self, get_test_status_files, TestStatus, Case): assert not success + @mock.patch("CIME.bless_test_results.bless_namelists") @mock.patch("CIME.bless_test_results.Case") @mock.patch("CIME.bless_test_results.TestStatus") @mock.patch("CIME.bless_test_results.get_test_status_files") - def test_baseline_name_none(self, get_test_status_files, TestStatus, Case): + def test_baseline_name_none( + self, get_test_status_files, TestStatus, Case, bless_namelists + ): + bless_namelists.return_value = (True, "") + get_test_status_files.return_value = [ "/tmp/cases/SMS.f19_g16.S.docker_gnu.12345/TestStatus", - "/tmp/cases/PET.f19_g16.S.docker-gnu.12345/TestStatus", ] ts = TestStatus.return_value @@ -767,7 +771,7 @@ def test_baseline_name_none(self, get_test_status_files, TestStatus, Case): force=True, ) - assert not success + assert success @mock.patch("CIME.bless_test_results.Case") @mock.patch("CIME.bless_test_results.TestStatus") From d05da07ceecc4ae113c0ca84029043c9e57faaa7 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 26 Oct 2023 18:52:07 -0700 Subject: [PATCH 49/57] Moves skipping message to debug --- CIME/bless_test_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index 8817f1f22a4..1c1a4c0d6fa 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -294,7 +294,7 @@ def bless_test_results( excluded = exclude.match(test_name) if exclude else False if (not has_no_tests and not match_test_name) or excluded: - logger.info("Skipping {!r}".format(test_name)) + logger.debug("Skipping {!r}".format(test_name)) continue From a66c75e10b22393a093ce2b319e21f77ace2e3c0 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 26 Oct 2023 22:48:29 -0700 Subject: [PATCH 50/57] Revert "Updates MEMCOMP and TPUTCOMP to DIFF if they're checked and fail" This reverts commit a367ab9b76bb8edb11bec505872b00941d929a1b. --- CIME/test_status.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/CIME/test_status.py b/CIME/test_status.py index 5f306b7db0e..90714631eb8 100644 --- a/CIME/test_status.py +++ b/CIME/test_status.py @@ -460,7 +460,7 @@ def _get_overall_status_based_on_phases( if rv == TEST_PASS_STATUS: rv = NAMELIST_FAIL_STATUS - elif phase in [BASELINE_PHASE, THROUGHPUT_PHASE, MEMCOMP_PHASE]: + elif phase == BASELINE_PHASE: if rv in [NAMELIST_FAIL_STATUS, TEST_PASS_STATUS]: phase_responsible_for_status = phase rv = TEST_DIFF_STATUS @@ -512,9 +512,7 @@ def get_overall_test_status( >>> _test_helper2('PASS ERS.foo.A RUN\nFAIL ERS.foo.A TPUTCOMP') ('PASS', 'RUN') >>> _test_helper2('PASS ERS.foo.A RUN\nFAIL ERS.foo.A TPUTCOMP', check_throughput=True) - ('DIFF', 'TPUTCOMP') - >>> _test_helper2('PASS ERS.foo.A RUN\nFAIL ERS.foo.A MEMCOMP', check_memory=True) - ('DIFF', 'MEMCOMP') + ('FAIL', 'TPUTCOMP') >>> _test_helper2('PASS ERS.foo.A MODEL_BUILD\nPASS ERS.foo.A RUN\nFAIL ERS.foo.A NLCOMP') ('NLFAIL', 'RUN') >>> _test_helper2('PASS ERS.foo.A MODEL_BUILD\nPEND ERS.foo.A RUN\nFAIL ERS.foo.A NLCOMP') From c6e3442f714d98f8a99227eeca435d093d2dbde7 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 27 Oct 2023 00:01:22 -0700 Subject: [PATCH 51/57] Updates performance bless to be separate from hist/nml --- CIME/Tools/bless_test_results | 12 +- CIME/baselines/performance.py | 4 + CIME/bless_test_results.py | 47 +++----- CIME/tests/test_unit_bless_test_results.py | 125 +++++++++++---------- 4 files changed, 94 insertions(+), 94 deletions(-) diff --git a/CIME/Tools/bless_test_results b/CIME/Tools/bless_test_results index d2bc707d3c2..3ef83ce3232 100755 --- a/CIME/Tools/bless_test_results +++ b/CIME/Tools/bless_test_results @@ -108,6 +108,8 @@ def create_bless_options(parser): mutual_bless_group = bless_group.add_mutually_exclusive_group() + mutual_std_group = mutual_bless_group.add_mutually_exclusive_group() + mutual_bless_group.add_argument( "-n", "--namelists-only", action="store_true", help="Only analyze namelists." ) @@ -116,12 +118,14 @@ def create_bless_options(parser): "--hist-only", action="store_true", help="Only analyze history files." ) - mutual_bless_group.add_argument( - "--tput-only", action="store_true", help="Only analyze throughput." + mutual_perf_group = mutual_bless_group.add_mutually_exclusive_group() + + mutual_perf_group.add_argument( + "--bless-throughput", action="store_true", help="Bless throughput" ) - mutual_bless_group.add_argument( - "--mem-only", action="store_true", help="Only analyze memory." + mutual_perf_group.add_argument( + "--bless-memory", action="store_true", help="Bless memory" ) diff --git a/CIME/baselines/performance.py b/CIME/baselines/performance.py index 8aa76083a82..0fcb729d1d5 100644 --- a/CIME/baselines/performance.py +++ b/CIME/baselines/performance.py @@ -129,6 +129,8 @@ def perf_write_baseline(case, basegen_dir, throughput=True, memory=True): write_baseline_file(baseline_file, tput) + logger.info("Updated throughput baseline to {!s}".format(tput)) + if memory: try: mem = perf_get_memory(case, config) @@ -139,6 +141,8 @@ def perf_write_baseline(case, basegen_dir, throughput=True, memory=True): write_baseline_file(baseline_file, mem) + logger.info("Updated memory usage baseline to {!s}".format(mem)) + def load_coupler_customization(case): """ diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index 1c1a4c0d6fa..38ef14a175c 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -22,7 +22,7 @@ logger = logging.getLogger(__name__) -def bless_throughput( +def _bless_throughput( case, test_name, baseline_root, @@ -32,6 +32,7 @@ def bless_throughput( ): success = True reason = None + below_threshold = False baseline_dir = os.path.join( baseline_root, baseline_name, case.get_value("CASEBASEID") @@ -42,20 +43,10 @@ def bless_throughput( case, baseline_dir=baseline_dir ) except FileNotFoundError as e: - success = False - comment = f"Could not read throughput file: {e!s}" except Exception as e: - success = False - comment = f"Error comparing throughput baseline: {e!s}" - # fail early - if not success: - logger.info(comment) - - return success, comment - if below_threshold: logger.info("Throughput diff appears to have been already resolved.") else: @@ -74,7 +65,7 @@ def bless_throughput( return success, reason -def bless_memory( +def _bless_memory( case, test_name, baseline_root, @@ -84,6 +75,7 @@ def bless_memory( ): success = True reason = None + below_threshold = False baseline_dir = os.path.join( baseline_root, baseline_name, case.get_value("CASEBASEID") @@ -94,20 +86,10 @@ def bless_memory( case, baseline_dir=baseline_dir ) except FileNotFoundError as e: - success = False - comment = f"Could not read memory usage file: {e!s}" except Exception as e: - success = False - comment = f"Error comparing memory baseline: {e!s}" - # fail early - if not success: - logger.info(comment) - - return success, comment - if below_threshold: logger.info("Memory usage diff appears to have been already resolved.") else: @@ -221,8 +203,6 @@ def bless_test_results( test_id=None, namelists_only=False, hist_only=False, - tput_only=False, - mem_only=False, report_only=False, force=False, pes_file=None, @@ -231,9 +211,11 @@ def bless_test_results( new_test_root=None, new_test_id=None, exclude=None, + bless_throughput=False, + bless_memory=False, **_, # Capture all for extra ): - bless_all = not (namelists_only | hist_only | tput_only | mem_only) + bless_all = not ((namelists_only | hist_only) or (bless_throughput | bless_memory)) test_status_files = get_test_status_files(test_root, compiler, test_id=test_id) @@ -301,8 +283,8 @@ def bless_test_results( overall_result, phase = ts.get_overall_test_status( ignore_namelists=True, ignore_memleak=True, - check_throughput=True, - check_memory=True, + check_throughput=False, + check_memory=False, ) # See if we need to bless namelist @@ -326,13 +308,13 @@ def bless_test_results( if hist_only or bless_all: hist_bless = bless_needed - if tput_only or bless_all: + if bless_throughput: tput_bless = bless_needed if not tput_bless: tput_bless = ts.get_status(THROUGHPUT_PHASE) != TEST_PASS_STATUS - if mem_only or bless_all: + if bless_memory: mem_bless = bless_needed if not mem_bless: @@ -366,8 +348,9 @@ def bless_test_results( if baseline_name is None: baseline_name_resolved = case.get_value("BASELINE_NAME_CMP") if not baseline_name_resolved: + cime_root = CIME.utils.get_cime_root() baseline_name_resolved = CIME.utils.get_current_branch( - repo=CIME.utils.get_cime_root() + repo=cime_root ) else: baseline_name_resolved = baseline_name @@ -423,7 +406,7 @@ def bless_test_results( broken_blesses.append((test_name, reason)) if tput_bless: - success, reason = bless_throughput( + success, reason = _bless_throughput( case, test_name, baseline_root_resolved, @@ -436,7 +419,7 @@ def bless_test_results( broken_blesses.append((test_name, reason)) if mem_bless: - success, reason = bless_memory( + success, reason = _bless_memory( case, test_name, baseline_root_resolved, diff --git a/CIME/tests/test_unit_bless_test_results.py b/CIME/tests/test_unit_bless_test_results.py index 69e6de56629..e3147dcc340 100644 --- a/CIME/tests/test_unit_bless_test_results.py +++ b/CIME/tests/test_unit_bless_test_results.py @@ -6,8 +6,8 @@ from CIME.bless_test_results import ( bless_test_results, - bless_throughput, - bless_memory, + _bless_throughput, + _bless_memory, bless_history, bless_namelists, is_bless_needed, @@ -232,7 +232,7 @@ def test_bless_memory_force_error( case = mock.MagicMock() - success, comment = bless_memory( + success, comment = _bless_memory( case, "SMS.f19_g16.S", "/tmp/baselines", "master", False, True ) @@ -252,7 +252,7 @@ def test_bless_memory_force( case = mock.MagicMock() - success, comment = bless_memory( + success, comment = _bless_memory( case, "SMS.f19_g16.S", "/tmp/baselines", "master", False, True ) @@ -266,38 +266,44 @@ def test_bless_memory_report_only(self, perf_compare_memory_baseline): case = mock.MagicMock() - success, comment = bless_memory( + success, comment = _bless_memory( case, "SMS.f19_g16.S", "/tmp/baselines", "master", True, False ) assert success assert comment is None + @mock.patch("CIME.bless_test_results.perf_write_baseline") @mock.patch("CIME.bless_test_results.perf_compare_memory_baseline") - def test_bless_memory_general_error(self, perf_compare_memory_baseline): + def test_bless_memory_general_error( + self, perf_compare_memory_baseline, perf_write_baseline + ): perf_compare_memory_baseline.side_effect = Exception case = mock.MagicMock() - success, comment = bless_memory( - case, "SMS.f19_g16.S", "/tmp/baselines", "master", False, False + success, comment = _bless_memory( + case, "SMS.f19_g16.S", "/tmp/baselines", "master", False, True ) - assert not success - assert comment == "Error comparing memory baseline: " + assert success + assert comment is None + @mock.patch("CIME.bless_test_results.perf_write_baseline") @mock.patch("CIME.bless_test_results.perf_compare_memory_baseline") - def test_bless_memory_file_not_found_error(self, perf_compare_memory_baseline): + def test_bless_memory_file_not_found_error( + self, perf_compare_memory_baseline, perf_write_baseline + ): perf_compare_memory_baseline.side_effect = FileNotFoundError case = mock.MagicMock() - success, comment = bless_memory( - case, "SMS.f19_g16.S", "/tmp/baselines", "master", False, False + success, comment = _bless_memory( + case, "SMS.f19_g16.S", "/tmp/baselines", "master", False, True ) - assert not success - assert comment == "Could not read memory usage file: " + assert success + assert comment is None @mock.patch("CIME.bless_test_results.perf_compare_memory_baseline") def test_bless_memory(self, perf_compare_memory_baseline): @@ -305,7 +311,7 @@ def test_bless_memory(self, perf_compare_memory_baseline): case = mock.MagicMock() - success, comment = bless_memory( + success, comment = _bless_memory( case, "SMS.f19_g16.S", "/tmp/baselines", "master", False, False ) @@ -322,7 +328,7 @@ def test_bless_throughput_force_error( case = mock.MagicMock() - success, comment = bless_throughput( + success, comment = _bless_throughput( case, "SMS.f19_g16.S", "/tmp/baselines", "master", False, True ) @@ -339,7 +345,7 @@ def test_bless_throughput_force( case = mock.MagicMock() - success, comment = bless_throughput( + success, comment = _bless_throughput( case, "SMS.f19_g16.S", "/tmp/baselines", "master", False, True ) @@ -353,7 +359,7 @@ def test_bless_throughput_report_only(self, perf_compare_throughput_baseline): case = mock.MagicMock() - success, comment = bless_throughput( + success, comment = _bless_throughput( case, "SMS.f19_g16.S", "/tmp/baselines", "master", True, False ) @@ -366,27 +372,30 @@ def test_bless_throughput_general_error(self, perf_compare_throughput_baseline): case = mock.MagicMock() - success, comment = bless_throughput( - case, "SMS.f19_g16.S", "/tmp/baselines", "master", False, False + success, comment = _bless_throughput( + case, "SMS.f19_g16.S", "/tmp/baselines", "master", False, True ) - assert not success - assert comment == "Error comparing throughput baseline: " + assert success + assert comment is None + @mock.patch("CIME.bless_test_results.perf_write_baseline") @mock.patch("CIME.bless_test_results.perf_compare_throughput_baseline") def test_bless_throughput_file_not_found_error( - self, perf_compare_throughput_baseline + self, + perf_compare_throughput_baseline, + perf_write_baseline, ): perf_compare_throughput_baseline.side_effect = FileNotFoundError case = mock.MagicMock() - success, comment = bless_throughput( - case, "SMS.f19_g16.S", "/tmp/baselines", "master", False, False + success, comment = _bless_throughput( + case, "SMS.f19_g16.S", "/tmp/baselines", "master", False, True ) - assert not success - assert comment == "Could not read throughput file: " + assert success + assert comment is None @mock.patch("CIME.bless_test_results.perf_compare_throughput_baseline") def test_bless_throughput(self, perf_compare_throughput_baseline): @@ -394,13 +403,13 @@ def test_bless_throughput(self, perf_compare_throughput_baseline): case = mock.MagicMock() - success, comment = bless_throughput( + success, comment = _bless_throughput( case, "SMS.f19_g16.S", "/tmp/baselines", "master", False, False ) assert success - @mock.patch("CIME.bless_test_results.bless_memory") + @mock.patch("CIME.bless_test_results._bless_memory") @mock.patch("CIME.bless_test_results.Case") @mock.patch("CIME.bless_test_results.TestStatus") @mock.patch("CIME.bless_test_results.get_test_status_files") @@ -409,7 +418,7 @@ def test_bless_memory_only( get_test_status_files, TestStatus, Case, - bless_memory, + _bless_memory, ): get_test_status_files.return_value = [ "/tmp/cases/SMS.f19_g16.S.docker_gnu/TestStatus", @@ -422,7 +431,7 @@ def test_bless_memory_only( case = Case.return_value.__enter__.return_value - bless_memory.return_value = (True, "") + _bless_memory.return_value = (True, "") success = bless_test_results( "master", @@ -430,13 +439,13 @@ def test_bless_memory_only( "/tmp/cases", "gnu", force=True, - mem_only=True, + bless_memory=True, ) assert success - bless_memory.assert_called() + _bless_memory.assert_called() - @mock.patch("CIME.bless_test_results.bless_throughput") + @mock.patch("CIME.bless_test_results._bless_throughput") @mock.patch("CIME.bless_test_results.Case") @mock.patch("CIME.bless_test_results.TestStatus") @mock.patch("CIME.bless_test_results.get_test_status_files") @@ -445,7 +454,7 @@ def test_bless_throughput_only( get_test_status_files, TestStatus, Case, - bless_throughput, + _bless_throughput, ): get_test_status_files.return_value = [ "/tmp/cases/SMS.f19_g16.S.docker_gnu/TestStatus", @@ -454,11 +463,11 @@ def test_bless_throughput_only( ts = TestStatus.return_value ts.get_name.return_value = "SMS.f19_g16.S.docker_gnu" ts.get_overall_test_status.return_value = ("PASS", "RUN") - ts.get_status.side_effect = ["PASS", "FAIL"] + ts.get_status.side_effect = ["PASS", "PASS", "FAIL"] case = Case.return_value.__enter__.return_value - bless_throughput.return_value = (True, "") + _bless_throughput.return_value = (True, "") success = bless_test_results( "master", @@ -466,11 +475,11 @@ def test_bless_throughput_only( "/tmp/cases", "gnu", force=True, - tput_only=True, + bless_throughput=True, ) assert success - bless_throughput.assert_called() + _bless_throughput.assert_called() @mock.patch("CIME.bless_test_results.bless_namelists") @mock.patch("CIME.bless_test_results.Case") @@ -571,8 +580,8 @@ def test_specific(self, get_test_status_files, TestStatus, Case): assert success - @mock.patch("CIME.bless_test_results.bless_memory") - @mock.patch("CIME.bless_test_results.bless_throughput") + @mock.patch("CIME.bless_test_results._bless_memory") + @mock.patch("CIME.bless_test_results._bless_throughput") @mock.patch("CIME.bless_test_results.bless_history") @mock.patch("CIME.bless_test_results.bless_namelists") @mock.patch("CIME.bless_test_results.Case") @@ -585,12 +594,12 @@ def test_bless_tests_results_homme( Case, bless_namelists, bless_history, - bless_throughput, - bless_memory, + _bless_throughput, + _bless_memory, ): - bless_memory.return_value = (False, "") + _bless_memory.return_value = (False, "") - bless_throughput.return_value = (False, "") + _bless_throughput.return_value = (False, "") bless_history.return_value = (False, "") @@ -619,8 +628,8 @@ def test_bless_tests_results_homme( assert not success - @mock.patch("CIME.bless_test_results.bless_memory") - @mock.patch("CIME.bless_test_results.bless_throughput") + @mock.patch("CIME.bless_test_results._bless_memory") + @mock.patch("CIME.bless_test_results._bless_throughput") @mock.patch("CIME.bless_test_results.bless_history") @mock.patch("CIME.bless_test_results.bless_namelists") @mock.patch("CIME.bless_test_results.Case") @@ -633,12 +642,12 @@ def test_bless_tests_results_fail( Case, bless_namelists, bless_history, - bless_throughput, - bless_memory, + _bless_throughput, + _bless_memory, ): - bless_memory.return_value = (False, "") + _bless_memory.return_value = (False, "") - bless_throughput.return_value = (False, "") + _bless_throughput.return_value = (False, "") bless_history.return_value = (False, "") @@ -667,8 +676,8 @@ def test_bless_tests_results_fail( assert not success - @mock.patch("CIME.bless_test_results.bless_memory") - @mock.patch("CIME.bless_test_results.bless_throughput") + @mock.patch("CIME.bless_test_results._bless_memory") + @mock.patch("CIME.bless_test_results._bless_throughput") @mock.patch("CIME.bless_test_results.bless_history") @mock.patch("CIME.bless_test_results.bless_namelists") @mock.patch("CIME.bless_test_results.Case") @@ -681,12 +690,12 @@ def test_no_skip_pass( Case, bless_namelists, bless_history, - bless_throughput, - bless_memory, + _bless_throughput, + _bless_memory, ): - bless_memory.return_value = (True, "") + _bless_memory.return_value = (True, "") - bless_throughput.return_value = (True, "") + _bless_throughput.return_value = (True, "") bless_history.return_value = (True, "") From 2f6d7c118e4aebe33dec21d306d8254ef121dae5 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Wed, 20 Sep 2023 14:45:04 -0700 Subject: [PATCH 52/57] Updates MEMCOMP and TPUTCOMP to DIFF if they're checked and fail --- CIME/test_status.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CIME/test_status.py b/CIME/test_status.py index 90714631eb8..5f306b7db0e 100644 --- a/CIME/test_status.py +++ b/CIME/test_status.py @@ -460,7 +460,7 @@ def _get_overall_status_based_on_phases( if rv == TEST_PASS_STATUS: rv = NAMELIST_FAIL_STATUS - elif phase == BASELINE_PHASE: + elif phase in [BASELINE_PHASE, THROUGHPUT_PHASE, MEMCOMP_PHASE]: if rv in [NAMELIST_FAIL_STATUS, TEST_PASS_STATUS]: phase_responsible_for_status = phase rv = TEST_DIFF_STATUS @@ -512,7 +512,9 @@ def get_overall_test_status( >>> _test_helper2('PASS ERS.foo.A RUN\nFAIL ERS.foo.A TPUTCOMP') ('PASS', 'RUN') >>> _test_helper2('PASS ERS.foo.A RUN\nFAIL ERS.foo.A TPUTCOMP', check_throughput=True) - ('FAIL', 'TPUTCOMP') + ('DIFF', 'TPUTCOMP') + >>> _test_helper2('PASS ERS.foo.A RUN\nFAIL ERS.foo.A MEMCOMP', check_memory=True) + ('DIFF', 'MEMCOMP') >>> _test_helper2('PASS ERS.foo.A MODEL_BUILD\nPASS ERS.foo.A RUN\nFAIL ERS.foo.A NLCOMP') ('NLFAIL', 'RUN') >>> _test_helper2('PASS ERS.foo.A MODEL_BUILD\nPEND ERS.foo.A RUN\nFAIL ERS.foo.A NLCOMP') From 9227c531789cfcd37ec62bd0769ec89cd5c9aa41 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Mon, 30 Oct 2023 12:35:09 -0700 Subject: [PATCH 53/57] Updates bless perf flags --- CIME/Tools/bless_test_results | 16 +++++-- CIME/bless_test_results.py | 17 +++++-- CIME/tests/test_unit_bless_test_results.py | 55 ++++++++++++++++++++-- 3 files changed, 74 insertions(+), 14 deletions(-) diff --git a/CIME/Tools/bless_test_results b/CIME/Tools/bless_test_results index 3ef83ce3232..eb9663dcf9a 100755 --- a/CIME/Tools/bless_test_results +++ b/CIME/Tools/bless_test_results @@ -108,8 +108,6 @@ def create_bless_options(parser): mutual_bless_group = bless_group.add_mutually_exclusive_group() - mutual_std_group = mutual_bless_group.add_mutually_exclusive_group() - mutual_bless_group.add_argument( "-n", "--namelists-only", action="store_true", help="Only analyze namelists." ) @@ -118,14 +116,22 @@ def create_bless_options(parser): "--hist-only", action="store_true", help="Only analyze history files." ) - mutual_perf_group = mutual_bless_group.add_mutually_exclusive_group() + mutual_perf_group = bless_group.add_mutually_exclusive_group() mutual_perf_group.add_argument( - "--bless-throughput", action="store_true", help="Bless throughput" + "--bless-tput", + action="store_true", + help="Bless throughput, use `--bless-perf` to bless throughput and memory", ) mutual_perf_group.add_argument( - "--bless-memory", action="store_true", help="Bless memory" + "--bless-mem", + action="store_true", + help="Bless memory, use `--bless-perf` to bless throughput and memory", + ) + + bless_group.add_argument( + "--bless-perf", action="store_true", help="Bless both throughput and memory" ) diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index 38ef14a175c..c6f0754ebd8 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -211,11 +211,12 @@ def bless_test_results( new_test_root=None, new_test_id=None, exclude=None, - bless_throughput=False, - bless_memory=False, + bless_tput=False, + bless_mem=False, + bless_perf=False, **_, # Capture all for extra ): - bless_all = not ((namelists_only | hist_only) or (bless_throughput | bless_memory)) + bless_all = not (namelists_only | hist_only) test_status_files = get_test_status_files(test_root, compiler, test_id=test_id) @@ -308,13 +309,13 @@ def bless_test_results( if hist_only or bless_all: hist_bless = bless_needed - if bless_throughput: + if bless_tput or bless_perf: tput_bless = bless_needed if not tput_bless: tput_bless = ts.get_status(THROUGHPUT_PHASE) != TEST_PASS_STATUS - if bless_memory: + if bless_mem or bless_perf: mem_bless = bless_needed if not mem_bless: @@ -328,6 +329,12 @@ def bless_test_results( ) ) else: + logger.debug("Determined blesses for {!r}".format(test_name)) + logger.debug("nl_bless = {}".format(nl_bless)) + logger.debug("hist_bless = {}".format(hist_bless)) + logger.debug("tput_bless = {}".format(tput_bless)) + logger.debug("mem_bless = {}".format(mem_bless)) + logger.info( "###############################################################################" ) diff --git a/CIME/tests/test_unit_bless_test_results.py b/CIME/tests/test_unit_bless_test_results.py index e3147dcc340..fe9003d1bd2 100644 --- a/CIME/tests/test_unit_bless_test_results.py +++ b/CIME/tests/test_unit_bless_test_results.py @@ -409,6 +409,48 @@ def test_bless_throughput(self, perf_compare_throughput_baseline): assert success + @mock.patch("CIME.bless_test_results._bless_throughput") + @mock.patch("CIME.bless_test_results._bless_memory") + @mock.patch("CIME.bless_test_results.Case") + @mock.patch("CIME.bless_test_results.TestStatus") + @mock.patch("CIME.bless_test_results.get_test_status_files") + def test_bless_perf( + self, + get_test_status_files, + TestStatus, + Case, + _bless_memory, + _bless_throughput, + ): + get_test_status_files.return_value = [ + "/tmp/cases/SMS.f19_g16.S.docker_gnu/TestStatus", + ] + + ts = TestStatus.return_value + ts.get_name.return_value = "SMS.f19_g16.S.docker_gnu" + ts.get_overall_test_status.return_value = ("PASS", "RUN") + ts.get_status.side_effect = ["PASS", "PASS", "PASS", "FAIL", "FAIL"] + + case = Case.return_value.__enter__.return_value + + _bless_memory.return_value = (True, "") + + _bless_throughput.return_value = (True, "") + + success = bless_test_results( + "master", + "/tmp/baseline", + "/tmp/cases", + "gnu", + force=True, + bless_perf=True, + ) + + assert success + _bless_memory.assert_called() + _bless_throughput.assert_called() + + @mock.patch("CIME.bless_test_results._bless_throughput") @mock.patch("CIME.bless_test_results._bless_memory") @mock.patch("CIME.bless_test_results.Case") @mock.patch("CIME.bless_test_results.TestStatus") @@ -419,6 +461,7 @@ def test_bless_memory_only( TestStatus, Case, _bless_memory, + _bless_throughput, ): get_test_status_files.return_value = [ "/tmp/cases/SMS.f19_g16.S.docker_gnu/TestStatus", @@ -427,7 +470,7 @@ def test_bless_memory_only( ts = TestStatus.return_value ts.get_name.return_value = "SMS.f19_g16.S.docker_gnu" ts.get_overall_test_status.return_value = ("PASS", "RUN") - ts.get_status.side_effect = ["PASS", "FAIL"] + ts.get_status.side_effect = ["PASS", "PASS", "PASS", "FAIL"] case = Case.return_value.__enter__.return_value @@ -439,13 +482,15 @@ def test_bless_memory_only( "/tmp/cases", "gnu", force=True, - bless_memory=True, + bless_mem=True, ) assert success _bless_memory.assert_called() + _bless_throughput.assert_not_called() @mock.patch("CIME.bless_test_results._bless_throughput") + @mock.patch("CIME.bless_test_results._bless_memory") @mock.patch("CIME.bless_test_results.Case") @mock.patch("CIME.bless_test_results.TestStatus") @mock.patch("CIME.bless_test_results.get_test_status_files") @@ -454,6 +499,7 @@ def test_bless_throughput_only( get_test_status_files, TestStatus, Case, + _bless_memory, _bless_throughput, ): get_test_status_files.return_value = [ @@ -463,7 +509,7 @@ def test_bless_throughput_only( ts = TestStatus.return_value ts.get_name.return_value = "SMS.f19_g16.S.docker_gnu" ts.get_overall_test_status.return_value = ("PASS", "RUN") - ts.get_status.side_effect = ["PASS", "PASS", "FAIL"] + ts.get_status.side_effect = ["PASS", "PASS", "PASS", "FAIL"] case = Case.return_value.__enter__.return_value @@ -475,10 +521,11 @@ def test_bless_throughput_only( "/tmp/cases", "gnu", force=True, - bless_throughput=True, + bless_tput=True, ) assert success + _bless_memory.assert_not_called() _bless_throughput.assert_called() @mock.patch("CIME.bless_test_results.bless_namelists") From f7868d3a478148dbeeb66fefb32e334eb6c66757 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Mon, 30 Oct 2023 17:12:11 -0700 Subject: [PATCH 54/57] Adds commit hash and date to baseline files --- CIME/baselines/performance.py | 7 ++++++- CIME/tests/test_unit_system_tests.py | 11 +++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/CIME/baselines/performance.py b/CIME/baselines/performance.py index 0fcb729d1d5..fe940309b28 100644 --- a/CIME/baselines/performance.py +++ b/CIME/baselines/performance.py @@ -4,7 +4,7 @@ import gzip import logging from CIME.config import Config -from CIME.utils import expect +from CIME.utils import expect, get_src_root, get_current_commit, get_timestamp logger = logging.getLogger(__name__) @@ -238,7 +238,12 @@ def write_baseline_file(baseline_file, value): value : str Value to write. """ + commit_hash = get_current_commit(repo=get_src_root()) + + timestamp = get_timestamp(timestamp_format="%Y-%m-%d_%H:%M:%S") + with open(baseline_file, "w") as fd: + fd.write(f"# sha:{commit_hash} date: {timestamp}\n") fd.write(value) diff --git a/CIME/tests/test_unit_system_tests.py b/CIME/tests/test_unit_system_tests.py index 99486178f36..609460fe9c0 100644 --- a/CIME/tests/test_unit_system_tests.py +++ b/CIME/tests/test_unit_system_tests.py @@ -3,6 +3,7 @@ import os import tempfile import gzip +import re from re import A import unittest from unittest import mock @@ -508,14 +509,16 @@ def test_generate_baseline(self): with open(baseline_dir / "cpl-tput.log") as fd: lines = fd.readlines() - assert len(lines) == 1 - assert lines[0] == "719.635" + assert len(lines) == 2 + assert re.match("# sha:.* date:.*", lines[0]) + assert lines[1] == "719.635" with open(baseline_dir / "cpl-mem.log") as fd: lines = fd.readlines() - assert len(lines) == 1 - assert lines[0] == "1673.89" + assert len(lines) == 2 + assert re.match("# sha:.* date:.*", lines[0]) + assert lines[1] == "1673.89" def test_kwargs(self): case = mock.MagicMock() From 60d9fdc2a45061bc4e1c4b452a9a2c0fbd32ca27 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Wed, 1 Nov 2023 21:32:39 -0600 Subject: [PATCH 55/57] Make gpu flags and case env vars visible to MPAS dycore --- CIME/Tools/Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CIME/Tools/Makefile b/CIME/Tools/Makefile index ff8bb42ce53..aaa73cdb490 100644 --- a/CIME/Tools/Makefile +++ b/CIME/Tools/Makefile @@ -562,8 +562,10 @@ ifdef MPAS_LIBDIR # is named libmpas.a), but adding the PHONY declaration provides an extra bit of safety .PHONY: libmpas libmpas: cam_abortutils.o physconst.o - $(MAKE) -C $(MPAS_LIBDIR) CC="$(CC)" FC="$(FC)" PIODEF="$(PIODEF)" FFLAGS='$(FREEFLAGS) $(FFLAGS)' \ - FCINCLUDES='$(INCLDIR) $(INCS) -I$(ABS_INSTALL_SHAREDPATH)/include -I$(ABS_ESMF_PATH)/include' + $(MAKE) -C $(MPAS_LIBDIR) CC="$(CC)" FC="$(FC)" PIODEF="$(PIODEF)" \ + FFLAGS='$(FREEFLAGS) $(FFLAGS)' GPUFLAGS='$(GPUFLAGS)' \ + CASEROOT='$(CASEROOT)' COMPILER='$(COMPILER)' MACH='$(MACH)' \ + FCINCLUDES='$(INCLDIR) $(INCS) -I$(ABS_INSTALL_SHAREDPATH)/include -I$(ABS_ESMF_PATH)/include' dyn_comp.o: libmpas dyn_grid.o: libmpas From a9e1dc2d14dc72c3d3410da20c9018c532eb8f68 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Wed, 1 Nov 2023 21:39:20 -0600 Subject: [PATCH 56/57] Add some comments --- CIME/Tools/Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CIME/Tools/Makefile b/CIME/Tools/Makefile index aaa73cdb490..c663b1edf26 100644 --- a/CIME/Tools/Makefile +++ b/CIME/Tools/Makefile @@ -561,6 +561,10 @@ ifdef MPAS_LIBDIR # this isn't necessary, since libmpas should never be an actual file (the library that is created # is named libmpas.a), but adding the PHONY declaration provides an extra bit of safety .PHONY: libmpas +# The CASEROOT, COMPILER and MACH are added so that the Depends file could be visible to +# the MPAS dycore. +# The GPUFLAGS is added so that the GPU flags defined in ccs_config_cesm could also be +# used to build the MPAS dycore if needed. libmpas: cam_abortutils.o physconst.o $(MAKE) -C $(MPAS_LIBDIR) CC="$(CC)" FC="$(FC)" PIODEF="$(PIODEF)" \ FFLAGS='$(FREEFLAGS) $(FFLAGS)' GPUFLAGS='$(GPUFLAGS)' \ From 06fa46ab1571edcffef6a4f9084ad5d1d2adc1ba Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Thu, 2 Nov 2023 18:34:23 -0500 Subject: [PATCH 57/57] Add --ignore-namelists to Jenkins scripts --- CIME/Tools/jenkins_generic_job | 9 +++++++++ CIME/jenkins_generic_job.py | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CIME/Tools/jenkins_generic_job b/CIME/Tools/jenkins_generic_job index 0c1a05a128e..9787aedab13 100755 --- a/CIME/Tools/jenkins_generic_job +++ b/CIME/Tools/jenkins_generic_job @@ -180,6 +180,12 @@ OR help="Do not fail if there are memleaks", ) + parser.add_argument( + "--ignore-namelists", + action="store_true", + help="Do not fail if there are namelist diffs", + ) + parser.add_argument( "--save-timing", action="store_true", @@ -265,6 +271,7 @@ OR args.check_throughput, args.check_memory, args.ignore_memleak, + args.ignore_namelists, args.save_timing, args.pes_file, args.jenkins_id, @@ -296,6 +303,7 @@ def _main_func(description): check_throughput, check_memory, ignore_memleak, + ignore_namelists, save_timing, pes_file, jenkins_id, @@ -325,6 +333,7 @@ def _main_func(description): check_throughput, check_memory, ignore_memleak, + ignore_namelists, save_timing, pes_file, jenkins_id, diff --git a/CIME/jenkins_generic_job.py b/CIME/jenkins_generic_job.py index e89e7ec2e9c..d68bc2b007c 100644 --- a/CIME/jenkins_generic_job.py +++ b/CIME/jenkins_generic_job.py @@ -279,6 +279,7 @@ def jenkins_generic_job( check_throughput, check_memory, ignore_memleak, + ignore_namelists, save_timing, pes_file, jenkins_id, @@ -421,7 +422,7 @@ def jenkins_generic_job( no_wait=not use_batch, # wait if using queue check_throughput=check_throughput, check_memory=check_memory, - ignore_namelists=False, # don't ignore namelist diffs + ignore_namelists=ignore_namelists, ignore_memleak=ignore_memleak, cdash_build_name=cdash_build_name, cdash_project=cdash_project,