From 698b373653f71831997560a5e31edb8300cd7e82 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 15 Sep 2023 13:26:58 -0700 Subject: [PATCH 01/31] Removes default coverage reporting and offloads to actions --- .github/workflows/testing.yml | 4 ++-- setup.cfg | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index e92b0b8c337..949f4c04fa6 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -129,7 +129,7 @@ jobs: init_cime - pytest -vvv --machine docker --no-fortran-run CIME/tests/test_unit* + pytest -vvv --cov=CIME --machine docker --no-fortran-run CIME/tests/test_unit* # Run system tests system-testing: @@ -182,7 +182,7 @@ jobs: conda activate base - pytest -vvv --machine docker --no-fortran-run --no-teardown CIME/tests/test_sys* + pytest -vvv --cov=CIME --machine docker --no-fortran-run --no-teardown CIME/tests/test_sys* - name: Create testing log archive if: ${{ failure() }} shell: bash diff --git a/setup.cfg b/setup.cfg index 772767f44b9..1c4058ebd85 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,7 +16,6 @@ console_scripts = [tool:pytest] junit_family=xunit2 -addopts = --cov=CIME --cov-report term-missing --cov-report html:test_coverage/html --cov-report xml:test_coverage/coverage.xml -s python_files = test_*.py testpaths = CIME/tests From e144fea5b591ef21cc11e6134552be906545ff81 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 15 Sep 2023 13:29:10 -0700 Subject: [PATCH 02/31] Adds test_coverage to ignore list --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 58e9dd92b66..f6351cf8996 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ scripts/Tools/JENKINS* components libraries share +test_coverage/** From 2245e742ee274b58dee13345bcec6feb495387a8 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Tue, 19 Sep 2023 17:06:58 -0700 Subject: [PATCH 03/31] Adds similar line as TPUTCOMP to inform user of difference --- CIME/SystemTests/system_tests_common.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index f15fbe959e0..7f9bbbd338e 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -775,6 +775,10 @@ def _compare_memory(self): tolerance = self._case.get_value("TEST_MEMLEAK_TOLERANCE") if tolerance is None: tolerance = 0.1 + comment = "MEMCOMP: Memory usage highwater has changed by {:.2f}% relative to baseline".format( + diff * 100 + ) + append_testlog(comment, self._orig_caseroot) if ( diff < tolerance and self._test_status.get_status(MEMCOMP_PHASE) is None From a6b8d8f0d56849c8cf61f042bce157f71456feb1 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Wed, 20 Sep 2023 13:07:09 -0700 Subject: [PATCH 04/31] Updates how performance baselines are written and compared to tests --- CIME/SystemTests/system_tests_common.py | 344 ++++++++++++++++++------ CIME/tests/test_unit_system_tests.py | 267 ++++++++++++++++++ 2 files changed, 533 insertions(+), 78 deletions(-) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index 7f9bbbd338e..eb367e33cee 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -751,6 +751,9 @@ def _get_latest_cpl_logs(self): return lastcpllogs def _compare_memory(self): + """ + Compares current test memory usage to baseline. + """ with self._test_status: # compare memory usage to baseline baseline_name = self._case.get_value("BASECMP_CASE") @@ -758,44 +761,68 @@ def _compare_memory(self): self._case.get_value("BASELINE_ROOT"), baseline_name ) newestcpllogfiles = self._get_latest_cpl_logs() - if len(newestcpllogfiles) > 0: - memlist = self._get_mem_usage(newestcpllogfiles[0]) + + # do we need this loop? when is there more than one coupler file? for cpllog in newestcpllogfiles: - m = re.search(r"/({}.*.log).*.gz".format(self._cpllog), cpllog) - if m is not None: - baselog = os.path.join(basecmp_dir, m.group(1)) + ".gz" - if baselog is None or not os.path.isfile(baselog): - # for backward compatibility - baselog = os.path.join(basecmp_dir, self._cpllog + ".log") - if os.path.isfile(baselog) and len(memlist) > 3: - blmem = self._get_mem_usage(baselog) - blmem = 0 if blmem == [] else blmem[-1][1] - curmem = memlist[-1][1] - diff = 0.0 if blmem == 0 else (curmem - blmem) / blmem - tolerance = self._case.get_value("TEST_MEMLEAK_TOLERANCE") - if tolerance is None: - tolerance = 0.1 - comment = "MEMCOMP: Memory usage highwater has changed by {:.2f}% relative to baseline".format( - diff * 100 - ) - append_testlog(comment, self._orig_caseroot) - if ( - diff < tolerance - and self._test_status.get_status(MEMCOMP_PHASE) is None - ): - self._test_status.set_status(MEMCOMP_PHASE, TEST_PASS_STATUS) - elif ( - self._test_status.get_status(MEMCOMP_PHASE) != TEST_FAIL_STATUS - ): - comment = "Error: Memory usage increase >{:d}% from baseline's {:f} to {:f}".format( - int(tolerance * 100), blmem, curmem - ) - self._test_status.set_status( - MEMCOMP_PHASE, TEST_FAIL_STATUS, comments=comment - ) - append_testlog(comment, self._orig_caseroot) + try: + baseline_mem = self._read_baseline_mem(basecmp_dir) + except FileNotFoundError as e: + logger.debug("Could not read baseline memory usage: %s", e) + + continue + else: + self._compare_current_memory(baseline_mem, cpllog) + + def _compare_current_memory(self, baseline, cpllog): + """ + Compares current test memory usage against ``baseline``. + + If the difference cannot be calculated then it's defaulted to "0.0". + + Parameters + ---------- + baseline : float + Baseline throughput value. + cpllog : str + Path to the current coupler log. + """ + memlist = self._get_mem_usage(cpllog) + + if len(memlist) > 3: + try: + current = memlist[-1][1] + except IndexError: + current = 0 + + try: + diff = (current - baseline) / baseline + except ZeroDivisionError: + diff = 0.0 + + tolerance = self._case.get_value("TEST_MEMLEAK_TOLERANCE") + if tolerance is None: + tolerance = 0.1 + comment = "MEMCOMP: Memory usage highwater has changed by {:.2f}% relative to baseline".format( + diff * 100 + ) + append_testlog(comment, self._orig_caseroot) + if diff < tolerance and self._test_status.get_status(MEMCOMP_PHASE) is None: + self._test_status.set_status(MEMCOMP_PHASE, TEST_PASS_STATUS) + elif self._test_status.get_status(MEMCOMP_PHASE) != TEST_FAIL_STATUS: + comment = "Error: Memory usage increase >{:d}% from baseline's {:f} to {:f}".format( + int(tolerance * 100), baseline, current + ) + self._test_status.set_status( + MEMCOMP_PHASE, TEST_FAIL_STATUS, comments=comment + ) + append_testlog(comment, self._orig_caseroot) + else: + logger.debug("Found less than 4 memory usage data points") def _compare_throughput(self): + """ + Compares current test throughput to baseline. + """ with self._test_status: # compare memory usage to baseline baseline_name = self._case.get_value("BASECMP_CASE") @@ -803,50 +830,68 @@ def _compare_throughput(self): self._case.get_value("BASELINE_ROOT"), baseline_name ) newestcpllogfiles = self._get_latest_cpl_logs() + # do we need this loop? when is there more than one coupler file? for cpllog in newestcpllogfiles: - m = re.search(r"/({}.*.log).*.gz".format(self._cpllog), cpllog) - if m is not None: - baselog = os.path.join(basecmp_dir, m.group(1)) + ".gz" - if baselog is None or not os.path.isfile(baselog): - # for backward compatibility - baselog = os.path.join(basecmp_dir, self._cpllog) - - if os.path.isfile(baselog): - # compare throughput to baseline - current = self._get_throughput(cpllog) - baseline = self._get_throughput(baselog) - # comparing ypd so bigger is better - if baseline is not None and current is not None: - diff = (baseline - current) / baseline - tolerance = self._case.get_value("TEST_TPUT_TOLERANCE") - if tolerance is None: - tolerance = 0.1 - expect( - tolerance > 0.0, - "Bad value for throughput tolerance in test", - ) - comment = "TPUTCOMP: Computation time changed by {:.2f}% relative to baseline".format( - diff * 100 - ) - append_testlog(comment, self._orig_caseroot) - if ( - diff < tolerance - and self._test_status.get_status(THROUGHPUT_PHASE) is None - ): - self._test_status.set_status( - THROUGHPUT_PHASE, TEST_PASS_STATUS - ) - elif ( - self._test_status.get_status(THROUGHPUT_PHASE) - != TEST_FAIL_STATUS - ): - comment = "Error: TPUTCOMP: Computation time increase > {:d}% from baseline".format( - int(tolerance * 100) - ) - self._test_status.set_status( - THROUGHPUT_PHASE, TEST_FAIL_STATUS, comments=comment - ) - append_testlog(comment, self._orig_caseroot) + try: + baseline_tput = self._read_baseline_tput(basecmp_dir) + except (FileNotFoundError, IndexError, ValueError) as e: + logger.debug("Could not read baseline throughput: %s", e) + + # Default behavior was to skip comparison if baseline file did not exist or no values found + continue + else: + self._compare_current_throughput(baseline_tput, cpllog) + + def _compare_current_throughput(self, baseline, cpllog): + """ + Compares current test throughput against ``baseline``. + + If the difference cannot be calculated then the phase is skipped. + + Parameters + ---------- + baseline : float + Baseline throughput value. + cpllog : str + Path to the current coupler log. + """ + current = self._get_throughput(cpllog) + + try: + # comparing ypd so bigger is better + diff = (baseline - current) / baseline + except ValueError: + # Should we default the diff to 0.0 as with _compare_current_memory? + logger.debug( + "Could not determine change in throughput between baseline %s and current %s", + baseline, + current, + ) + else: + tolerance = self._case.get_value("TEST_TPUT_TOLERANCE") + if tolerance is None: + tolerance = 0.1 + expect( + tolerance > 0.0, + "Bad value for throughput tolerance in test", + ) + comment = "TPUTCOMP: Computation time changed by {:.2f}% relative to baseline".format( + diff * 100 + ) + append_testlog(comment, self._orig_caseroot) + if ( + diff < tolerance + and self._test_status.get_status(THROUGHPUT_PHASE) is None + ): + self._test_status.set_status(THROUGHPUT_PHASE, TEST_PASS_STATUS) + elif self._test_status.get_status(THROUGHPUT_PHASE) != TEST_FAIL_STATUS: + comment = "Error: TPUTCOMP: Computation time increase > {:d}% from baseline".format( + int(tolerance * 100) + ) + self._test_status.set_status( + THROUGHPUT_PHASE, TEST_FAIL_STATUS, comments=comment + ) + append_testlog(comment, self._orig_caseroot) def _compare_baseline(self): """ @@ -900,6 +945,149 @@ def _generate_baseline(self): preserve_meta=False, ) + self._write_baseline_tput(basegen_dir, cpllog) + + self._write_baseline_mem(basegen_dir, cpllog) + + def _write_baseline_tput(self, baseline_dir, cpllog): + """ + Writes baseline throughput to file. + + A "-1" indicates that no throughput data was available from the coupler log. + + Parameters + ---------- + baseline_dir : str + Path to the baseline directory. + cpllog : str + Path to the current coupler log. + """ + tput_file = os.path.join(baseline_dir, "cpl-tput.log") + + tput = self._get_throughput(cpllog) + + with open(tput_file, "w") as fd: + fd.write("# Throughput in simulated years per compute day\n") + fd.write( + "# A -1 indicates no throughput data was available from the test\n" + ) + + if tput is None: + fd.write("-1") + else: + fd.write(str(tput)) + + def _write_baseline_mem(self, baseline_dir, cpllog): + """ + Writes baseline memory usage highwater to file. + + A "-1" indicates that no memory usage data was available from the coupler log. + + Parameters + ---------- + baseline_dir : str + Path to the baseline directory. + cpllog : str + Path to the current coupler log. + """ + mem_file = os.path.join(baseline_dir, "cpl-mem.log") + + mem = self._get_mem_usage(cpllog) + + with open(mem_file, "w") as fd: + fd.write("# Memory usage highwater\n") + fd.write( + "# A -1 indicates no memory usage data was available from the test\n" + ) + + try: + fd.write(str(mem[-1][1])) + except IndexError: + fd.write("-1") + + def _read_baseline_tput(self, baseline_dir): + """ + Reads throughput baseline. + + Parameters + ---------- + baseline_dir : str + Path to the baseline directory. + + Returns + ------- + float + Value of the throughput. + + Raises + ------ + FileNotFoundError + If baseline file does not exist. + IndexError + If no throughput value is found. + ValueError + If throughput value is not a float. + """ + return self._read_baseline_value(os.path.join(baseline_dir, "cpl-tput.log")) + + def _read_baseline_mem(self, baseline_dir): + """ + Reads memory usage highwater baseline. + + The default behvaior was to return 0 if no usage data was found. + + Parameters + ---------- + baseline_dir : str + Path to the baseline directory. + + Returns + ------- + float + Value of the highwater memory usage. + + Raises + ------ + FileNotFoundError + If baseline file does not exist. + """ + try: + memory = self._read_baseline_value( + os.path.join(baseline_dir, "cpl-mem.log") + ) + except (IndexError, ValueError): + memory = 0 + + return memory + + def _read_baseline_value(self, baseline_file): + """ + Read baseline value from file. + + Parameters + ---------- + baseline_file : str + Path to baseline file. + + Returns + ------- + float + Baseline value. + + Raises + ------ + FileNotFoundError + If ``baseline_file`` is not found. + IndexError + If not values are present in ``baseline_file``. + ValueError + If value in ``baseline_file`` is not a float. + """ + with open(baseline_file) as fd: + lines = [x for x in fd.readlines() if not x.startswith("#")] + + return float(lines[0]) + class FakeTest(SystemTestsCommon): """ diff --git a/CIME/tests/test_unit_system_tests.py b/CIME/tests/test_unit_system_tests.py index 3bd091900e3..e456b675047 100644 --- a/CIME/tests/test_unit_system_tests.py +++ b/CIME/tests/test_unit_system_tests.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import os +import tempfile +import gzip from re import A import unittest from unittest import mock @@ -10,8 +12,273 @@ from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo from CIME.SystemTests.system_tests_compare_n import SystemTestsCompareN +CPLLOG = """ + tStamp_write: model date = 00010102 0 wall clock = 2023-09-19 19:39:42 avg dt = 0.33 dt = 0.33 + memory_write: model date = 00010102 0 memory = 1673.89 MB (highwater) 387.77 MB (usage) (pe= 0 comps= cpl ATM LND ICE OCN GLC ROF WAV IAC ESP) + tStamp_write: model date = 00010103 0 wall clock = 2023-09-19 19:39:42 avg dt = 0.33 dt = 0.33 + memory_write: model date = 00010103 0 memory = 1673.89 MB (highwater) 390.09 MB (usage) (pe= 0 comps= cpl ATM LND ICE OCN GLC ROF WAV IAC ESP) + tStamp_write: model date = 00010104 0 wall clock = 2023-09-19 19:39:42 avg dt = 0.33 dt = 0.33 + memory_write: model date = 00010104 0 memory = 1673.89 MB (highwater) 391.64 MB (usage) (pe= 0 comps= cpl ATM LND ICE OCN GLC ROF WAV IAC ESP) + tStamp_write: model date = 00010105 0 wall clock = 2023-09-19 19:39:43 avg dt = 0.33 dt = 0.33 + memory_write: model date = 00010105 0 memory = 1673.89 MB (highwater) 392.67 MB (usage) (pe= 0 comps= cpl ATM LND ICE OCN GLC ROF WAV IAC ESP) + tStamp_write: model date = 00010106 0 wall clock = 2023-09-19 19:39:43 avg dt = 0.33 dt = 0.33 + memory_write: model date = 00010106 0 memory = 1673.89 MB (highwater) 393.44 MB (usage) (pe= 0 comps= cpl ATM LND ICE OCN GLC ROF WAV IAC ESP) + +(seq_mct_drv): =============== SUCCESSFUL TERMINATION OF CPL7-e3sm =============== +(seq_mct_drv): =============== at YMD,TOD = 00010106 0 =============== +(seq_mct_drv): =============== # simulated days (this run) = 5.000 =============== +(seq_mct_drv): =============== compute time (hrs) = 0.000 =============== +(seq_mct_drv): =============== # simulated years / cmp-day = 719.635 =============== +(seq_mct_drv): =============== pes min memory highwater (MB) 851.957 =============== +(seq_mct_drv): =============== pes max memory highwater (MB) 1673.891 =============== +(seq_mct_drv): =============== pes min memory last usage (MB) 182.742 =============== +(seq_mct_drv): =============== pes max memory last usage (MB) 393.441 =============== +""" + + +def create_mock_case(tempdir, idx=None, cpllog_data=None): + if idx is None: + idx = 0 + + case = mock.MagicMock() + + caseroot = Path(tempdir, str(idx), "caseroot") + baseline_root = caseroot.parent / "baselines" + run_dir = caseroot / "run" + run_dir.mkdir(parents=True, exist_ok=False) + + if cpllog_data is not None: + cpllog = run_dir / "cpl.log.gz" + + with gzip.open(cpllog, "w") as fd: + fd.write(cpllog_data.encode("utf-8")) + + case.get_latest_cpl_log.return_value = str(cpllog) + + hist_file = run_dir / "cpl.hi.2023-01-01.nc" + hist_file.touch() + + case.get_env.return_value.get_latest_hist_files.return_value = [str(hist_file)] + + case.get_compset_components.return_value = [] + + return case, caseroot, baseline_root, run_dir + class TestCaseSubmit(unittest.TestCase): + @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") + def test_compare_throughput(self, append_testlog): + with tempfile.TemporaryDirectory() as tempdir: + case, caseroot, baseline_root, run_dir = create_mock_case( + tempdir, cpllog_data=CPLLOG + ) + + base_case, base_caseroot, _, _ = create_mock_case( + tempdir, idx=1, cpllog_data=CPLLOG + ) + + case.get_value.side_effect = ( + str(caseroot), + "ERIO.ne30_g16_rx1.A.docker_gnu", + "mct", + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + str(baseline_root), + str(run_dir), + 0.05, + ) + + baseline_dir = baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" + baseline_dir.mkdir(parents=True, exist_ok=False) + baseline_mem = baseline_dir / "cpl-mem.log" + + with open(baseline_mem, "w") as fd: + fd.write(str("1673.89")) + + common = SystemTestsCommon(case) + + common._compare_memory() + + assert common._test_status.get_overall_test_status() == ("PASS", None) + + append_testlog.assert_any_call( + "MEMCOMP: Memory usage highwater has changed by 0.00% relative to baseline", + str(caseroot), + ) + + @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") + def test_compare_throughput_fail(self, append_testlog): + with tempfile.TemporaryDirectory() as tempdir: + case, caseroot, baseline_root, run_dir = create_mock_case( + tempdir, cpllog_data=CPLLOG + ) + + base_case, base_caseroot, _, _ = create_mock_case( + tempdir, idx=1, cpllog_data=CPLLOG + ) + + case.get_value.side_effect = ( + str(caseroot), + "ERIO.ne30_g16_rx1.A.docker_gnu", + "mct", + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + str(baseline_root), + str(run_dir), + 0.05, + ) + + baseline_dir = baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" + baseline_dir.mkdir(parents=True, exist_ok=False) + baseline_mem = baseline_dir / "cpl-mem.log" + + with open(baseline_mem, "w") as fd: + fd.write(str("1473.89")) + + common = SystemTestsCommon(case) + + common._compare_memory() + + assert common._test_status.get_overall_test_status() == ("PASS", None) + + append_testlog.assert_any_call( + "MEMCOMP: Memory usage highwater has changed by 13.57% relative to baseline", + str(caseroot), + ) + append_testlog.assert_any_call( + "Error: Memory usage increase >5% from baseline's 1473.890000 to 1673.890000", + str(caseroot), + ) + + @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") + def test_compare_memory(self, append_testlog): + with tempfile.TemporaryDirectory() as tempdir: + case, caseroot, baseline_root, run_dir = create_mock_case( + tempdir, cpllog_data=CPLLOG + ) + + base_case, base_caseroot, _, _ = create_mock_case( + tempdir, idx=1, cpllog_data=CPLLOG + ) + + case.get_value.side_effect = ( + str(caseroot), + "ERIO.ne30_g16_rx1.A.docker_gnu", + "mct", + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + str(baseline_root), + str(run_dir), + 0.05, + ) + + baseline_dir = baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" + baseline_dir.mkdir(parents=True, exist_ok=False) + baseline_tput = baseline_dir / "cpl-tput.log" + + with open(baseline_tput, "w") as fd: + fd.write(str("719.635")) + + common = SystemTestsCommon(case) + + common._compare_throughput() + + assert common._test_status.get_overall_test_status() == ("PASS", None) + + append_testlog.assert_any_call( + "TPUTCOMP: Computation time changed by 0.00% relative to baseline", + str(caseroot), + ) + + @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") + def test_compare_memory_fail(self, append_testlog): + with tempfile.TemporaryDirectory() as tempdir: + case, caseroot, baseline_root, run_dir = create_mock_case( + tempdir, cpllog_data=CPLLOG + ) + + base_case, base_caseroot, _, _ = create_mock_case( + tempdir, idx=1, cpllog_data=CPLLOG + ) + + case.get_value.side_effect = ( + str(caseroot), + "ERIO.ne30_g16_rx1.A.docker_gnu", + "mct", + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + str(baseline_root), + str(run_dir), + 0.05, + ) + + baseline_dir = baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" + baseline_dir.mkdir(parents=True, exist_ok=False) + baseline_tput = baseline_dir / "cpl-tput.log" + + with open(baseline_tput, "w") as fd: + fd.write(str("900.635")) + + common = SystemTestsCommon(case) + + common._compare_throughput() + + assert common._test_status.get_overall_test_status() == ("PASS", None) + + append_testlog.assert_any_call( + "TPUTCOMP: Computation time changed by 20.10% relative to baseline", + str(caseroot), + ) + append_testlog.assert_any_call( + "Error: TPUTCOMP: Computation time increase > 5% from baseline", + str(caseroot), + ) + + def test_generate_baseline(self): + with tempfile.TemporaryDirectory() as tempdir: + case, caseroot, baseline_root, run_dir = create_mock_case( + tempdir, cpllog_data=CPLLOG + ) + + case.get_value.side_effect = ( + str(caseroot), + "ERIO.ne30_g16_rx1.A.docker_gnu", + "mct", + str(run_dir), + "case.std", + str(baseline_root), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + "ERIO.ne30_g16_rx1.A.docker_gnu.G.20230919_193255_z9hg2w", + "mct", + str(run_dir), + "ERIO", + "ERIO.ne30_g16_rx1.A.docker_gnu", + os.getcwd(), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + str(baseline_root), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + str(run_dir), + ) + + common = SystemTestsCommon(case) + + common._generate_baseline() + + baseline_dir = baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" + + assert (baseline_dir / "cpl.log.gz").exists() + assert (baseline_dir / "cpl-tput.log").exists() + assert (baseline_dir / "cpl-mem.log").exists() + assert (baseline_dir / "cpl.hi.2023-01-01.nc").exists() + + with open(baseline_dir / "cpl-tput.log") as fd: + lines = fd.readlines() + + assert len(lines) == 3 + assert lines[-1] == "719.635" + + with open(baseline_dir / "cpl-mem.log") as fd: + lines = fd.readlines() + + assert len(lines) == 3 + assert lines[-1] == "1673.89" + def test_kwargs(self): case = mock.MagicMock() From a367ab9b76bb8edb11bec505872b00941d929a1b Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Wed, 20 Sep 2023 14:45:04 -0700 Subject: [PATCH 05/31] Updates MEMCOMP and TPUTCOMP to DIFF if they're checked and fail --- CIME/test_status.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CIME/test_status.py b/CIME/test_status.py index 90714631eb8..5f306b7db0e 100644 --- a/CIME/test_status.py +++ b/CIME/test_status.py @@ -460,7 +460,7 @@ def _get_overall_status_based_on_phases( if rv == TEST_PASS_STATUS: rv = NAMELIST_FAIL_STATUS - elif phase == BASELINE_PHASE: + elif phase in [BASELINE_PHASE, THROUGHPUT_PHASE, MEMCOMP_PHASE]: if rv in [NAMELIST_FAIL_STATUS, TEST_PASS_STATUS]: phase_responsible_for_status = phase rv = TEST_DIFF_STATUS @@ -512,7 +512,9 @@ def get_overall_test_status( >>> _test_helper2('PASS ERS.foo.A RUN\nFAIL ERS.foo.A TPUTCOMP') ('PASS', 'RUN') >>> _test_helper2('PASS ERS.foo.A RUN\nFAIL ERS.foo.A TPUTCOMP', check_throughput=True) - ('FAIL', 'TPUTCOMP') + ('DIFF', 'TPUTCOMP') + >>> _test_helper2('PASS ERS.foo.A RUN\nFAIL ERS.foo.A MEMCOMP', check_memory=True) + ('DIFF', 'MEMCOMP') >>> _test_helper2('PASS ERS.foo.A MODEL_BUILD\nPASS ERS.foo.A RUN\nFAIL ERS.foo.A NLCOMP') ('NLFAIL', 'RUN') >>> _test_helper2('PASS ERS.foo.A MODEL_BUILD\nPEND ERS.foo.A RUN\nFAIL ERS.foo.A NLCOMP') From 2cc83524770a3bb7b7b66c77f9f9a913933953fd Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 21 Sep 2023 12:42:05 -0700 Subject: [PATCH 06/31] Refactors _compare_throughput and _compare_memory methods --- CIME/SystemTests/system_tests_common.py | 379 +++----------------- CIME/baselines.py | 320 +++++++++++++++++ CIME/tests/test_unit_baselines.py | 437 ++++++++++++++++++++++++ CIME/tests/test_unit_system_tests.py | 209 ++++++------ 4 files changed, 913 insertions(+), 432 deletions(-) create mode 100644 CIME/baselines.py create mode 100644 CIME/tests/test_unit_baselines.py diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index eb367e33cee..37dd63ad49c 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -26,6 +26,14 @@ from CIME.config import Config from CIME.provenance import save_test_time, get_test_success from CIME.locked_files import LOCKED_DIR, lock_file, is_locked +from CIME.baselines import ( + get_latest_cpl_logs, + get_mem_usage, + compare_memory, + compare_throughput, + write_baseline_mem, + write_baseline_tput, +) import CIME.build as build import glob, gzip, time, traceback, os @@ -493,7 +501,7 @@ def run_indv( self._case.case_st_archive(resubmit=True) def _coupler_log_indicates_run_complete(self): - newestcpllogfiles = self._get_latest_cpl_logs() + newestcpllogfiles = get_latest_cpl_logs(self._case) logger.debug("Latest Coupler log file(s) {}".format(newestcpllogfiles)) # Exception is raised if the file is not compressed allgood = len(newestcpllogfiles) @@ -598,43 +606,6 @@ def _st_archive_case_test(self): else: self._test_status.set_status(STARCHIVE_PHASE, TEST_FAIL_STATUS) - def _get_mem_usage(self, cpllog): - """ - Examine memory usage as recorded in the cpl log file and look for unexpected - increases. - """ - memlist = [] - meminfo = re.compile( - r".*model date =\s+(\w+).*memory =\s+(\d+\.?\d+).*highwater" - ) - if cpllog is not None and os.path.isfile(cpllog): - if ".gz" == cpllog[-3:]: - fopen = gzip.open - else: - fopen = open - with fopen(cpllog, "rb") as f: - for line in f: - m = meminfo.match(line.decode("utf-8")) - if m: - memlist.append((float(m.group(1)), float(m.group(2)))) - # Remove the last mem record, it's sometimes artificially high - if len(memlist) > 0: - memlist.pop() - return memlist - - def _get_throughput(self, cpllog): - """ - Examine memory usage as recorded in the cpl log file and look for unexpected - increases. - """ - if cpllog is not None and os.path.isfile(cpllog): - with gzip.open(cpllog, "rb") as f: - cpltext = f.read().decode("utf-8") - m = re.search(r"# simulated years / cmp-day =\s+(\d+\.\d+)\s", cpltext) - if m: - return float(m.group(1)) - return None - def _phase_modifying_call(self, phase, function): """ Ensures that unexpected exceptions from phases will result in a FAIL result @@ -662,9 +633,9 @@ def _check_for_memleak(self): increases. """ with self._test_status: - latestcpllogs = self._get_latest_cpl_logs() + latestcpllogs = get_latest_cpl_logs(self._case) for cpllog in latestcpllogs: - memlist = self._get_mem_usage(cpllog) + memlist = get_mem_usage(cpllog) if len(memlist) < 3: self._test_status.set_status( @@ -728,170 +699,61 @@ def compare_env_run(self, expected=None): return False return True - def _get_latest_cpl_logs(self): - """ - find and return the latest cpl log file in the run directory - """ - coupler_log_path = self._case.get_value("RUNDIR") - cpllogs = glob.glob( - os.path.join(coupler_log_path, "{}*.log.*".format(self._cpllog)) - ) - lastcpllogs = [] - if cpllogs: - lastcpllogs.append(max(cpllogs, key=os.path.getctime)) - basename = os.path.basename(lastcpllogs[0]) - suffix = basename.split(".", 1)[1] - for log in cpllogs: - if log in lastcpllogs: - continue - - if log.endswith(suffix): - lastcpllogs.append(log) - - return lastcpllogs - def _compare_memory(self): """ Compares current test memory usage to baseline. """ with self._test_status: - # compare memory usage to baseline - baseline_name = self._case.get_value("BASECMP_CASE") - basecmp_dir = os.path.join( - self._case.get_value("BASELINE_ROOT"), baseline_name + below_tolerance, diff, tolerance, baseline, current = compare_memory( + self._case ) - newestcpllogfiles = self._get_latest_cpl_logs() - - # do we need this loop? when is there more than one coupler file? - for cpllog in newestcpllogfiles: - try: - baseline_mem = self._read_baseline_mem(basecmp_dir) - except FileNotFoundError as e: - logger.debug("Could not read baseline memory usage: %s", e) - - continue - else: - self._compare_current_memory(baseline_mem, cpllog) - - def _compare_current_memory(self, baseline, cpllog): - """ - Compares current test memory usage against ``baseline``. - If the difference cannot be calculated then it's defaulted to "0.0". - - Parameters - ---------- - baseline : float - Baseline throughput value. - cpllog : str - Path to the current coupler log. - """ - memlist = self._get_mem_usage(cpllog) - - if len(memlist) > 3: - try: - current = memlist[-1][1] - except IndexError: - current = 0 - - try: - diff = (current - baseline) / baseline - except ZeroDivisionError: - diff = 0.0 - - tolerance = self._case.get_value("TEST_MEMLEAK_TOLERANCE") - if tolerance is None: - tolerance = 0.1 - comment = "MEMCOMP: Memory usage highwater has changed by {:.2f}% relative to baseline".format( - diff * 100 - ) - append_testlog(comment, self._orig_caseroot) - if diff < tolerance and self._test_status.get_status(MEMCOMP_PHASE) is None: - self._test_status.set_status(MEMCOMP_PHASE, TEST_PASS_STATUS) - elif self._test_status.get_status(MEMCOMP_PHASE) != TEST_FAIL_STATUS: - comment = "Error: Memory usage increase >{:d}% from baseline's {:f} to {:f}".format( - int(tolerance * 100), baseline, current - ) - self._test_status.set_status( - MEMCOMP_PHASE, TEST_FAIL_STATUS, comments=comment + if below_tolerance is not None: + comment = "MEMCOMP: Memory usage highwater has changed by {:.2f}% relative to baseline".format( + diff * 100 ) + append_testlog(comment, self._orig_caseroot) - else: - logger.debug("Found less than 4 memory usage data points") + + if ( + below_tolerance + and self._test_status.get_status(MEMCOMP_PHASE) is None + ): + self._test_status.set_status(MEMCOMP_PHASE, TEST_PASS_STATUS) + elif self._test_status.get_status(MEMCOMP_PHASE) != TEST_FAIL_STATUS: + comment = "Error: Memory usage increase >{:d}% from baseline's {:f} to {:f}".format( + int(tolerance * 100), baseline, current + ) + self._test_status.set_status( + MEMCOMP_PHASE, TEST_FAIL_STATUS, comments=comment + ) + append_testlog(comment, self._orig_caseroot) def _compare_throughput(self): """ Compares current test throughput to baseline. """ with self._test_status: - # compare memory usage to baseline - baseline_name = self._case.get_value("BASECMP_CASE") - basecmp_dir = os.path.join( - self._case.get_value("BASELINE_ROOT"), baseline_name - ) - newestcpllogfiles = self._get_latest_cpl_logs() - # do we need this loop? when is there more than one coupler file? - for cpllog in newestcpllogfiles: - try: - baseline_tput = self._read_baseline_tput(basecmp_dir) - except (FileNotFoundError, IndexError, ValueError) as e: - logger.debug("Could not read baseline throughput: %s", e) - - # Default behavior was to skip comparison if baseline file did not exist or no values found - continue - else: - self._compare_current_throughput(baseline_tput, cpllog) - - def _compare_current_throughput(self, baseline, cpllog): - """ - Compares current test throughput against ``baseline``. - - If the difference cannot be calculated then the phase is skipped. - - Parameters - ---------- - baseline : float - Baseline throughput value. - cpllog : str - Path to the current coupler log. - """ - current = self._get_throughput(cpllog) + below_tolerance, diff, tolerance, _, _ = compare_throughput(self._case) - try: - # comparing ypd so bigger is better - diff = (baseline - current) / baseline - except ValueError: - # Should we default the diff to 0.0 as with _compare_current_memory? - logger.debug( - "Could not determine change in throughput between baseline %s and current %s", - baseline, - current, - ) - else: - tolerance = self._case.get_value("TEST_TPUT_TOLERANCE") - if tolerance is None: - tolerance = 0.1 - expect( - tolerance > 0.0, - "Bad value for throughput tolerance in test", - ) - comment = "TPUTCOMP: Computation time changed by {:.2f}% relative to baseline".format( - diff * 100 - ) - append_testlog(comment, self._orig_caseroot) - if ( - diff < tolerance - and self._test_status.get_status(THROUGHPUT_PHASE) is None - ): - self._test_status.set_status(THROUGHPUT_PHASE, TEST_PASS_STATUS) - elif self._test_status.get_status(THROUGHPUT_PHASE) != TEST_FAIL_STATUS: - comment = "Error: TPUTCOMP: Computation time increase > {:d}% from baseline".format( - int(tolerance * 100) - ) - self._test_status.set_status( - THROUGHPUT_PHASE, TEST_FAIL_STATUS, comments=comment + if below_tolerance is not None: + comment = "TPUTCOMP: Computation time changed by {:.2f}% relative to baseline".format( + diff * 100 ) append_testlog(comment, self._orig_caseroot) + if ( + below_tolerance + and self._test_status.get_status(THROUGHPUT_PHASE) is None + ): + self._test_status.set_status(THROUGHPUT_PHASE, TEST_PASS_STATUS) + elif self._test_status.get_status(THROUGHPUT_PHASE) != TEST_FAIL_STATUS: + comment = "Error: TPUTCOMP: Computation time increase > {:d}% from baseline".format( + int(tolerance * 100) + ) + self._test_status.set_status( + THROUGHPUT_PHASE, TEST_FAIL_STATUS, comments=comment + ) + append_testlog(comment, self._orig_caseroot) def _compare_baseline(self): """ @@ -933,7 +795,7 @@ def _generate_baseline(self): ) # copy latest cpl log to baseline # drop the date so that the name is generic - newestcpllogfiles = self._get_latest_cpl_logs() + newestcpllogfiles = get_latest_cpl_logs(self._case) with SharedArea(): for cpllog in newestcpllogfiles: m = re.search(r"/({}.*.log).*.gz".format(self._cpllog), cpllog) @@ -945,148 +807,9 @@ def _generate_baseline(self): preserve_meta=False, ) - self._write_baseline_tput(basegen_dir, cpllog) - - self._write_baseline_mem(basegen_dir, cpllog) - - def _write_baseline_tput(self, baseline_dir, cpllog): - """ - Writes baseline throughput to file. - - A "-1" indicates that no throughput data was available from the coupler log. - - Parameters - ---------- - baseline_dir : str - Path to the baseline directory. - cpllog : str - Path to the current coupler log. - """ - tput_file = os.path.join(baseline_dir, "cpl-tput.log") - - tput = self._get_throughput(cpllog) - - with open(tput_file, "w") as fd: - fd.write("# Throughput in simulated years per compute day\n") - fd.write( - "# A -1 indicates no throughput data was available from the test\n" - ) - - if tput is None: - fd.write("-1") - else: - fd.write(str(tput)) - - def _write_baseline_mem(self, baseline_dir, cpllog): - """ - Writes baseline memory usage highwater to file. - - A "-1" indicates that no memory usage data was available from the coupler log. - - Parameters - ---------- - baseline_dir : str - Path to the baseline directory. - cpllog : str - Path to the current coupler log. - """ - mem_file = os.path.join(baseline_dir, "cpl-mem.log") - - mem = self._get_mem_usage(cpllog) - - with open(mem_file, "w") as fd: - fd.write("# Memory usage highwater\n") - fd.write( - "# A -1 indicates no memory usage data was available from the test\n" - ) - - try: - fd.write(str(mem[-1][1])) - except IndexError: - fd.write("-1") - - def _read_baseline_tput(self, baseline_dir): - """ - Reads throughput baseline. - - Parameters - ---------- - baseline_dir : str - Path to the baseline directory. - - Returns - ------- - float - Value of the throughput. - - Raises - ------ - FileNotFoundError - If baseline file does not exist. - IndexError - If no throughput value is found. - ValueError - If throughput value is not a float. - """ - return self._read_baseline_value(os.path.join(baseline_dir, "cpl-tput.log")) - - def _read_baseline_mem(self, baseline_dir): - """ - Reads memory usage highwater baseline. - - The default behvaior was to return 0 if no usage data was found. - - Parameters - ---------- - baseline_dir : str - Path to the baseline directory. - - Returns - ------- - float - Value of the highwater memory usage. - - Raises - ------ - FileNotFoundError - If baseline file does not exist. - """ - try: - memory = self._read_baseline_value( - os.path.join(baseline_dir, "cpl-mem.log") - ) - except (IndexError, ValueError): - memory = 0 - - return memory - - def _read_baseline_value(self, baseline_file): - """ - Read baseline value from file. - - Parameters - ---------- - baseline_file : str - Path to baseline file. - - Returns - ------- - float - Baseline value. - - Raises - ------ - FileNotFoundError - If ``baseline_file`` is not found. - IndexError - If not values are present in ``baseline_file``. - ValueError - If value in ``baseline_file`` is not a float. - """ - with open(baseline_file) as fd: - lines = [x for x in fd.readlines() if not x.startswith("#")] + write_baseline_tput(basegen_dir, cpllog) - return float(lines[0]) + write_baseline_mem(basegen_dir, cpllog) class FakeTest(SystemTestsCommon): diff --git a/CIME/baselines.py b/CIME/baselines.py new file mode 100644 index 00000000000..e371488f294 --- /dev/null +++ b/CIME/baselines.py @@ -0,0 +1,320 @@ +import os +import glob +import re +import gzip +import logging +from CIME.utils import expect + +logger = logging.getLogger(__name__) + +def get_latest_cpl_logs(case): + """ + find and return the latest cpl log file in the run directory + """ + coupler_log_path = case.get_value("RUNDIR") + cpllog_name = "drv" if case.get_value("COMP_INTERFACE") == "nuopc" else "cpl" + cpllogs = glob.glob( + os.path.join(coupler_log_path, "{}*.log.*".format(cpllog_name)) + ) + lastcpllogs = [] + if cpllogs: + lastcpllogs.append(max(cpllogs, key=os.path.getctime)) + basename = os.path.basename(lastcpllogs[0]) + suffix = basename.split(".", 1)[1] + for log in cpllogs: + if log in lastcpllogs: + continue + + if log.endswith(suffix): + lastcpllogs.append(log) + + return lastcpllogs + +def compare_memory(case): + baseline_root = case.get_value("BASELINE_ROOT") + + baseline_name = case.get_value("BASECMP_CASE") + + baseline_dir = os.path.join(baseline_root, baseline_name) + + latest_cpl_logs = get_latest_cpl_logs(case) + + diff, baseline, current = [None]*3 + + for cpllog in latest_cpl_logs: + try: + baseline = read_baseline_mem(baseline_dir) + except FileNotFoundError as e: + logger.debug("Could not read baseline memory usage: %s", e) + + continue + + if baseline is None: + baseline = 0.0 + + memlist = get_mem_usage(cpllog) + + if len(memlist) <= 3: + logger.debug(f"Found {len(memlist)} memory usage samples, need atleast 4") + + continue + + current = memlist[-1][1] + + try: + diff = (current - baseline) / baseline + except ZeroDivisionError: + diff = 0.0 + + tolerance = case.get_value("TEST_MEMLEAK_TOLERANCE") + + if tolerance is None: + tolerance = 0.1 + + # Should we check if tolerance is above 0 + below_tolerance = None + + if diff is not None: + below_tolerance = diff < tolerance + + return below_tolerance, diff, tolerance, baseline, current + +def compare_throughput(case): + baseline_root = case.get_value("BASELINE_ROOT") + + baseline_name = case.get_value("BASECMP_CASE") + + baseline_dir = os.path.join(baseline_root, baseline_name) + + latest_cpl_logs = get_latest_cpl_logs(case) + + diff, baseline, current = [None]*3 + + # Do we need this loop, when are there multiple cpl logs? + for cpllog in latest_cpl_logs: + try: + baseline = read_baseline_tput(baseline_dir) + except (FileNotFoundError, IndexError, ValueError) as e: + logger.debug("Could not read baseline throughput: %s", e) + + continue + + current = get_throughput(cpllog) + + try: + # comparing ypd so bigger is better + diff = (baseline - current) / baseline + except (ValueError, TypeError): + # Should we default the diff to 0.0 as with _compare_current_memory? + logger.debug( + "Could not determine change in throughput between baseline %s and current %s", + baseline, + current, + ) + + continue + + tolerance = case.get_value("TEST_TPUT_TOLERANCE") + + if tolerance is None: + tolerance = 0.1 + + expect( + tolerance > 0.0, + "Bad value for throughput tolerance in test", + ) + + below_tolerance = None + + if diff is not None: + below_tolerance = diff < tolerance + + return below_tolerance, diff, tolerance, baseline, current + +def write_baseline_tput(baseline_dir, cpllog): + """ + Writes baseline throughput to file. + + A "-1" indicates that no throughput data was available from the coupler log. + + Parameters + ---------- + baseline_dir : str + Path to the baseline directory. + cpllog : str + Path to the current coupler log. + """ + tput_file = os.path.join(baseline_dir, "cpl-tput.log") + + tput = get_throughput(cpllog) + + with open(tput_file, "w") as fd: + fd.write("# Throughput in simulated years per compute day\n") + fd.write( + "# A -1 indicates no throughput data was available from the test\n" + ) + + if tput is None: + fd.write("-1") + else: + fd.write(str(tput)) + +def write_baseline_mem(baseline_dir, cpllog): + """ + Writes baseline memory usage highwater to file. + + A "-1" indicates that no memory usage data was available from the coupler log. + + Parameters + ---------- + baseline_dir : str + Path to the baseline directory. + cpllog : str + Path to the current coupler log. + """ + mem_file = os.path.join(baseline_dir, "cpl-mem.log") + + mem = get_mem_usage(cpllog) + + with open(mem_file, "w") as fd: + fd.write("# Memory usage highwater\n") + fd.write( + "# A -1 indicates no memory usage data was available from the test\n" + ) + + try: + fd.write(str(mem[-1][1])) + except IndexError: + fd.write("-1") + +def read_baseline_tput(baseline_dir): + """ + Reads throughput baseline. + + Parameters + ---------- + baseline_dir : str + Path to the baseline directory. + + Returns + ------- + float + Value of the throughput. + + Raises + ------ + FileNotFoundError + If baseline file does not exist. + IndexError + If no throughput value is found. + ValueError + If throughput value is not a float. + """ + try: + tput = read_baseline_value(os.path.join(baseline_dir, "cpl-tput.log")) + except (IndexError, ValueError): + tput = None + + if tput == -1: + tput = None + + return tput + +def read_baseline_mem(baseline_dir): + """ + Reads memory usage highwater baseline. + + The default behvaior was to return 0 if no usage data was found. + + Parameters + ---------- + baseline_dir : str + Path to the baseline directory. + + Returns + ------- + float + Value of the highwater memory usage. + + Raises + ------ + FileNotFoundError + If baseline file does not exist. + """ + try: + memory = read_baseline_value( + os.path.join(baseline_dir, "cpl-mem.log") + ) + except (IndexError, ValueError): + memory = 0 + + if memory == -1: + memory = 0 + + return memory + +def read_baseline_value(baseline_file): + """ + Read baseline value from file. + + Parameters + ---------- + baseline_file : str + Path to baseline file. + + Returns + ------- + float + Baseline value. + + Raises + ------ + FileNotFoundError + If ``baseline_file`` is not found. + IndexError + If not values are present in ``baseline_file``. + ValueError + If value in ``baseline_file`` is not a float. + """ + with open(baseline_file) as fd: + lines = [x for x in fd.readlines() if not x.startswith("#")] + + return float(lines[0]) + +def get_mem_usage(cpllog): + """ + Examine memory usage as recorded in the cpl log file and look for unexpected + increases. + """ + memlist = [] + meminfo = re.compile( + r".*model date =\s+(\w+).*memory =\s+(\d+\.?\d+).*highwater" + ) + if cpllog is not None and os.path.isfile(cpllog): + if ".gz" == cpllog[-3:]: + fopen = gzip.open + else: + fopen = open + with fopen(cpllog, "rb") as f: + for line in f: + m = meminfo.match(line.decode("utf-8")) + if m: + memlist.append((float(m.group(1)), float(m.group(2)))) + # Remove the last mem record, it's sometimes artificially high + if len(memlist) > 0: + memlist.pop() + return memlist + +def get_throughput(cpllog): + """ + Examine memory usage as recorded in the cpl log file and look for unexpected + increases. + """ + if cpllog is not None and os.path.isfile(cpllog): + with gzip.open(cpllog, "rb") as f: + cpltext = f.read().decode("utf-8") + m = re.search(r"# simulated years / cmp-day =\s+(\d+\.\d+)\s", cpltext) + if m: + return float(m.group(1)) + return None + diff --git a/CIME/tests/test_unit_baselines.py b/CIME/tests/test_unit_baselines.py new file mode 100644 index 00000000000..77ce61ac20f --- /dev/null +++ b/CIME/tests/test_unit_baselines.py @@ -0,0 +1,437 @@ +#!/usr/bin/env python3 + +import gzip +import tempfile +import unittest +from unittest import mock +from pathlib import Path + +from CIME import baselines +from CIME.tests.test_unit_system_tests import CPLLOG + +def create_mock_case(tempdir, get_latest_cpl_logs): + caseroot = Path(tempdir, "0", "caseroot") + rundir = caseroot / "run" + + get_latest_cpl_logs.return_value = ( + str(rundir / "cpl.log.gz"), + ) + + baseline_root = Path(tempdir, "baselines") + + case = mock.MagicMock() + + return case, caseroot, rundir, baseline_root + +class TestUnitBaseline(unittest.TestCase): + def test_get_throughput_no_file(self): + throughput = baselines.get_throughput("/tmp/cpl.log") + + assert throughput is None + + def test_get_throughput(self): + with tempfile.TemporaryDirectory() as tempdir: + cpl_log_path = Path(tempdir, "cpl.log.gz") + + with gzip.open(cpl_log_path, "w") as fd: + fd.write(CPLLOG.encode("utf-8")) + + throughput = baselines.get_throughput(str(cpl_log_path)) + + assert throughput == 719.635 + + def test_get_mem_usage_gz(self): + with tempfile.TemporaryDirectory() as tempdir: + cpl_log_path = Path(tempdir, "cpl.log.gz") + + with gzip.open(cpl_log_path, "w") as fd: + fd.write(CPLLOG.encode("utf-8")) + + mem_usage = baselines.get_mem_usage(str(cpl_log_path)) + + assert mem_usage == [(10102.0, 1673.89), (10103.0, 1673.89), (10104.0, 1673.89), (10105.0, 1673.89)] + + @mock.patch("CIME.baselines.os.path.isfile") + def test_get_mem_usage(self, isfile): + isfile.return_value = True + + with mock.patch("builtins.open", mock.mock_open(read_data=CPLLOG.encode("utf-8"))) as mock_file: + mem_usage = baselines.get_mem_usage("/tmp/cpl.log") + + assert mem_usage == [(10102.0, 1673.89), (10103.0, 1673.89), (10104.0, 1673.89), (10105.0, 1673.89)] + + def test_read_baseline_mem_empty(self): + with mock.patch("builtins.open", mock.mock_open(read_data="")) as mock_file: + baseline = baselines.read_baseline_mem("/tmp/cpl-mem.log") + + assert baseline is 0 + + def test_read_baseline_mem_none(self): + with mock.patch("builtins.open", mock.mock_open(read_data="-1")) as mock_file: + baseline = baselines.read_baseline_mem("/tmp/cpl-mem.log") + + assert baseline is 0 + + def test_read_baseline_mem(self): + with mock.patch("builtins.open", mock.mock_open(read_data="200")) as mock_file: + baseline = baselines.read_baseline_mem("/tmp/cpl-mem.log") + + assert baseline == 200 + + def test_read_baseline_tput_empty(self): + with mock.patch("builtins.open", mock.mock_open(read_data="")) as mock_file: + baseline = baselines.read_baseline_tput("/tmp/cpl-tput.log") + + assert baseline is None + + def test_read_baseline_tput_none(self): + with mock.patch("builtins.open", mock.mock_open(read_data="-1")) as mock_file: + baseline = baselines.read_baseline_tput("/tmp/cpl-tput.log") + + assert baseline is None + + def test_read_baseline_tput(self): + with mock.patch("builtins.open", mock.mock_open(read_data="200")) as mock_file: + baseline = baselines.read_baseline_tput("/tmp/cpl-tput.log") + + assert baseline == 200 + + @mock.patch("CIME.baselines.get_mem_usage") + def test_write_baseline_mem_no_value(self, get_mem_usage): + get_mem_usage.return_value = [] + + with mock.patch("builtins.open", mock.mock_open()) as mock_file: + baselines.write_baseline_mem("/tmp", "/tmp/cpl.log") + + mock_file.assert_called_with("/tmp/cpl-mem.log", "w") + mock_file.return_value.write.assert_called_with("-1") + + @mock.patch("CIME.baselines.get_mem_usage") + def test_write_baseline_mem(self, get_mem_usage): + get_mem_usage.return_value = [(1, 200)] + + with mock.patch("builtins.open", mock.mock_open()) as mock_file: + baselines.write_baseline_mem("/tmp", "/tmp/cpl.log") + + mock_file.assert_called_with("/tmp/cpl-mem.log", "w") + mock_file.return_value.write.assert_called_with("200") + + @mock.patch("CIME.baselines.get_throughput") + def test_write_baseline_tput_no_value(self, get_throughput): + get_throughput.return_value = None + + with mock.patch("builtins.open", mock.mock_open()) as mock_file: + baselines.write_baseline_tput("/tmp", "/tmp/cpl.log") + + mock_file.assert_called_with("/tmp/cpl-tput.log", "w") + mock_file.return_value.write.assert_called_with("-1") + + @mock.patch("CIME.baselines.get_throughput") + def test_write_baseline_tput(self, get_throughput): + get_throughput.return_value = 200 + + with mock.patch("builtins.open", mock.mock_open()) as mock_file: + baselines.write_baseline_tput("/tmp", "/tmp/cpl.log") + + mock_file.assert_called_with("/tmp/cpl-tput.log", "w") + mock_file.return_value.write.assert_called_with("200") + + @mock.patch("CIME.baselines.get_throughput") + @mock.patch("CIME.baselines.read_baseline_tput") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_compare_throughput_no_baseline_file(self, get_latest_cpl_logs, read_baseline_tput, get_throughput): + read_baseline_tput.side_effect = FileNotFoundError + + get_throughput.return_value = 504 + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + case.get_value.side_effect = ( + str(baseline_root), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + 0.05, + ) + + below_tolerance, diff, tolerance, baseline, current = baselines.compare_throughput(case) + + assert below_tolerance is None + assert diff == None + assert tolerance == 0.05 + assert baseline == None + assert current == None + + @mock.patch("CIME.baselines.get_throughput") + @mock.patch("CIME.baselines.read_baseline_tput") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_compare_throughput_no_baseline(self, get_latest_cpl_logs, read_baseline_tput, get_throughput): + read_baseline_tput.return_value = None + + get_throughput.return_value = 504 + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + case.get_value.side_effect = ( + str(baseline_root), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + 0.05, + ) + + below_tolerance, diff, tolerance, baseline, current = baselines.compare_throughput(case) + + assert below_tolerance is None + assert diff == None + assert tolerance == 0.05 + assert baseline == None + assert current == 504 + + @mock.patch("CIME.baselines.get_throughput") + @mock.patch("CIME.baselines.read_baseline_tput") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_compare_throughput_no_tolerance(self, get_latest_cpl_logs, read_baseline_tput, get_throughput): + read_baseline_tput.return_value = 500 + + get_throughput.return_value = 504 + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + case.get_value.side_effect = ( + str(baseline_root), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + None + ) + + below_tolerance, diff, tolerance, baseline, current = baselines.compare_throughput(case) + + assert below_tolerance + assert diff == -0.008 + assert tolerance == 0.1 + assert baseline == 500 + assert current == 504 + + @mock.patch("CIME.baselines.get_throughput") + @mock.patch("CIME.baselines.read_baseline_tput") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_compare_throughput(self, get_latest_cpl_logs, read_baseline_tput, get_throughput): + read_baseline_tput.return_value = 500 + + get_throughput.return_value = 504 + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + case.get_value.side_effect = ( + str(baseline_root), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + 0.05, + ) + + below_tolerance, diff, tolerance, baseline, current = baselines.compare_throughput(case) + + assert below_tolerance + assert diff == -0.008 + assert tolerance == 0.05 + assert baseline == 500 + assert current == 504 + + @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.read_baseline_mem") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_compare_memory_no_baseline(self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage): + read_baseline_mem.return_value = None + + get_mem_usage.return_value = [ + (1, 1000.0), + (2, 1001.0), + (3, 1002.0), + (4, 1003.0), + ] + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + case.get_value.side_effect = ( + str(baseline_root), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + 0.05, + ) + + below_tolerance, diff, tolerance, baseline, current = baselines.compare_memory(case) + + assert below_tolerance + assert diff == 0.0 + assert tolerance == 0.05 + assert baseline == 0.0 + assert current == 1003.0 + + @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.read_baseline_mem") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_compare_memory_not_enough_samples(self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage): + read_baseline_mem.return_value = 1000.0 + + get_mem_usage.return_value = [ + (1, 1000.0), + (2, 1001.0), + ] + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + case.get_value.side_effect = ( + str(baseline_root), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + 0.05, + ) + + below_tolerance, diff, tolerance, baseline, current = baselines.compare_memory(case) + + assert below_tolerance is None + assert diff == None + assert tolerance == 0.05 + assert baseline == 1000.0 + assert current == None + + @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.read_baseline_mem") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_compare_memory_no_baseline_file(self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage): + read_baseline_mem.side_effect = FileNotFoundError + + get_mem_usage.return_value = [ + (1, 1000.0), + (2, 1001.0), + (3, 1002.0), + (4, 1003.0), + ] + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + case.get_value.side_effect = ( + str(baseline_root), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + 0.05, + ) + + below_tolerance, diff, tolerance, baseline, current = baselines.compare_memory(case) + + assert below_tolerance is None + assert diff == None + assert tolerance == 0.05 + assert baseline == None + assert current == None + + @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.read_baseline_mem") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_compare_memory_no_tolerance(self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage): + read_baseline_mem.return_value = 1000.0 + + get_mem_usage.return_value = [ + (1, 1000.0), + (2, 1001.0), + (3, 1002.0), + (4, 1003.0), + ] + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + case.get_value.side_effect = ( + str(baseline_root), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + None, + ) + + below_tolerance, diff, tolerance, baseline, current = baselines.compare_memory(case) + + assert below_tolerance + assert diff == 0.003 + assert tolerance == 0.1 + assert baseline == 1000.0 + assert current == 1003.0 + + @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.read_baseline_mem") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_compare_memory(self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage): + read_baseline_mem.return_value = 1000.0 + + get_mem_usage.return_value = [ + (1, 1000.0), + (2, 1001.0), + (3, 1002.0), + (4, 1003.0), + ] + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + case.get_value.side_effect = ( + str(baseline_root), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + 0.05, + ) + + below_tolerance, diff, tolerance, baseline, current = baselines.compare_memory(case) + + assert below_tolerance + assert diff == 0.003 + assert tolerance == 0.05 + assert baseline == 1000.0 + assert current == 1003.0 + + def test_get_latest_cpl_logs_found_multiple(self): + with tempfile.TemporaryDirectory() as tempdir: + run_dir = Path(tempdir) / "run" + run_dir.mkdir(parents=True, exist_ok=False) + + cpl_log_path = run_dir / "cpl.log.gz" + cpl_log_path.touch() + + cpl_log_2_path = run_dir / "cpl-2023-01-01.log.gz" + cpl_log_2_path.touch() + + case = mock.MagicMock() + case.get_value.side_effect = ( + str(run_dir), + "mct", + ) + + latest_cpl_logs = baselines.get_latest_cpl_logs(case) + + assert len(latest_cpl_logs) == 2 + assert sorted(latest_cpl_logs) == sorted([str(cpl_log_path), str(cpl_log_2_path)]) + + def test_get_latest_cpl_logs_found_single(self): + with tempfile.TemporaryDirectory() as tempdir: + run_dir = Path(tempdir) / "run" + run_dir.mkdir(parents=True, exist_ok=False) + + cpl_log_path = run_dir / "cpl.log.gz" + cpl_log_path.touch() + + case = mock.MagicMock() + case.get_value.side_effect = ( + str(run_dir), + "mct", + ) + + latest_cpl_logs = baselines.get_latest_cpl_logs(case) + + assert len(latest_cpl_logs) == 1 + assert latest_cpl_logs[0] == str(cpl_log_path) + + def test_get_latest_cpl_logs(self): + case = mock.MagicMock() + case.get_value.side_effect = ( + f"/tmp/run", + "mct", + ) + + latest_cpl_logs = baselines.get_latest_cpl_logs(case) + + assert len(latest_cpl_logs) == 0 diff --git a/CIME/tests/test_unit_system_tests.py b/CIME/tests/test_unit_system_tests.py index e456b675047..e81bd6bca10 100644 --- a/CIME/tests/test_unit_system_tests.py +++ b/CIME/tests/test_unit_system_tests.py @@ -66,169 +66,169 @@ def create_mock_case(tempdir, idx=None, cpllog_data=None): class TestCaseSubmit(unittest.TestCase): + @mock.patch("CIME.SystemTests.system_tests_common.compare_throughput") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - def test_compare_throughput(self, append_testlog): - with tempfile.TemporaryDirectory() as tempdir: - case, caseroot, baseline_root, run_dir = create_mock_case( - tempdir, cpllog_data=CPLLOG - ) + def test_compare_throughput(self, append_testlog, compare_throughput): + compare_throughput.return_value = (True, 0.02, 0.05, 200, 201) - base_case, base_caseroot, _, _ = create_mock_case( - tempdir, idx=1, cpllog_data=CPLLOG - ) + with tempfile.TemporaryDirectory() as tempdir: + caseroot = Path(tempdir) / "caseroot" + caseroot.mkdir(parents=True, exist_ok=False) + case = mock.MagicMock() case.get_value.side_effect = ( - str(caseroot), + str(Path(tempdir) / "caseroot"), "ERIO.ne30_g16_rx1.A.docker_gnu", "mct", - "master/ERIO.ne30_g16_rx1.A.docker_gnu", - str(baseline_root), - str(run_dir), - 0.05, ) - baseline_dir = baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" - baseline_dir.mkdir(parents=True, exist_ok=False) - baseline_mem = baseline_dir / "cpl-mem.log" + common = SystemTestsCommon(case) - with open(baseline_mem, "w") as fd: - fd.write(str("1673.89")) + common._compare_throughput() - common = SystemTestsCommon(case) + assert common._test_status.get_overall_test_status() == ("PASS", None) - common._compare_memory() + append_testlog.assert_any_call( + "TPUTCOMP: Computation time changed by 2.00% relative to baseline", + str(caseroot), + ) - assert common._test_status.get_overall_test_status() == ("PASS", None) + @mock.patch("CIME.SystemTests.system_tests_common.compare_throughput") + @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") + def test_compare_throughput_error_diff(self, append_testlog, compare_throughput): + compare_throughput.return_value = (None, 0.02, 0.05, 200, 201) - append_testlog.assert_any_call( - "MEMCOMP: Memory usage highwater has changed by 0.00% relative to baseline", - str(caseroot), + with tempfile.TemporaryDirectory() as tempdir: + caseroot = Path(tempdir) / "caseroot" + caseroot.mkdir(parents=True, exist_ok=False) + + case = mock.MagicMock() + case.get_value.side_effect = ( + str(Path(tempdir) / "caseroot"), + "ERIO.ne30_g16_rx1.A.docker_gnu", + "mct", ) + common = SystemTestsCommon(case) + + common._compare_throughput() + + assert common._test_status.get_overall_test_status() == ("PASS", None) + + append_testlog.assert_not_called() + + @mock.patch("CIME.SystemTests.system_tests_common.compare_throughput") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - def test_compare_throughput_fail(self, append_testlog): + def test_compare_throughput_fail(self, append_testlog, compare_throughput): + compare_throughput.return_value = (False, 0.02, 0.05, 200, 201) + with tempfile.TemporaryDirectory() as tempdir: - case, caseroot, baseline_root, run_dir = create_mock_case( - tempdir, cpllog_data=CPLLOG - ) + caseroot = Path(tempdir) / "caseroot" + caseroot.mkdir(parents=True, exist_ok=False) - base_case, base_caseroot, _, _ = create_mock_case( - tempdir, idx=1, cpllog_data=CPLLOG + case = mock.MagicMock() + case.get_value.side_effect = ( + str(Path(tempdir) / "caseroot"), + "ERIO.ne30_g16_rx1.A.docker_gnu", + "mct", ) + common = SystemTestsCommon(case) + + common._compare_throughput() + + assert common._test_status.get_overall_test_status() == ("PASS", None) + + append_testlog.assert_any_call( + "TPUTCOMP: Computation time changed by 2.00% relative to baseline", + str(caseroot), + ) + append_testlog.assert_any_call( + "Error: TPUTCOMP: Computation time increase > 5% from baseline", + str(caseroot), + ) + + @mock.patch("CIME.SystemTests.system_tests_common.compare_memory") + @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") + def test_compare_memory(self, append_testlog, compare_memory): + compare_memory.return_value = (True, 0.02, 0.05, 1000, 1002) + + with tempfile.TemporaryDirectory() as tempdir: + caseroot = Path(tempdir) / "caseroot" + caseroot.mkdir(parents=True, exist_ok=False) + + case = mock.MagicMock() case.get_value.side_effect = ( str(caseroot), "ERIO.ne30_g16_rx1.A.docker_gnu", "mct", - "master/ERIO.ne30_g16_rx1.A.docker_gnu", - str(baseline_root), - str(run_dir), - 0.05, ) - baseline_dir = baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" - baseline_dir.mkdir(parents=True, exist_ok=False) - baseline_mem = baseline_dir / "cpl-mem.log" - - with open(baseline_mem, "w") as fd: - fd.write(str("1473.89")) - common = SystemTestsCommon(case) common._compare_memory() - assert common._test_status.get_overall_test_status() == ("PASS", None) + assert common._test_status.get_overall_test_status() == ("PASS", None) - append_testlog.assert_any_call( - "MEMCOMP: Memory usage highwater has changed by 13.57% relative to baseline", - str(caseroot), - ) - append_testlog.assert_any_call( - "Error: Memory usage increase >5% from baseline's 1473.890000 to 1673.890000", - str(caseroot), - ) + append_testlog.assert_any_call( + "MEMCOMP: Memory usage highwater has changed by 2.00% relative to baseline", + str(caseroot), + ) + @mock.patch("CIME.SystemTests.system_tests_common.compare_memory") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - def test_compare_memory(self, append_testlog): - with tempfile.TemporaryDirectory() as tempdir: - case, caseroot, baseline_root, run_dir = create_mock_case( - tempdir, cpllog_data=CPLLOG - ) + def test_compare_memory_erorr_diff(self, append_testlog, compare_memory): + compare_memory.return_value = (None, 0.02, 0.05, 1000, 1002) - base_case, base_caseroot, _, _ = create_mock_case( - tempdir, idx=1, cpllog_data=CPLLOG - ) + with tempfile.TemporaryDirectory() as tempdir: + caseroot = Path(tempdir) / "caseroot" + caseroot.mkdir(parents=True, exist_ok=False) + case = mock.MagicMock() case.get_value.side_effect = ( str(caseroot), "ERIO.ne30_g16_rx1.A.docker_gnu", "mct", - "master/ERIO.ne30_g16_rx1.A.docker_gnu", - str(baseline_root), - str(run_dir), - 0.05, ) - baseline_dir = baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" - baseline_dir.mkdir(parents=True, exist_ok=False) - baseline_tput = baseline_dir / "cpl-tput.log" - - with open(baseline_tput, "w") as fd: - fd.write(str("719.635")) - common = SystemTestsCommon(case) - common._compare_throughput() + common._compare_memory() - assert common._test_status.get_overall_test_status() == ("PASS", None) + assert common._test_status.get_overall_test_status() == ("PASS", None) - append_testlog.assert_any_call( - "TPUTCOMP: Computation time changed by 0.00% relative to baseline", - str(caseroot), - ) + append_testlog.assert_not_called() + @mock.patch("CIME.SystemTests.system_tests_common.compare_memory") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - def test_compare_memory_fail(self, append_testlog): - with tempfile.TemporaryDirectory() as tempdir: - case, caseroot, baseline_root, run_dir = create_mock_case( - tempdir, cpllog_data=CPLLOG - ) + def test_compare_memory_erorr_fail(self, append_testlog, compare_memory): + compare_memory.return_value = (False, 0.02, 0.05, 1000, 1002) - base_case, base_caseroot, _, _ = create_mock_case( - tempdir, idx=1, cpllog_data=CPLLOG - ) + with tempfile.TemporaryDirectory() as tempdir: + caseroot = Path(tempdir) / "caseroot" + caseroot.mkdir(parents=True, exist_ok=False) + case = mock.MagicMock() case.get_value.side_effect = ( str(caseroot), "ERIO.ne30_g16_rx1.A.docker_gnu", "mct", - "master/ERIO.ne30_g16_rx1.A.docker_gnu", - str(baseline_root), - str(run_dir), - 0.05, ) - baseline_dir = baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" - baseline_dir.mkdir(parents=True, exist_ok=False) - baseline_tput = baseline_dir / "cpl-tput.log" - - with open(baseline_tput, "w") as fd: - fd.write(str("900.635")) - common = SystemTestsCommon(case) - common._compare_throughput() + common._compare_memory() - assert common._test_status.get_overall_test_status() == ("PASS", None) + assert common._test_status.get_overall_test_status() == ("PASS", None) - append_testlog.assert_any_call( - "TPUTCOMP: Computation time changed by 20.10% relative to baseline", - str(caseroot), - ) - append_testlog.assert_any_call( - "Error: TPUTCOMP: Computation time increase > 5% from baseline", - str(caseroot), - ) + append_testlog.assert_any_call( + "MEMCOMP: Memory usage highwater has changed by 2.00% relative to baseline", + str(caseroot), + ) + append_testlog.assert_any_call( + "Error: Memory usage increase >5% from baseline's 1000.000000 to 1002.000000", + str(caseroot), + ) def test_generate_baseline(self): with tempfile.TemporaryDirectory() as tempdir: @@ -254,6 +254,7 @@ def test_generate_baseline(self): str(baseline_root), "master/ERIO.ne30_g16_rx1.A.docker_gnu", str(run_dir), + "mct", ) common = SystemTestsCommon(case) From 5ac9e1ea3748ae81729fea0116781344e0447abf Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 22 Sep 2023 15:34:40 -0700 Subject: [PATCH 07/31] Refactors compare_throughput and compare_memory functions --- CIME/SystemTests/system_tests_common.py | 31 ++-- CIME/baselines.py | 107 ++++++----- CIME/tests/test_unit_baselines.py | 230 +++++++++++++++--------- CIME/tests/test_unit_system_tests.py | 32 ++-- 4 files changed, 240 insertions(+), 160 deletions(-) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index 37dd63ad49c..e1c5d59388c 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -28,6 +28,7 @@ from CIME.locked_files import LOCKED_DIR, lock_file, is_locked from CIME.baselines import ( get_latest_cpl_logs, + get_throughput, get_mem_usage, compare_memory, compare_throughput, @@ -704,15 +705,9 @@ def _compare_memory(self): Compares current test memory usage to baseline. """ with self._test_status: - below_tolerance, diff, tolerance, baseline, current = compare_memory( - self._case - ) + below_tolerance, comment = compare_memory(self._case) if below_tolerance is not None: - comment = "MEMCOMP: Memory usage highwater has changed by {:.2f}% relative to baseline".format( - diff * 100 - ) - append_testlog(comment, self._orig_caseroot) if ( @@ -721,39 +716,29 @@ def _compare_memory(self): ): self._test_status.set_status(MEMCOMP_PHASE, TEST_PASS_STATUS) elif self._test_status.get_status(MEMCOMP_PHASE) != TEST_FAIL_STATUS: - comment = "Error: Memory usage increase >{:d}% from baseline's {:f} to {:f}".format( - int(tolerance * 100), baseline, current - ) self._test_status.set_status( MEMCOMP_PHASE, TEST_FAIL_STATUS, comments=comment ) - append_testlog(comment, self._orig_caseroot) def _compare_throughput(self): """ Compares current test throughput to baseline. """ with self._test_status: - below_tolerance, diff, tolerance, _, _ = compare_throughput(self._case) + below_tolerance, comment = compare_throughput(self._case) if below_tolerance is not None: - comment = "TPUTCOMP: Computation time changed by {:.2f}% relative to baseline".format( - diff * 100 - ) append_testlog(comment, self._orig_caseroot) + if ( below_tolerance and self._test_status.get_status(THROUGHPUT_PHASE) is None ): self._test_status.set_status(THROUGHPUT_PHASE, TEST_PASS_STATUS) elif self._test_status.get_status(THROUGHPUT_PHASE) != TEST_FAIL_STATUS: - comment = "Error: TPUTCOMP: Computation time increase > {:d}% from baseline".format( - int(tolerance * 100) - ) self._test_status.set_status( THROUGHPUT_PHASE, TEST_FAIL_STATUS, comments=comment ) - append_testlog(comment, self._orig_caseroot) def _compare_baseline(self): """ @@ -807,9 +792,13 @@ def _generate_baseline(self): preserve_meta=False, ) - write_baseline_tput(basegen_dir, cpllog) + tput = get_throughput(cpllog) + + write_baseline_tput(basegen_dir, tput) + + mem = get_mem_usage(cpllog) - write_baseline_mem(basegen_dir, cpllog) + write_baseline_mem(basegen_dir, mem) class FakeTest(SystemTestsCommon): diff --git a/CIME/baselines.py b/CIME/baselines.py index e371488f294..cbdb3a2cf61 100644 --- a/CIME/baselines.py +++ b/CIME/baselines.py @@ -7,15 +7,14 @@ logger = logging.getLogger(__name__) + def get_latest_cpl_logs(case): """ find and return the latest cpl log file in the run directory """ coupler_log_path = case.get_value("RUNDIR") cpllog_name = "drv" if case.get_value("COMP_INTERFACE") == "nuopc" else "cpl" - cpllogs = glob.glob( - os.path.join(coupler_log_path, "{}*.log.*".format(cpllog_name)) - ) + cpllogs = glob.glob(os.path.join(coupler_log_path, "{}*.log.*".format(cpllog_name))) lastcpllogs = [] if cpllogs: lastcpllogs.append(max(cpllogs, key=os.path.getctime)) @@ -30,22 +29,27 @@ def get_latest_cpl_logs(case): return lastcpllogs -def compare_memory(case): - baseline_root = case.get_value("BASELINE_ROOT") - baseline_name = case.get_value("BASECMP_CASE") +def compare_memory(case, baseline_dir=None): + if baseline_dir is None: + baseline_root = case.get_value("BASELINE_ROOT") - baseline_dir = os.path.join(baseline_root, baseline_name) + baseline_name = case.get_value("BASECMP_CASE") + + baseline_dir = os.path.join(baseline_root, baseline_name) latest_cpl_logs = get_latest_cpl_logs(case) - diff, baseline, current = [None]*3 + diff, baseline, current = [None] * 3 + comment = "" for cpllog in latest_cpl_logs: try: baseline = read_baseline_mem(baseline_dir) except FileNotFoundError as e: - logger.debug("Could not read baseline memory usage: %s", e) + comment = f"Could not read baseline memory usage: {e!s}" + + logger.debug(comment) continue @@ -55,7 +59,9 @@ def compare_memory(case): memlist = get_mem_usage(cpllog) if len(memlist) <= 3: - logger.debug(f"Found {len(memlist)} memory usage samples, need atleast 4") + comment = f"Found {len(memlist)} memory usage samples, need atleast 4" + + logger.debug(comment) continue @@ -77,25 +83,39 @@ def compare_memory(case): if diff is not None: below_tolerance = diff < tolerance - return below_tolerance, diff, tolerance, baseline, current + if below_tolerance: + comment = "MEMCOMP: Memory usage highwater has changed by {:.2f}% relative to baseline".format( + diff * 100 + ) + else: + comment = "Error: Memory usage increase >{:d}% from baseline's {:f} to {:f}".format( + int(tolerance * 100), baseline, current + ) -def compare_throughput(case): - baseline_root = case.get_value("BASELINE_ROOT") + return below_tolerance, comment - baseline_name = case.get_value("BASECMP_CASE") - baseline_dir = os.path.join(baseline_root, baseline_name) +def compare_throughput(case, baseline_dir=None): + if baseline_dir is None: + baseline_root = case.get_value("BASELINE_ROOT") + + baseline_name = case.get_value("BASECMP_CASE") + + baseline_dir = os.path.join(baseline_root, baseline_name) latest_cpl_logs = get_latest_cpl_logs(case) - diff, baseline, current = [None]*3 + diff, baseline, current = [None] * 3 + comment = "" # Do we need this loop, when are there multiple cpl logs? for cpllog in latest_cpl_logs: try: baseline = read_baseline_tput(baseline_dir) - except (FileNotFoundError, IndexError, ValueError) as e: - logger.debug("Could not read baseline throughput: %s", e) + except FileNotFoundError as e: + comment = f"Could not read baseline throughput file: {e!s}" + + logger.debug(comment) continue @@ -106,11 +126,9 @@ def compare_throughput(case): diff = (baseline - current) / baseline except (ValueError, TypeError): # Should we default the diff to 0.0 as with _compare_current_memory? - logger.debug( - "Could not determine change in throughput between baseline %s and current %s", - baseline, - current, - ) + comment = f"Could not determine diff with baseline {baseline!r} and current {current!r}" + + logger.debug(comment) continue @@ -129,9 +147,19 @@ def compare_throughput(case): if diff is not None: below_tolerance = diff < tolerance - return below_tolerance, diff, tolerance, baseline, current + if below_tolerance: + comment = "TPUTCOMP: Computation time changed by {:.2f}% relative to baseline".format( + diff * 100 + ) + else: + comment = "Error: TPUTCOMP: Computation time increase > {:d}% from baseline".format( + int(tolerance * 100) + ) + + return below_tolerance, comment + -def write_baseline_tput(baseline_dir, cpllog): +def write_baseline_tput(baseline_dir, tput): """ Writes baseline throughput to file. @@ -146,20 +174,17 @@ def write_baseline_tput(baseline_dir, cpllog): """ tput_file = os.path.join(baseline_dir, "cpl-tput.log") - tput = get_throughput(cpllog) - with open(tput_file, "w") as fd: fd.write("# Throughput in simulated years per compute day\n") - fd.write( - "# A -1 indicates no throughput data was available from the test\n" - ) + fd.write("# A -1 indicates no throughput data was available from the test\n") if tput is None: fd.write("-1") else: fd.write(str(tput)) -def write_baseline_mem(baseline_dir, cpllog): + +def write_baseline_mem(baseline_dir, mem): """ Writes baseline memory usage highwater to file. @@ -174,19 +199,16 @@ def write_baseline_mem(baseline_dir, cpllog): """ mem_file = os.path.join(baseline_dir, "cpl-mem.log") - mem = get_mem_usage(cpllog) - with open(mem_file, "w") as fd: fd.write("# Memory usage highwater\n") - fd.write( - "# A -1 indicates no memory usage data was available from the test\n" - ) + fd.write("# A -1 indicates no memory usage data was available from the test\n") try: fd.write(str(mem[-1][1])) except IndexError: fd.write("-1") + def read_baseline_tput(baseline_dir): """ Reads throughput baseline. @@ -220,6 +242,7 @@ def read_baseline_tput(baseline_dir): return tput + def read_baseline_mem(baseline_dir): """ Reads memory usage highwater baseline. @@ -242,9 +265,7 @@ def read_baseline_mem(baseline_dir): If baseline file does not exist. """ try: - memory = read_baseline_value( - os.path.join(baseline_dir, "cpl-mem.log") - ) + memory = read_baseline_value(os.path.join(baseline_dir, "cpl-mem.log")) except (IndexError, ValueError): memory = 0 @@ -253,6 +274,7 @@ def read_baseline_mem(baseline_dir): return memory + def read_baseline_value(baseline_file): """ Read baseline value from file. @@ -281,15 +303,14 @@ def read_baseline_value(baseline_file): return float(lines[0]) + def get_mem_usage(cpllog): """ Examine memory usage as recorded in the cpl log file and look for unexpected increases. """ memlist = [] - meminfo = re.compile( - r".*model date =\s+(\w+).*memory =\s+(\d+\.?\d+).*highwater" - ) + meminfo = re.compile(r".*model date =\s+(\w+).*memory =\s+(\d+\.?\d+).*highwater") if cpllog is not None and os.path.isfile(cpllog): if ".gz" == cpllog[-3:]: fopen = gzip.open @@ -305,6 +326,7 @@ def get_mem_usage(cpllog): memlist.pop() return memlist + def get_throughput(cpllog): """ Examine memory usage as recorded in the cpl log file and look for unexpected @@ -317,4 +339,3 @@ def get_throughput(cpllog): if m: return float(m.group(1)) return None - diff --git a/CIME/tests/test_unit_baselines.py b/CIME/tests/test_unit_baselines.py index 77ce61ac20f..1d4f11d1c1e 100644 --- a/CIME/tests/test_unit_baselines.py +++ b/CIME/tests/test_unit_baselines.py @@ -9,13 +9,12 @@ from CIME import baselines from CIME.tests.test_unit_system_tests import CPLLOG + def create_mock_case(tempdir, get_latest_cpl_logs): caseroot = Path(tempdir, "0", "caseroot") rundir = caseroot / "run" - get_latest_cpl_logs.return_value = ( - str(rundir / "cpl.log.gz"), - ) + get_latest_cpl_logs.return_value = (str(rundir / "cpl.log.gz"),) baseline_root = Path(tempdir, "baselines") @@ -23,6 +22,7 @@ def create_mock_case(tempdir, get_latest_cpl_logs): return case, caseroot, rundir, baseline_root + class TestUnitBaseline(unittest.TestCase): def test_get_throughput_no_file(self): throughput = baselines.get_throughput("/tmp/cpl.log") @@ -49,16 +49,28 @@ def test_get_mem_usage_gz(self): mem_usage = baselines.get_mem_usage(str(cpl_log_path)) - assert mem_usage == [(10102.0, 1673.89), (10103.0, 1673.89), (10104.0, 1673.89), (10105.0, 1673.89)] + assert mem_usage == [ + (10102.0, 1673.89), + (10103.0, 1673.89), + (10104.0, 1673.89), + (10105.0, 1673.89), + ] @mock.patch("CIME.baselines.os.path.isfile") def test_get_mem_usage(self, isfile): isfile.return_value = True - with mock.patch("builtins.open", mock.mock_open(read_data=CPLLOG.encode("utf-8"))) as mock_file: + with mock.patch( + "builtins.open", mock.mock_open(read_data=CPLLOG.encode("utf-8")) + ) as mock_file: mem_usage = baselines.get_mem_usage("/tmp/cpl.log") - assert mem_usage == [(10102.0, 1673.89), (10103.0, 1673.89), (10104.0, 1673.89), (10105.0, 1673.89)] + assert mem_usage == [ + (10102.0, 1673.89), + (10103.0, 1673.89), + (10104.0, 1673.89), + (10105.0, 1673.89), + ] def test_read_baseline_mem_empty(self): with mock.patch("builtins.open", mock.mock_open(read_data="")) as mock_file: @@ -96,42 +108,30 @@ def test_read_baseline_tput(self): assert baseline == 200 - @mock.patch("CIME.baselines.get_mem_usage") - def test_write_baseline_mem_no_value(self, get_mem_usage): - get_mem_usage.return_value = [] - + def test_write_baseline_mem_no_value(self): with mock.patch("builtins.open", mock.mock_open()) as mock_file: - baselines.write_baseline_mem("/tmp", "/tmp/cpl.log") + baselines.write_baseline_mem("/tmp", []) mock_file.assert_called_with("/tmp/cpl-mem.log", "w") mock_file.return_value.write.assert_called_with("-1") - @mock.patch("CIME.baselines.get_mem_usage") - def test_write_baseline_mem(self, get_mem_usage): - get_mem_usage.return_value = [(1, 200)] - + def test_write_baseline_mem(self): with mock.patch("builtins.open", mock.mock_open()) as mock_file: - baselines.write_baseline_mem("/tmp", "/tmp/cpl.log") + baselines.write_baseline_mem("/tmp", [(1, 200)]) mock_file.assert_called_with("/tmp/cpl-mem.log", "w") mock_file.return_value.write.assert_called_with("200") - @mock.patch("CIME.baselines.get_throughput") - def test_write_baseline_tput_no_value(self, get_throughput): - get_throughput.return_value = None - + def test_write_baseline_tput_no_value(self): with mock.patch("builtins.open", mock.mock_open()) as mock_file: - baselines.write_baseline_tput("/tmp", "/tmp/cpl.log") + baselines.write_baseline_tput("/tmp", None) mock_file.assert_called_with("/tmp/cpl-tput.log", "w") mock_file.return_value.write.assert_called_with("-1") - @mock.patch("CIME.baselines.get_throughput") - def test_write_baseline_tput(self, get_throughput): - get_throughput.return_value = 200 - + def test_write_baseline_tput(self): with mock.patch("builtins.open", mock.mock_open()) as mock_file: - baselines.write_baseline_tput("/tmp", "/tmp/cpl.log") + baselines.write_baseline_tput("/tmp", 200) mock_file.assert_called_with("/tmp/cpl-tput.log", "w") mock_file.return_value.write.assert_called_with("200") @@ -139,7 +139,9 @@ def test_write_baseline_tput(self, get_throughput): @mock.patch("CIME.baselines.get_throughput") @mock.patch("CIME.baselines.read_baseline_tput") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_throughput_no_baseline_file(self, get_latest_cpl_logs, read_baseline_tput, get_throughput): + def test_compare_throughput_no_baseline_file( + self, get_latest_cpl_logs, read_baseline_tput, get_throughput + ): read_baseline_tput.side_effect = FileNotFoundError get_throughput.return_value = 504 @@ -153,18 +155,17 @@ def test_compare_throughput_no_baseline_file(self, get_latest_cpl_logs, read_bas 0.05, ) - below_tolerance, diff, tolerance, baseline, current = baselines.compare_throughput(case) + (below_tolerance, comment) = baselines.compare_throughput(case) assert below_tolerance is None - assert diff == None - assert tolerance == 0.05 - assert baseline == None - assert current == None + assert comment == "Could not read baseline throughput file: " @mock.patch("CIME.baselines.get_throughput") @mock.patch("CIME.baselines.read_baseline_tput") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_throughput_no_baseline(self, get_latest_cpl_logs, read_baseline_tput, get_throughput): + def test_compare_throughput_no_baseline( + self, get_latest_cpl_logs, read_baseline_tput, get_throughput + ): read_baseline_tput.return_value = None get_throughput.return_value = 504 @@ -178,18 +179,17 @@ def test_compare_throughput_no_baseline(self, get_latest_cpl_logs, read_baseline 0.05, ) - below_tolerance, diff, tolerance, baseline, current = baselines.compare_throughput(case) + (below_tolerance, comment) = baselines.compare_throughput(case) assert below_tolerance is None - assert diff == None - assert tolerance == 0.05 - assert baseline == None - assert current == 504 + assert comment == "Could not determine diff with baseline None and current 504" @mock.patch("CIME.baselines.get_throughput") @mock.patch("CIME.baselines.read_baseline_tput") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_throughput_no_tolerance(self, get_latest_cpl_logs, read_baseline_tput, get_throughput): + def test_compare_throughput_no_tolerance( + self, get_latest_cpl_logs, read_baseline_tput, get_throughput + ): read_baseline_tput.return_value = 500 get_throughput.return_value = 504 @@ -200,21 +200,49 @@ def test_compare_throughput_no_tolerance(self, get_latest_cpl_logs, read_baselin case.get_value.side_effect = ( str(baseline_root), "master/ERIO.ne30_g16_rx1.A.docker_gnu", - None + None, ) - below_tolerance, diff, tolerance, baseline, current = baselines.compare_throughput(case) + (below_tolerance, comment) = baselines.compare_throughput(case) assert below_tolerance - assert diff == -0.008 - assert tolerance == 0.1 - assert baseline == 500 - assert current == 504 + assert ( + comment + == "TPUTCOMP: Computation time changed by -0.80% relative to baseline" + ) @mock.patch("CIME.baselines.get_throughput") @mock.patch("CIME.baselines.read_baseline_tput") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_throughput(self, get_latest_cpl_logs, read_baseline_tput, get_throughput): + def test_compare_throughput_above_threshold( + self, get_latest_cpl_logs, read_baseline_tput, get_throughput + ): + read_baseline_tput.return_value = 1000 + + get_throughput.return_value = 504 + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + case.get_value.side_effect = ( + str(baseline_root), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + 0.05, + ) + + (below_tolerance, comment) = baselines.compare_throughput(case) + + assert not below_tolerance + assert ( + comment == "Error: TPUTCOMP: Computation time increase > 5% from baseline" + ) + + @mock.patch("CIME.baselines.get_throughput") + @mock.patch("CIME.baselines.read_baseline_tput") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_compare_throughput( + self, get_latest_cpl_logs, read_baseline_tput, get_throughput + ): read_baseline_tput.return_value = 500 get_throughput.return_value = 504 @@ -228,18 +256,20 @@ def test_compare_throughput(self, get_latest_cpl_logs, read_baseline_tput, get_t 0.05, ) - below_tolerance, diff, tolerance, baseline, current = baselines.compare_throughput(case) + (below_tolerance, comment) = baselines.compare_throughput(case) assert below_tolerance - assert diff == -0.008 - assert tolerance == 0.05 - assert baseline == 500 - assert current == 504 + assert ( + comment + == "TPUTCOMP: Computation time changed by -0.80% relative to baseline" + ) @mock.patch("CIME.baselines.get_mem_usage") @mock.patch("CIME.baselines.read_baseline_mem") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_memory_no_baseline(self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage): + def test_compare_memory_no_baseline( + self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage + ): read_baseline_mem.return_value = None get_mem_usage.return_value = [ @@ -258,18 +288,20 @@ def test_compare_memory_no_baseline(self, get_latest_cpl_logs, read_baseline_mem 0.05, ) - below_tolerance, diff, tolerance, baseline, current = baselines.compare_memory(case) + (below_tolerance, comment) = baselines.compare_memory(case) assert below_tolerance - assert diff == 0.0 - assert tolerance == 0.05 - assert baseline == 0.0 - assert current == 1003.0 + assert ( + comment + == "MEMCOMP: Memory usage highwater has changed by 0.00% relative to baseline" + ) @mock.patch("CIME.baselines.get_mem_usage") @mock.patch("CIME.baselines.read_baseline_mem") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_memory_not_enough_samples(self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage): + def test_compare_memory_not_enough_samples( + self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage + ): read_baseline_mem.return_value = 1000.0 get_mem_usage.return_value = [ @@ -286,18 +318,17 @@ def test_compare_memory_not_enough_samples(self, get_latest_cpl_logs, read_basel 0.05, ) - below_tolerance, diff, tolerance, baseline, current = baselines.compare_memory(case) + (below_tolerance, comment) = baselines.compare_memory(case) assert below_tolerance is None - assert diff == None - assert tolerance == 0.05 - assert baseline == 1000.0 - assert current == None + assert comment == "Found 2 memory usage samples, need atleast 4" @mock.patch("CIME.baselines.get_mem_usage") @mock.patch("CIME.baselines.read_baseline_mem") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_memory_no_baseline_file(self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage): + def test_compare_memory_no_baseline_file( + self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage + ): read_baseline_mem.side_effect = FileNotFoundError get_mem_usage.return_value = [ @@ -316,18 +347,17 @@ def test_compare_memory_no_baseline_file(self, get_latest_cpl_logs, read_baselin 0.05, ) - below_tolerance, diff, tolerance, baseline, current = baselines.compare_memory(case) + (below_tolerance, comment) = baselines.compare_memory(case) assert below_tolerance is None - assert diff == None - assert tolerance == 0.05 - assert baseline == None - assert current == None + assert comment == "Could not read baseline memory usage: " @mock.patch("CIME.baselines.get_mem_usage") @mock.patch("CIME.baselines.read_baseline_mem") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_memory_no_tolerance(self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage): + def test_compare_memory_no_tolerance( + self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage + ): read_baseline_mem.return_value = 1000.0 get_mem_usage.return_value = [ @@ -346,18 +376,52 @@ def test_compare_memory_no_tolerance(self, get_latest_cpl_logs, read_baseline_me None, ) - below_tolerance, diff, tolerance, baseline, current = baselines.compare_memory(case) + (below_tolerance, comment) = baselines.compare_memory(case) assert below_tolerance - assert diff == 0.003 - assert tolerance == 0.1 - assert baseline == 1000.0 - assert current == 1003.0 + assert ( + comment + == "MEMCOMP: Memory usage highwater has changed by 0.30% relative to baseline" + ) + + @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.read_baseline_mem") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_compare_memory_above_threshold( + self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage + ): + read_baseline_mem.return_value = 1000.0 + + get_mem_usage.return_value = [ + (1, 2000.0), + (2, 2001.0), + (3, 2002.0), + (4, 2003.0), + ] + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + case.get_value.side_effect = ( + str(baseline_root), + "master/ERIO.ne30_g16_rx1.A.docker_gnu", + 0.05, + ) + + (below_tolerance, comment) = baselines.compare_memory(case) + + assert not below_tolerance + assert ( + comment + == "Error: Memory usage increase >5% from baseline's 1000.000000 to 2003.000000" + ) @mock.patch("CIME.baselines.get_mem_usage") @mock.patch("CIME.baselines.read_baseline_mem") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_memory(self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage): + def test_compare_memory( + self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage + ): read_baseline_mem.return_value = 1000.0 get_mem_usage.return_value = [ @@ -376,13 +440,13 @@ def test_compare_memory(self, get_latest_cpl_logs, read_baseline_mem, get_mem_us 0.05, ) - below_tolerance, diff, tolerance, baseline, current = baselines.compare_memory(case) + (below_tolerance, comment) = baselines.compare_memory(case) assert below_tolerance - assert diff == 0.003 - assert tolerance == 0.05 - assert baseline == 1000.0 - assert current == 1003.0 + assert ( + comment + == "MEMCOMP: Memory usage highwater has changed by 0.30% relative to baseline" + ) def test_get_latest_cpl_logs_found_multiple(self): with tempfile.TemporaryDirectory() as tempdir: @@ -404,7 +468,9 @@ def test_get_latest_cpl_logs_found_multiple(self): latest_cpl_logs = baselines.get_latest_cpl_logs(case) assert len(latest_cpl_logs) == 2 - assert sorted(latest_cpl_logs) == sorted([str(cpl_log_path), str(cpl_log_2_path)]) + assert sorted(latest_cpl_logs) == sorted( + [str(cpl_log_path), str(cpl_log_2_path)] + ) def test_get_latest_cpl_logs_found_single(self): with tempfile.TemporaryDirectory() as tempdir: diff --git a/CIME/tests/test_unit_system_tests.py b/CIME/tests/test_unit_system_tests.py index e81bd6bca10..a43b7ef8f80 100644 --- a/CIME/tests/test_unit_system_tests.py +++ b/CIME/tests/test_unit_system_tests.py @@ -69,7 +69,10 @@ class TestCaseSubmit(unittest.TestCase): @mock.patch("CIME.SystemTests.system_tests_common.compare_throughput") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") def test_compare_throughput(self, append_testlog, compare_throughput): - compare_throughput.return_value = (True, 0.02, 0.05, 200, 201) + compare_throughput.return_value = ( + True, + "TPUTCOMP: Computation time changed by 2.00% relative to baseline", + ) with tempfile.TemporaryDirectory() as tempdir: caseroot = Path(tempdir) / "caseroot" @@ -96,7 +99,7 @@ def test_compare_throughput(self, append_testlog, compare_throughput): @mock.patch("CIME.SystemTests.system_tests_common.compare_throughput") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") def test_compare_throughput_error_diff(self, append_testlog, compare_throughput): - compare_throughput.return_value = (None, 0.02, 0.05, 200, 201) + compare_throughput.return_value = (None, "Error diff value") with tempfile.TemporaryDirectory() as tempdir: caseroot = Path(tempdir) / "caseroot" @@ -120,7 +123,10 @@ def test_compare_throughput_error_diff(self, append_testlog, compare_throughput) @mock.patch("CIME.SystemTests.system_tests_common.compare_throughput") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") def test_compare_throughput_fail(self, append_testlog, compare_throughput): - compare_throughput.return_value = (False, 0.02, 0.05, 200, 201) + compare_throughput.return_value = ( + False, + "Error: TPUTCOMP: Computation time increase > 5% from baseline", + ) with tempfile.TemporaryDirectory() as tempdir: caseroot = Path(tempdir) / "caseroot" @@ -139,10 +145,6 @@ def test_compare_throughput_fail(self, append_testlog, compare_throughput): assert common._test_status.get_overall_test_status() == ("PASS", None) - append_testlog.assert_any_call( - "TPUTCOMP: Computation time changed by 2.00% relative to baseline", - str(caseroot), - ) append_testlog.assert_any_call( "Error: TPUTCOMP: Computation time increase > 5% from baseline", str(caseroot), @@ -151,7 +153,10 @@ def test_compare_throughput_fail(self, append_testlog, compare_throughput): @mock.patch("CIME.SystemTests.system_tests_common.compare_memory") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") def test_compare_memory(self, append_testlog, compare_memory): - compare_memory.return_value = (True, 0.02, 0.05, 1000, 1002) + compare_memory.return_value = ( + True, + "MEMCOMP: Memory usage highwater has changed by 2.00% relative to baseline", + ) with tempfile.TemporaryDirectory() as tempdir: caseroot = Path(tempdir) / "caseroot" @@ -178,7 +183,7 @@ def test_compare_memory(self, append_testlog, compare_memory): @mock.patch("CIME.SystemTests.system_tests_common.compare_memory") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") def test_compare_memory_erorr_diff(self, append_testlog, compare_memory): - compare_memory.return_value = (None, 0.02, 0.05, 1000, 1002) + compare_memory.return_value = (None, "Error diff value") with tempfile.TemporaryDirectory() as tempdir: caseroot = Path(tempdir) / "caseroot" @@ -202,7 +207,10 @@ def test_compare_memory_erorr_diff(self, append_testlog, compare_memory): @mock.patch("CIME.SystemTests.system_tests_common.compare_memory") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") def test_compare_memory_erorr_fail(self, append_testlog, compare_memory): - compare_memory.return_value = (False, 0.02, 0.05, 1000, 1002) + compare_memory.return_value = ( + False, + "Error: Memory usage increase >5% from baseline's 1000.000000 to 1002.000000", + ) with tempfile.TemporaryDirectory() as tempdir: caseroot = Path(tempdir) / "caseroot" @@ -221,10 +229,6 @@ def test_compare_memory_erorr_fail(self, append_testlog, compare_memory): assert common._test_status.get_overall_test_status() == ("PASS", None) - append_testlog.assert_any_call( - "MEMCOMP: Memory usage highwater has changed by 2.00% relative to baseline", - str(caseroot), - ) append_testlog.assert_any_call( "Error: Memory usage increase >5% from baseline's 1000.000000 to 1002.000000", str(caseroot), From 9bab6396e9f1007282c7b168618d380752a493ca Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 22 Sep 2023 22:28:35 -0700 Subject: [PATCH 08/31] Adds --tput-only and --mem-only --- CIME/Tools/bless_test_results | 14 +++- CIME/bless_test_results.py | 127 ++++++++++++++++++++++++++++++++-- 2 files changed, 135 insertions(+), 6 deletions(-) diff --git a/CIME/Tools/bless_test_results b/CIME/Tools/bless_test_results index d630aff69bd..e4236c172f6 100755 --- a/CIME/Tools/bless_test_results +++ b/CIME/Tools/bless_test_results @@ -59,6 +59,12 @@ OR "--hist-only", action="store_true", help="Only analyze history files." ) + parser.add_argument( + "--tput-only", action="store_true", help="Only analyze throughput." + ) + + parser.add_argument("--mem-only", action="store_true", help="Only analyze memory.") + parser.add_argument( "-b", "--baseline-name", @@ -140,7 +146,7 @@ OR "Makes no sense to use -r and -f simultaneously", ) expect( - not (args.namelists_only and args.hist_only), + sum([args.namelists_only, args.hist_only, args.tput_only, args.mem_only]) <= 1, "Makes no sense to use --namelists-only and --hist-only simultaneously", ) @@ -152,6 +158,8 @@ OR args.test_id, args.namelists_only, args.hist_only, + args.tput_only, + args.mem_only, args.report_only, args.force, args.pes_file, @@ -173,6 +181,8 @@ def _main_func(description): test_id, namelists_only, hist_only, + tput_only, + mem_only, report_only, force, pes_file, @@ -190,6 +200,8 @@ def _main_func(description): test_id=test_id, namelists_only=namelists_only, hist_only=hist_only, + tput_only=tput_only, + mem_only=mem_only, report_only=report_only, force=force, pesfile=pes_file, diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index 62637851cb4..a44c2ec5fe5 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -11,10 +11,94 @@ from CIME.hist_utils import generate_baseline, compare_baseline from CIME.case import Case from CIME.test_utils import get_test_status_files +from CIME.baselines import ( + get_latest_cpl_logs, + get_mem_usage, + get_throughput, + compare_throughput, + compare_memory, + write_baseline_tput, + write_baseline_mem, +) import os, time logger = logging.getLogger(__name__) + +def bless_throughput( + case, + test_name, + baseline_root, + baseline_name, + report_only, + force, +): + success = True + reason = None + + baseline_dir = os.path.join( + baseline_root, baseline_name, case.get_value("CASEBASEID") + ) + + below_threshold, comment = compare_throughput(case, baseline_dir=baseline_dir) + + if below_threshold: + logger.info("Diff appears to have been already resolved.") + else: + logger.info(comment) + + if not report_only and ( + force or input("Update this diff (y/n)? ").upper() in ["Y", "YES"] + ): + try: + latest_cpl_logs = get_latest_cpl_logs(case) + current = get_throughput(latest_cpl_logs[0]) + write_baseline_tput(baseline_dir, current) + except Exception as e: + success = False + + reason = f"Failed to write baseline throughput for {test_name!r}: {e!s}" + + return success, reason + + +def bless_memory( + case, + test_name, + baseline_root, + baseline_name, + report_only, + force, +): + success = True + reason = None + + baseline_dir = os.path.join( + baseline_root, baseline_name, case.get_value("CASEBASEID") + ) + + below_threshold, comment = compare_memory(case, baseline_dir=baseline_dir) + + if below_threshold: + logger.info("Diff appears to have been already resolved.") + else: + logger.info(comment) + + if not report_only and ( + force or input("Update this diff (y/n)? ").upper() in ["Y", "YES"] + ): + try: + latest_cpl_logs = get_latest_cpl_logs(case) + current = get_mem_usage(latest_cpl_logs[0]) + write_baseline_mem(baseline_dir, current) + except Exception as e: + success = False + + reason = f"Failed to write baseline memory usage for test {test_name!r}: {e!s}" + + return success, reason + + ############################################################################### def bless_namelists( test_name, @@ -112,6 +196,8 @@ def bless_test_results( test_id=None, namelists_only=False, hist_only=False, + tput_only=False, + mem_only=False, report_only=False, force=False, pesfile=None, @@ -172,9 +258,15 @@ def bless_test_results( if bless_tests in [[], None] or CIME.utils.match_any( test_name, bless_tests_counts ): - overall_result, phase = ts.get_overall_test_status( - ignore_namelists=True, ignore_memleak=True - ) + ts_kwargs = dict(ignore_namelists=True, ignore_memleak=True) + + if tput_only: + ts_kwargs["check_throughput"] = True + + if mem_only: + ts_kwargs["check_memory"] = True + + overall_result, phase = ts.get_overall_test_status(**ts_kwargs) # See if we need to bless namelist if not hist_only: @@ -219,14 +311,13 @@ def bless_test_results( hist_bless = False # Now, do the bless - if not nl_bless and not hist_bless: + if not nl_bless and not hist_bless and not tput_only and not mem_only: logger.info( "Nothing to bless for test: {}, overall status: {}".format( test_name, overall_result ) ) else: - logger.info( "###############################################################################" ) @@ -305,6 +396,32 @@ def bless_test_results( if not success: broken_blesses.append((test_name, reason)) + if tput_only: + success, reason = bless_throughput( + case, + test_name, + baseline_root, + baseline_name, + report_only, + force, + ) + + if not success: + broken_blesses.append((test_name, reason)) + + if mem_only: + success, reason = bless_memory( + case, + test_name, + baseline_root, + baseline_name, + report_only, + force, + ) + + if not success: + broken_blesses.append((test_name, reason)) + # Emit a warning if items in bless_tests did not match anything if bless_tests: for bless_test, bless_count in bless_tests_counts.items(): From 52c8d7b56a887efb0d53febfbf9a8e006e7b25fa Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 29 Sep 2023 12:36:36 -0700 Subject: [PATCH 09/31] Fixes failing test --- CIME/tests/test_unit_system_tests.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/CIME/tests/test_unit_system_tests.py b/CIME/tests/test_unit_system_tests.py index a43b7ef8f80..066d4fbd193 100644 --- a/CIME/tests/test_unit_system_tests.py +++ b/CIME/tests/test_unit_system_tests.py @@ -8,6 +8,7 @@ from unittest import mock from pathlib import Path +from CIME.config import Config from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo from CIME.SystemTests.system_tests_compare_n import SystemTestsCompareN @@ -240,7 +241,7 @@ def test_generate_baseline(self): tempdir, cpllog_data=CPLLOG ) - case.get_value.side_effect = ( + get_value_calls = [ str(caseroot), "ERIO.ne30_g16_rx1.A.docker_gnu", "mct", @@ -253,13 +254,17 @@ def test_generate_baseline(self): str(run_dir), "ERIO", "ERIO.ne30_g16_rx1.A.docker_gnu", - os.getcwd(), "master/ERIO.ne30_g16_rx1.A.docker_gnu", str(baseline_root), "master/ERIO.ne30_g16_rx1.A.docker_gnu", str(run_dir), "mct", - ) + ] + + if Config.instance().create_bless_log: + get_value_calls.insert(12, os.getcwd()) + + case.get_value.side_effect = get_value_calls common = SystemTestsCommon(case) From 6dc623cb3493ef5265d5b99db2f207727eae994f Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Mon, 2 Oct 2023 17:03:09 -0700 Subject: [PATCH 10/31] Refactors and updates bless_test_results --- CIME/Tools/bless_test_results | 202 ++++++++++++++-------------------- CIME/bless_test_results.py | 1 + CIME/utils.py | 11 +- 3 files changed, 90 insertions(+), 124 deletions(-) diff --git a/CIME/Tools/bless_test_results b/CIME/Tools/bless_test_results index e4236c172f6..cb6bd2f972a 100755 --- a/CIME/Tools/bless_test_results +++ b/CIME/Tools/bless_test_results @@ -8,20 +8,21 @@ blessing of diffs. You may need to load modules for cprnc to work. """ - from standard_script_setup import * from CIME.utils import expect from CIME.XML.machines import Machines from CIME.bless_test_results import bless_test_results -import argparse, sys, os +import argparse +import sys +import os +import logging _MACHINE = Machines() -############################################################################### + def parse_command_line(args, description): - ############################################################################### parser = argparse.ArgumentParser( usage="""\n{0} [-n] [-r ] [-b ] [-c ] [ ...] [--verbose] OR @@ -45,41 +46,18 @@ OR formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - default_compiler = _MACHINE.get_default_compiler() - scratch_root = _MACHINE.get_value("CIME_OUTPUT_ROOT") - default_testroot = os.path.join(scratch_root) + create_bless_options(parser) - CIME.utils.setup_standard_logging_options(parser) + create_baseline_options(parser) - parser.add_argument( - "-n", "--namelists-only", action="store_true", help="Only analyze namelists." - ) + create_test_options(parser) - parser.add_argument( - "--hist-only", action="store_true", help="Only analyze history files." - ) - - parser.add_argument( - "--tput-only", action="store_true", help="Only analyze throughput." - ) - - parser.add_argument("--mem-only", action="store_true", help="Only analyze memory.") - - parser.add_argument( - "-b", - "--baseline-name", - help="Name of baselines to use. Default will use BASELINE_NAME_CMP first if possible, otherwise branch name.", - ) - - parser.add_argument( - "--baseline-root", - help="Root of baselines. Default will use the BASELINE_ROOT from the case.", - ) + CIME.utils.setup_standard_logging_options(parser) parser.add_argument( "-c", "--compiler", - default=default_compiler, + default=_MACHINE.get_default_compiler(), help="Compiler of run you want to bless", ) @@ -91,36 +69,15 @@ OR "This option forces the bless to happen regardless.", ) - parser.add_argument( + mutual_execution = parser.add_mutually_exclusive_group() + + mutual_execution.add_argument( "--report-only", action="store_true", help="Only report what files will be overwritten and why. Caution is a good thing when updating baselines", ) - parser.add_argument( - "-r", - "--test-root", - default=default_testroot, - help="Path to test results that are being blessed", - ) - - parser.add_argument( - "--new-test-root", - help="If bless_test_results needs to create cases (for blessing namelists), use this root area", - ) - - parser.add_argument( - "--new-test-id", - help="If bless_test_results needs to create cases (for blessing namelists), use this test id", - ) - - parser.add_argument( - "-t", - "--test-id", - help="Limit processes to case dirs matching this test-id. Can be useful if mutiple runs dumped into the same dir.", - ) - - parser.add_argument( + mutual_execution.add_argument( "-f", "--force", action="store_true", @@ -141,79 +98,82 @@ OR args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) - expect( - not (args.report_only and args.force), - "Makes no sense to use -r and -f simultaneously", + return vars(args) + + +def create_bless_options(parser): + bless_group = parser.add_argument_group("Bless options") + + mutual_bless_group = bless_group.add_mutually_exclusive_group(required=True) + + mutual_bless_group.add_argument( + "-n", "--namelists-only", action="store_true", help="Only analyze namelists." ) - expect( - sum([args.namelists_only, args.hist_only, args.tput_only, args.mem_only]) <= 1, - "Makes no sense to use --namelists-only and --hist-only simultaneously", + + mutual_bless_group.add_argument( + "--hist-only", action="store_true", help="Only analyze history files." + ) + + mutual_bless_group.add_argument( + "--tput-only", action="store_true", help="Only analyze throughput." ) - return ( - args.baseline_name, - args.baseline_root, - args.test_root, - args.compiler, - args.test_id, - args.namelists_only, - args.hist_only, - args.tput_only, - args.mem_only, - args.report_only, - args.force, - args.pes_file, - args.bless_tests, - args.no_skip_pass, - args.new_test_root, - args.new_test_id, + mutual_bless_group.add_argument( + "--mem-only", action="store_true", help="Only analyze memory." ) -############################################################################### -def _main_func(description): - ############################################################################### - ( - baseline_name, - baseline_root, - test_root, - compiler, - test_id, - namelists_only, - hist_only, - tput_only, - mem_only, - report_only, - force, - pes_file, - bless_tests, - no_skip_pass, - new_test_root, - new_test_id, - ) = parse_command_line(sys.argv, description) - - success = bless_test_results( - baseline_name, - baseline_root, - test_root, - compiler, - test_id=test_id, - namelists_only=namelists_only, - hist_only=hist_only, - tput_only=tput_only, - mem_only=mem_only, - report_only=report_only, - force=force, - pesfile=pes_file, - bless_tests=bless_tests, - no_skip_pass=no_skip_pass, - new_test_root=new_test_root, - new_test_id=new_test_id, +def create_baseline_options(parser): + baseline_group = parser.add_argument_group("Baseline options") + + baseline_group.add_argument( + "-b", + "--baseline-name", + help="Name of baselines to use. Default will use BASELINE_NAME_CMP first if possible, otherwise branch name.", ) - sys.exit(0 if success else 1) + baseline_group.add_argument( + "--baseline-root", + help="Root of baselines. Default will use the BASELINE_ROOT from the case.", + ) + + +def create_test_options(parser): + default_testroot = _MACHINE.get_value("CIME_OUTPUT_ROOT") + + test_group = parser.add_argument_group("Test options") + + test_group.add_argument( + "-r", + "--test-root", + default=default_testroot, + help="Path to test results that are being blessed", + ) + + test_group.add_argument( + "--new-test-root", + help="If bless_test_results needs to create cases (for blessing namelists), use this root area", + ) + + test_group.add_argument( + "--new-test-id", + help="If bless_test_results needs to create cases (for blessing namelists), use this test id", + ) + + test_group.add_argument( + "-t", + "--test-id", + help="Limit processes to case dirs matching this test-id. Can be useful if mutiple runs dumped into the same dir.", + ) + + +def _main_func(description): + kwargs = parse_command_line(sys.argv, description) + + success = bless_test_results(**kwargs) + + sys.exit(0 if success else 1) -############################################################################### if __name__ == "__main__": _main_func(__doc__) diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index a44c2ec5fe5..2c7d5e671a7 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -205,6 +205,7 @@ def bless_test_results( no_skip_pass=False, new_test_root=None, new_test_id=None, + **_, # Capture all for extra ): ############################################################################### test_status_files = get_test_status_files(test_root, compiler, test_id=test_id) diff --git a/CIME/utils.py b/CIME/utils.py index 1293f6d3171..1a32319c8d1 100644 --- a/CIME/utils.py +++ b/CIME/utils.py @@ -1625,20 +1625,25 @@ def find_files(rootdir, pattern): def setup_standard_logging_options(parser): + group = parser.add_argument_group("Logging options") + helpfile = os.path.join(os.getcwd(), os.path.basename("{}.log".format(sys.argv[0]))) - parser.add_argument( + + group.add_argument( "-d", "--debug", action="store_true", help="Print debug information (very verbose) to file {}".format(helpfile), ) - parser.add_argument( + + group.add_argument( "-v", "--verbose", action="store_true", help="Add additional context (time and file) to log messages", ) - parser.add_argument( + + group.add_argument( "-s", "--silent", action="store_true", From 94502568282a5ac578313925825ac97c2a17c4ca Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 5 Oct 2023 10:00:58 -0700 Subject: [PATCH 11/31] Refactors base config class --- CIME/config.py | 227 +++++++++++++++++++++++++------------------------ 1 file changed, 118 insertions(+), 109 deletions(-) diff --git a/CIME/config.py b/CIME/config.py index 3cef6cc0530..d2306d354d0 100644 --- a/CIME/config.py +++ b/CIME/config.py @@ -9,19 +9,132 @@ logger = logging.getLogger(__name__) -class Config: +class ConfigBase: def __new__(cls): if not hasattr(cls, "_instance"): - cls._instance = super(Config, cls).__new__(cls) + cls._instance = super(ConfigBase, cls).__new__(cls) return cls._instance def __init__(self): - if getattr(self, "_loaded", False): - return - self._attribute_config = {} + @property + def loaded(self): + return getattr(self, "_loaded", False) + + @classmethod + def instance(cls): + """Access singleton. + + Explicit way to access singleton, same as calling constructor. + """ + return cls() + + @classmethod + def load(cls, customize_path): + obj = cls() + + logger.debug("Searching %r for files to load", customize_path) + + customize_files = glob.glob(f"{customize_path}/**/*.py", recursive=True) + + # filter out any tests + customize_files = [ + x for x in customize_files if "tests" not in x and "conftest" not in x + ] + + customize_module_spec = importlib.machinery.ModuleSpec("cime_customize", None) + + customize_module = importlib.util.module_from_spec(customize_module_spec) + + sys.modules["CIME.customize"] = customize_module + + for x in sorted(customize_files): + obj._load_file(x, customize_module) + + setattr(obj, "_loaded", True) + + return obj + + def _load_file(self, file_path, customize_module): + logger.debug("Loading file %r", file_path) + + raw_config = utils.import_from_file("raw_config", file_path) + + # filter user define variables and functions + user_defined = [x for x in dir(raw_config) if not x.endswith("__")] + + # set values on this object, will overwrite existing + for x in user_defined: + try: + value = getattr(raw_config, x) + except AttributeError: + # should never hit this + logger.fatal("Attribute %r missing on obejct", x) + + sys.exit(1) + else: + setattr(customize_module, x, value) + + self._set_attribute(x, value) + + def _set_attribute(self, name, value, desc=None): + if hasattr(self, name): + logger.debug("Overwriting %r attribute", name) + + logger.debug("Setting attribute %r with value %r", name, value) + + setattr(self, name, value) + + self._attribute_config[name] = { + "desc": desc, + "default": value, + } + + def print_rst_table(self): + max_variable = max([len(x) for x in self._attribute_config.keys()]) + max_default = max( + [len(str(x["default"])) for x in self._attribute_config.values()] + ) + max_type = max( + [len(type(x["default"]).__name__) for x in self._attribute_config.values()] + ) + max_desc = max([len(x["desc"]) for x in self._attribute_config.values()]) + + divider_row = ( + f"{'='*max_variable} {'='*max_default} {'='*max_type} {'='*max_desc}" + ) + + rows = [ + divider_row, + f"Variable{' '*(max_variable-8)} Default{' '*(max_default-7)} Type{' '*(max_type-4)} Description{' '*(max_desc-11)}", + divider_row, + ] + + for variable, value in sorted( + self._attribute_config.items(), key=lambda x: x[0] + ): + variable_fill = max_variable - len(variable) + default_fill = max_default - len(str(value["default"])) + type_fill = max_type - len(type(value["default"]).__name__) + + rows.append( + f"{variable}{' '*variable_fill} {value['default']}{' '*default_fill} {type(value['default']).__name__}{' '*type_fill} {value['desc']}" + ) + + rows.append(divider_row) + + print("\n".join(rows)) + + +class Config(ConfigBase): + def __init__(self): + super().__init__() + + if self.loaded: + return + self._set_attribute( "additional_archive_components", ("drv", "dart"), @@ -195,107 +308,3 @@ def __init__(self): "{srcroot}/libraries/mct", desc="Sets the path to the mct library.", ) - - @classmethod - def instance(cls): - """Access singleton. - - Explicit way to access singleton, same as calling constructor. - """ - return cls() - - @classmethod - def load(cls, customize_path): - obj = cls() - - logger.debug("Searching %r for files to load", customize_path) - - customize_files = glob.glob(f"{customize_path}/**/*.py", recursive=True) - - # filter out any tests - customize_files = [ - x for x in customize_files if "tests" not in x and "conftest" not in x - ] - - customize_module_spec = importlib.machinery.ModuleSpec("cime_customize", None) - - customize_module = importlib.util.module_from_spec(customize_module_spec) - - sys.modules["CIME.customize"] = customize_module - - for x in sorted(customize_files): - obj._load_file(x, customize_module) - - setattr(obj, "_loaded", True) - - return obj - - def _load_file(self, file_path, customize_module): - logger.debug("Loading file %r", file_path) - - raw_config = utils.import_from_file("raw_config", file_path) - - # filter user define variables and functions - user_defined = [x for x in dir(raw_config) if not x.endswith("__")] - - # set values on this object, will overwrite existing - for x in user_defined: - try: - value = getattr(raw_config, x) - except AttributeError: - # should never hit this - logger.fatal("Attribute %r missing on obejct", x) - - sys.exit(1) - else: - setattr(customize_module, x, value) - - self._set_attribute(x, value) - - def _set_attribute(self, name, value, desc=None): - if hasattr(self, name): - logger.debug("Overwriting %r attribute", name) - - logger.debug("Setting attribute %r with value %r", name, value) - - setattr(self, name, value) - - self._attribute_config[name] = { - "desc": desc, - "default": value, - } - - def print_rst_table(self): - max_variable = max([len(x) for x in self._attribute_config.keys()]) - max_default = max( - [len(str(x["default"])) for x in self._attribute_config.values()] - ) - max_type = max( - [len(type(x["default"]).__name__) for x in self._attribute_config.values()] - ) - max_desc = max([len(x["desc"]) for x in self._attribute_config.values()]) - - divider_row = ( - f"{'='*max_variable} {'='*max_default} {'='*max_type} {'='*max_desc}" - ) - - rows = [ - divider_row, - f"Variable{' '*(max_variable-8)} Default{' '*(max_default-7)} Type{' '*(max_type-4)} Description{' '*(max_desc-11)}", - divider_row, - ] - - for variable, value in sorted( - self._attribute_config.items(), key=lambda x: x[0] - ): - variable_fill = max_variable - len(variable) - default_fill = max_default - len(str(value["default"])) - type_fill = max_type - len(type(value["default"]).__name__) - - rows.append( - f"{variable}{' '*variable_fill} {value['default']}{' '*default_fill} {type(value['default']).__name__}{' '*type_fill} {value['desc']}" - ) - - rows.append(divider_row) - - print("\n".join(rows)) From d948986b4739af86485adb44867f29a95a583628 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 6 Oct 2023 00:25:25 -0700 Subject: [PATCH 12/31] Adds feature for coupler to define custom read/compare functions --- CIME/SystemTests/system_tests_common.py | 19 +- CIME/baselines.py | 359 +++++++++++++++--------- CIME/bless_test_results.py | 14 +- CIME/case/case.py | 7 + CIME/tests/test_unit_baselines.py | 205 +++++++++----- CIME/tests/test_unit_system_tests.py | 13 +- 6 files changed, 398 insertions(+), 219 deletions(-) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index e1c5d59388c..38263cd7cea 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -28,12 +28,10 @@ from CIME.locked_files import LOCKED_DIR, lock_file, is_locked from CIME.baselines import ( get_latest_cpl_logs, - get_throughput, - get_mem_usage, + default_get_mem_usage, compare_memory, compare_throughput, - write_baseline_mem, - write_baseline_tput, + write_baseline, ) import CIME.build as build @@ -636,7 +634,7 @@ def _check_for_memleak(self): with self._test_status: latestcpllogs = get_latest_cpl_logs(self._case) for cpllog in latestcpllogs: - memlist = get_mem_usage(cpllog) + memlist = default_get_mem_usage(self._case, cpllog) if len(memlist) < 3: self._test_status.set_status( @@ -782,23 +780,20 @@ def _generate_baseline(self): # drop the date so that the name is generic newestcpllogfiles = get_latest_cpl_logs(self._case) with SharedArea(): + # TODO ever actually more than one cpl log? for cpllog in newestcpllogfiles: m = re.search(r"/({}.*.log).*.gz".format(self._cpllog), cpllog) + if m is not None: baselog = os.path.join(basegen_dir, m.group(1)) + ".gz" + safe_copy( cpllog, os.path.join(basegen_dir, baselog), preserve_meta=False, ) - tput = get_throughput(cpllog) - - write_baseline_tput(basegen_dir, tput) - - mem = get_mem_usage(cpllog) - - write_baseline_mem(basegen_dir, mem) + write_baseline(self._case, basegen_dir, cpllog) class FakeTest(SystemTestsCommon): diff --git a/CIME/baselines.py b/CIME/baselines.py index cbdb3a2cf61..0f7d1c16b19 100644 --- a/CIME/baselines.py +++ b/CIME/baselines.py @@ -3,6 +3,7 @@ import re import gzip import logging +from CIME.config import Config from CIME.utils import expect logger = logging.getLogger(__name__) @@ -13,13 +14,20 @@ def get_latest_cpl_logs(case): find and return the latest cpl log file in the run directory """ coupler_log_path = case.get_value("RUNDIR") + cpllog_name = "drv" if case.get_value("COMP_INTERFACE") == "nuopc" else "cpl" + cpllogs = glob.glob(os.path.join(coupler_log_path, "{}*.log.*".format(cpllog_name))) + lastcpllogs = [] + if cpllogs: lastcpllogs.append(max(cpllogs, key=os.path.getctime)) + basename = os.path.basename(lastcpllogs[0]) + suffix = basename.split(".", 1)[1] + for log in cpllogs: if log in lastcpllogs: continue @@ -31,54 +39,75 @@ def get_latest_cpl_logs(case): def compare_memory(case, baseline_dir=None): - if baseline_dir is None: - baseline_root = case.get_value("BASELINE_ROOT") - - baseline_name = case.get_value("BASECMP_CASE") + """ + Compare current memory usage to baseline. - baseline_dir = os.path.join(baseline_root, baseline_name) + Parameters + ---------- + case : CIME.case.case.Case + Case object. + baseline_dir : str or None + Path to the baseline directory. - latest_cpl_logs = get_latest_cpl_logs(case) + Returns + ------- + below_tolerance : bool + Whether the current memory usage is below the baseline. + comments : str + Comments about baseline comparison. + """ + if baseline_dir is None: + baseline_dir = case.get_baseline_dir() - diff, baseline, current = [None] * 3 - comment = "" + config = load_coupler_customization(case) - for cpllog in latest_cpl_logs: + # TODO need better handling + try: try: - baseline = read_baseline_mem(baseline_dir) - except FileNotFoundError as e: - comment = f"Could not read baseline memory usage: {e!s}" + current = config.get_mem_usage(case) + except AttributeError: + current = default_get_mem_usage(case) + except RuntimeError as e: + return None, str(e) - logger.debug(comment) + try: + baseline = read_baseline_mem(baseline_dir) + except FileNotFoundError as e: + comment = f"Could not read baseline memory usage: {e!s}" - continue + logger.debug(comment) - if baseline is None: - baseline = 0.0 + return None, comment - memlist = get_mem_usage(cpllog) + if baseline is None: + baseline = 0.0 - if len(memlist) <= 3: - comment = f"Found {len(memlist)} memory usage samples, need atleast 4" + tolerance = case.get_value("TEST_MEMLEAK_TOLERANCE") - logger.debug(comment) + if tolerance is None: + tolerance = 0.1 - continue + try: + below_tolerance, comments = config.compare_memory_baseline( + current, baseline, tolerance + ) + except AttributeError: + below_tolerance, comments = compare_memory_baseline( + current, baseline, tolerance + ) - current = memlist[-1][1] + return below_tolerance, comments - try: - diff = (current - baseline) / baseline - except ZeroDivisionError: - diff = 0.0 - tolerance = case.get_value("TEST_MEMLEAK_TOLERANCE") - - if tolerance is None: - tolerance = 0.1 +def compare_memory_baseline(current, baseline, tolerance): + try: + diff = (current - baseline) / baseline + except ZeroDivisionError: + diff = 0.0 # Should we check if tolerance is above 0 below_tolerance = None + comment = "" if diff is not None: below_tolerance = diff < tolerance @@ -97,40 +126,23 @@ def compare_memory(case, baseline_dir=None): def compare_throughput(case, baseline_dir=None): if baseline_dir is None: - baseline_root = case.get_value("BASELINE_ROOT") - - baseline_name = case.get_value("BASECMP_CASE") - - baseline_dir = os.path.join(baseline_root, baseline_name) + baseline_dir = case.get_baseline_dir() - latest_cpl_logs = get_latest_cpl_logs(case) + config = load_coupler_customization(case) - diff, baseline, current = [None] * 3 - comment = "" - - # Do we need this loop, when are there multiple cpl logs? - for cpllog in latest_cpl_logs: - try: - baseline = read_baseline_tput(baseline_dir) - except FileNotFoundError as e: - comment = f"Could not read baseline throughput file: {e!s}" - - logger.debug(comment) - - continue - - current = get_throughput(cpllog) + try: + current = config.get_throughput(case) + except AttributeError: + current = default_get_throughput(case) - try: - # comparing ypd so bigger is better - diff = (baseline - current) / baseline - except (ValueError, TypeError): - # Should we default the diff to 0.0 as with _compare_current_memory? - comment = f"Could not determine diff with baseline {baseline!r} and current {current!r}" + try: + baseline = read_baseline_tput(baseline_dir) + except FileNotFoundError as e: + comment = f"Could not read baseline throughput file: {e!s}" - logger.debug(comment) + logger.debug(comment) - continue + return None, comment tolerance = case.get_value("TEST_TPUT_TOLERANCE") @@ -142,6 +154,38 @@ def compare_throughput(case, baseline_dir=None): "Bad value for throughput tolerance in test", ) + try: + below_tolerance, comment = config.compare_baseline_throughput( + current, baseline, tolerance + ) + except AttributeError: + below_tolerance, comment = compare_baseline_throughput( + current, baseline, tolerance + ) + + return below_tolerance, comment + + +def load_coupler_customization(case): + comp_root_dir_cpl = case.get_value("COMP_ROOT_DIR_CPL") + + cpl_customize = os.path.join(comp_root_dir_cpl, "cime_config", "customize") + + return Config.load(cpl_customize) + + +def compare_baseline_throughput(current, baseline, tolerance): + try: + # comparing ypd so bigger is better + diff = (baseline - current) / baseline + except (ValueError, TypeError): + # Should we default the diff to 0.0 as with _compare_current_memory? + comment = f"Could not determine diff with baseline {baseline!r} and current {current!r}" + + logger.debug(comment) + + diff = None + below_tolerance = None if diff is not None: @@ -159,59 +203,124 @@ def compare_throughput(case, baseline_dir=None): return below_tolerance, comment +def write_baseline(case, basegen_dir, throughput=True, memory=True): + config = load_coupler_customization(case) + + if throughput: + try: + tput = config.get_throughput(case) + except AttributeError: + tput = str(default_get_throughput(case)) + + write_baseline_tput(basegen_dir, tput) + + if memory: + try: + mem = config.get_mem_usage(case) + except AttributeError: + mem = str(default_get_mem_usage(case)) + + write_baseline_mem(basegen_dir, mem) + + +def default_get_throughput(case): + """ + Parameters + ---------- + cpllog : str + Path to the coupler log. + + Returns + ------- + str + Last recorded highwater memory usage. + """ + cpllog = get_latest_cpl_logs(case) + + try: + tput = get_cpl_throughput(cpllog[0]) + except (FileNotFoundError, IndexError): + tput = None + + return tput + + +def default_get_mem_usage(case, cpllog=None): + """ + Parameters + ---------- + cpllog : str + Path to the coupler log. + + Returns + ------- + str + Last recorded highwater memory usage. + + Raises + ------ + RuntimeError + If not enough sample were found. + """ + if cpllog is None: + cpllog = get_latest_cpl_logs(case) + + try: + memlist = get_cpl_mem_usage(cpllog[0]) + except (FileNotFoundError, IndexError): + memlist = [(None, None)] + else: + if len(memlist) <= 3: + raise RuntimeError( + f"Found {len(memlist)} memory usage samples, need atleast 4" + ) + + return memlist[-1][1] + + def write_baseline_tput(baseline_dir, tput): """ - Writes baseline throughput to file. + Writes throughput to baseline file. - A "-1" indicates that no throughput data was available from the coupler log. + The format is arbitrary, it's the callers responsibilty + to decode the data. Parameters ---------- baseline_dir : str Path to the baseline directory. - cpllog : str - Path to the current coupler log. + tput : str + Model throughput. """ tput_file = os.path.join(baseline_dir, "cpl-tput.log") with open(tput_file, "w") as fd: - fd.write("# Throughput in simulated years per compute day\n") - fd.write("# A -1 indicates no throughput data was available from the test\n") - - if tput is None: - fd.write("-1") - else: - fd.write(str(tput)) + fd.write(tput) def write_baseline_mem(baseline_dir, mem): """ - Writes baseline memory usage highwater to file. + Writes memory usage to baseline file. - A "-1" indicates that no memory usage data was available from the coupler log. + The format is arbitrary, it's the callers responsibilty + to decode the data. Parameters ---------- baseline_dir : str Path to the baseline directory. - cpllog : str - Path to the current coupler log. + mem : str + Model memory usage. """ mem_file = os.path.join(baseline_dir, "cpl-mem.log") with open(mem_file, "w") as fd: - fd.write("# Memory usage highwater\n") - fd.write("# A -1 indicates no memory usage data was available from the test\n") - - try: - fd.write(str(mem[-1][1])) - except IndexError: - fd.write("-1") + fd.write(mem) def read_baseline_tput(baseline_dir): """ - Reads throughput baseline. + Reads the raw lines of the throughput baseline file. Parameters ---------- @@ -220,34 +329,20 @@ def read_baseline_tput(baseline_dir): Returns ------- - float - Value of the throughput. + list + Contents of the throughput baseline file. Raises ------ FileNotFoundError If baseline file does not exist. - IndexError - If no throughput value is found. - ValueError - If throughput value is not a float. """ - try: - tput = read_baseline_value(os.path.join(baseline_dir, "cpl-tput.log")) - except (IndexError, ValueError): - tput = None - - if tput == -1: - tput = None - - return tput + return read_baseline_value(os.path.join(baseline_dir, "cpl-tput.log")) def read_baseline_mem(baseline_dir): """ - Reads memory usage highwater baseline. - - The default behvaior was to return 0 if no usage data was found. + Read the raw lines of the memory baseline file. Parameters ---------- @@ -256,28 +351,20 @@ def read_baseline_mem(baseline_dir): Returns ------- - float - Value of the highwater memory usage. + list + Contents of the memory baseline file. Raises ------ FileNotFoundError If baseline file does not exist. """ - try: - memory = read_baseline_value(os.path.join(baseline_dir, "cpl-mem.log")) - except (IndexError, ValueError): - memory = 0 - - if memory == -1: - memory = 0 - - return memory + return read_baseline_value(os.path.join(baseline_dir, "cpl-mem.log")) def read_baseline_value(baseline_file): """ - Read baseline value from file. + Generic read function, ignores lines prepended by `#`. Parameters ---------- @@ -286,56 +373,78 @@ def read_baseline_value(baseline_file): Returns ------- - float - Baseline value. + list + Lines contained in the baseline file without comments. Raises ------ FileNotFoundError If ``baseline_file`` is not found. - IndexError - If not values are present in ``baseline_file``. - ValueError - If value in ``baseline_file`` is not a float. """ with open(baseline_file) as fd: lines = [x for x in fd.readlines() if not x.startswith("#")] - return float(lines[0]) + return lines -def get_mem_usage(cpllog): +def get_cpl_mem_usage(cpllog): """ - Examine memory usage as recorded in the cpl log file and look for unexpected - increases. + Read memory usage from coupler log. + + Parameters + ---------- + cpllog : str + Path to the coupler log. + + Returns + ------- + list + Memory usage (data, highwater) as recorded by the coupler or empty list. """ memlist = [] + meminfo = re.compile(r".*model date =\s+(\w+).*memory =\s+(\d+\.?\d+).*highwater") + if cpllog is not None and os.path.isfile(cpllog): if ".gz" == cpllog[-3:]: fopen = gzip.open else: fopen = open + with fopen(cpllog, "rb") as f: for line in f: m = meminfo.match(line.decode("utf-8")) + if m: memlist.append((float(m.group(1)), float(m.group(2)))) + # Remove the last mem record, it's sometimes artificially high if len(memlist) > 0: memlist.pop() + return memlist -def get_throughput(cpllog): +def get_cpl_throughput(cpllog): """ - Examine memory usage as recorded in the cpl log file and look for unexpected - increases. + Reads throuhgput from coupler log. + + Parameters + ---------- + cpllog : str + Path to the coupler log. + + Returns + ------- + int or None + Throughput as recorded by the coupler or None """ if cpllog is not None and os.path.isfile(cpllog): with gzip.open(cpllog, "rb") as f: cpltext = f.read().decode("utf-8") + m = re.search(r"# simulated years / cmp-day =\s+(\d+\.\d+)\s", cpltext) + if m: return float(m.group(1)) return None diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index 2c7d5e671a7..d2a88d4b2c8 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -12,13 +12,9 @@ from CIME.case import Case from CIME.test_utils import get_test_status_files from CIME.baselines import ( - get_latest_cpl_logs, - get_mem_usage, - get_throughput, compare_throughput, compare_memory, - write_baseline_tput, - write_baseline_mem, + write_baseline, ) import os, time @@ -51,9 +47,7 @@ def bless_throughput( force or input("Update this diff (y/n)? ").upper() in ["Y", "YES"] ): try: - latest_cpl_logs = get_latest_cpl_logs(case) - current = get_throughput(latest_cpl_logs[0]) - write_baseline_tput(baseline_dir, current) + write_baseline(case, baseline_dir, memory=False) except Exception as e: success = False @@ -88,9 +82,7 @@ def bless_memory( force or input("Update this diff (y/n)? ").upper() in ["Y", "YES"] ): try: - latest_cpl_logs = get_latest_cpl_logs(case) - current = get_mem_usage(latest_cpl_logs[0]) - write_baseline_mem(baseline_dir, current) + write_baseline(case, baseline_dir, throughput=False) except Exception as e: success = False diff --git a/CIME/case/case.py b/CIME/case/case.py index 6de8bb2a217..14bb0069024 100644 --- a/CIME/case/case.py +++ b/CIME/case/case.py @@ -207,6 +207,13 @@ def __init__(self, case_root=None, read_only=True, record=False, non_local=False self.initialize_derived_attributes() + def get_baseline_dir(self): + baseline_root = self.get_value("BASELINE_ROOT") + + baseline_name = self.get_value("BASECMP_self") + + return os.path.join(baseline_root, baseline_name) + def check_if_comp_var(self, vid): for env_file in self._env_entryid_files: new_vid, new_comp, iscompvar = env_file.check_if_comp_var(vid) diff --git a/CIME/tests/test_unit_baselines.py b/CIME/tests/test_unit_baselines.py index 1d4f11d1c1e..644c6b28dcb 100644 --- a/CIME/tests/test_unit_baselines.py +++ b/CIME/tests/test_unit_baselines.py @@ -12,42 +12,45 @@ def create_mock_case(tempdir, get_latest_cpl_logs): caseroot = Path(tempdir, "0", "caseroot") + rundir = caseroot / "run" get_latest_cpl_logs.return_value = (str(rundir / "cpl.log.gz"),) baseline_root = Path(tempdir, "baselines") + baseline_root.mkdir(parents=True, exist_ok=False) + case = mock.MagicMock() return case, caseroot, rundir, baseline_root class TestUnitBaseline(unittest.TestCase): - def test_get_throughput_no_file(self): - throughput = baselines.get_throughput("/tmp/cpl.log") + def test_get_cpl_throughput_no_file(self): + throughput = baselines.get_cpl_throughput("/tmp/cpl.log") assert throughput is None - def test_get_throughput(self): + def test_get_cpl_throughput(self): with tempfile.TemporaryDirectory() as tempdir: cpl_log_path = Path(tempdir, "cpl.log.gz") with gzip.open(cpl_log_path, "w") as fd: fd.write(CPLLOG.encode("utf-8")) - throughput = baselines.get_throughput(str(cpl_log_path)) + throughput = baselines.get_cpl_throughput(str(cpl_log_path)) assert throughput == 719.635 - def test_get_mem_usage_gz(self): + def test_get_cpl_mem_usage_gz(self): with tempfile.TemporaryDirectory() as tempdir: cpl_log_path = Path(tempdir, "cpl.log.gz") with gzip.open(cpl_log_path, "w") as fd: fd.write(CPLLOG.encode("utf-8")) - mem_usage = baselines.get_mem_usage(str(cpl_log_path)) + mem_usage = baselines.get_cpl_mem_usage(str(cpl_log_path)) assert mem_usage == [ (10102.0, 1673.89), @@ -57,13 +60,13 @@ def test_get_mem_usage_gz(self): ] @mock.patch("CIME.baselines.os.path.isfile") - def test_get_mem_usage(self, isfile): + def test_get_cpl_mem_usage(self, isfile): isfile.return_value = True with mock.patch( "builtins.open", mock.mock_open(read_data=CPLLOG.encode("utf-8")) ) as mock_file: - mem_usage = baselines.get_mem_usage("/tmp/cpl.log") + mem_usage = baselines.get_cpl_mem_usage("/tmp/cpl.log") assert mem_usage == [ (10102.0, 1673.89), @@ -76,75 +79,116 @@ def test_read_baseline_mem_empty(self): with mock.patch("builtins.open", mock.mock_open(read_data="")) as mock_file: baseline = baselines.read_baseline_mem("/tmp/cpl-mem.log") - assert baseline is 0 + assert baseline == [] def test_read_baseline_mem_none(self): with mock.patch("builtins.open", mock.mock_open(read_data="-1")) as mock_file: baseline = baselines.read_baseline_mem("/tmp/cpl-mem.log") - assert baseline is 0 + assert baseline == ["-1"] def test_read_baseline_mem(self): with mock.patch("builtins.open", mock.mock_open(read_data="200")) as mock_file: baseline = baselines.read_baseline_mem("/tmp/cpl-mem.log") - assert baseline == 200 + assert baseline == ["200"] def test_read_baseline_tput_empty(self): with mock.patch("builtins.open", mock.mock_open(read_data="")) as mock_file: baseline = baselines.read_baseline_tput("/tmp/cpl-tput.log") - assert baseline is None + assert baseline == [] def test_read_baseline_tput_none(self): with mock.patch("builtins.open", mock.mock_open(read_data="-1")) as mock_file: baseline = baselines.read_baseline_tput("/tmp/cpl-tput.log") - assert baseline is None + assert baseline == ["-1"] def test_read_baseline_tput(self): with mock.patch("builtins.open", mock.mock_open(read_data="200")) as mock_file: baseline = baselines.read_baseline_tput("/tmp/cpl-tput.log") - assert baseline == 200 + assert baseline == ["200"] def test_write_baseline_mem_no_value(self): with mock.patch("builtins.open", mock.mock_open()) as mock_file: - baselines.write_baseline_mem("/tmp", []) + baselines.write_baseline_mem("/tmp", "-1") mock_file.assert_called_with("/tmp/cpl-mem.log", "w") mock_file.return_value.write.assert_called_with("-1") def test_write_baseline_mem(self): with mock.patch("builtins.open", mock.mock_open()) as mock_file: - baselines.write_baseline_mem("/tmp", [(1, 200)]) + baselines.write_baseline_mem("/tmp", "200") mock_file.assert_called_with("/tmp/cpl-mem.log", "w") mock_file.return_value.write.assert_called_with("200") def test_write_baseline_tput_no_value(self): with mock.patch("builtins.open", mock.mock_open()) as mock_file: - baselines.write_baseline_tput("/tmp", None) + baselines.write_baseline_tput("/tmp", "-1") mock_file.assert_called_with("/tmp/cpl-tput.log", "w") mock_file.return_value.write.assert_called_with("-1") def test_write_baseline_tput(self): with mock.patch("builtins.open", mock.mock_open()) as mock_file: - baselines.write_baseline_tput("/tmp", 200) + baselines.write_baseline_tput("/tmp", "200") mock_file.assert_called_with("/tmp/cpl-tput.log", "w") mock_file.return_value.write.assert_called_with("200") - @mock.patch("CIME.baselines.get_throughput") + @mock.patch("CIME.baselines.get_cpl_throughput") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_default_get_throughput(self, get_latest_cpl_logs, get_cpl_throughput): + get_cpl_throughput.side_effect = FileNotFoundError() + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + tput = baselines.default_get_throughput(case) + + assert tput == None + + @mock.patch("CIME.baselines.get_cpl_mem_usage") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_default_get_mem_usage(self, get_latest_cpl_logs, get_cpl_mem_usage): + get_cpl_mem_usage.side_effect = FileNotFoundError() + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + mem = baselines.default_get_mem_usage(case) + + assert mem == None + + @mock.patch("CIME.baselines.get_cpl_mem_usage") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_write_baseline(self, get_latest_cpl_logs, get_cpl_mem_usage): + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + get_cpl_mem_usage.return_value = [ + (1, 1000.0), + (2, 1001.0), + (3, 1002.0), + (4, 1003.0), + ] + + baselines.write_baseline( + case, baseline_root, get_latest_cpl_logs.return_value + ) + + @mock.patch("CIME.baselines.get_cpl_throughput") @mock.patch("CIME.baselines.read_baseline_tput") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_throughput_no_baseline_file( - self, get_latest_cpl_logs, read_baseline_tput, get_throughput + self, get_latest_cpl_logs, read_baseline_tput, get_cpl_throughput ): read_baseline_tput.side_effect = FileNotFoundError - get_throughput.return_value = 504 + get_cpl_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -152,6 +196,7 @@ def test_compare_throughput_no_baseline_file( case.get_value.side_effect = ( str(baseline_root), "master/ERIO.ne30_g16_rx1.A.docker_gnu", + "/tmp/components/cpl", 0.05, ) @@ -160,22 +205,25 @@ def test_compare_throughput_no_baseline_file( assert below_tolerance is None assert comment == "Could not read baseline throughput file: " - @mock.patch("CIME.baselines.get_throughput") + @mock.patch("CIME.baselines.default_get_throughput") @mock.patch("CIME.baselines.read_baseline_tput") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_throughput_no_baseline( - self, get_latest_cpl_logs, read_baseline_tput, get_throughput + self, get_latest_cpl_logs, read_baseline_tput, default_get_throughput ): read_baseline_tput.return_value = None - get_throughput.return_value = 504 + default_get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + case.get_baseline_dir.return_value = str( + baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" + ) + case.get_value.side_effect = ( - str(baseline_root), - "master/ERIO.ne30_g16_rx1.A.docker_gnu", + "/tmp/components/cpl", 0.05, ) @@ -184,22 +232,25 @@ def test_compare_throughput_no_baseline( assert below_tolerance is None assert comment == "Could not determine diff with baseline None and current 504" - @mock.patch("CIME.baselines.get_throughput") + @mock.patch("CIME.baselines.default_get_throughput") @mock.patch("CIME.baselines.read_baseline_tput") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_throughput_no_tolerance( - self, get_latest_cpl_logs, read_baseline_tput, get_throughput + self, get_latest_cpl_logs, read_baseline_tput, default_get_throughput ): read_baseline_tput.return_value = 500 - get_throughput.return_value = 504 + default_get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + case.get_baseline_dir.return_value = str( + baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" + ) + case.get_value.side_effect = ( - str(baseline_root), - "master/ERIO.ne30_g16_rx1.A.docker_gnu", + "/tmp/components/cpl", None, ) @@ -211,22 +262,25 @@ def test_compare_throughput_no_tolerance( == "TPUTCOMP: Computation time changed by -0.80% relative to baseline" ) - @mock.patch("CIME.baselines.get_throughput") + @mock.patch("CIME.baselines.default_get_throughput") @mock.patch("CIME.baselines.read_baseline_tput") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_throughput_above_threshold( - self, get_latest_cpl_logs, read_baseline_tput, get_throughput + self, get_latest_cpl_logs, read_baseline_tput, default_get_throughput ): read_baseline_tput.return_value = 1000 - get_throughput.return_value = 504 + default_get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + case.get_baseline_dir.return_value = str( + baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" + ) + case.get_value.side_effect = ( - str(baseline_root), - "master/ERIO.ne30_g16_rx1.A.docker_gnu", + "/tmp/components/cpl", 0.05, ) @@ -237,22 +291,25 @@ def test_compare_throughput_above_threshold( comment == "Error: TPUTCOMP: Computation time increase > 5% from baseline" ) - @mock.patch("CIME.baselines.get_throughput") + @mock.patch("CIME.baselines.default_get_throughput") @mock.patch("CIME.baselines.read_baseline_tput") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_throughput( - self, get_latest_cpl_logs, read_baseline_tput, get_throughput + self, get_latest_cpl_logs, read_baseline_tput, default_get_throughput ): read_baseline_tput.return_value = 500 - get_throughput.return_value = 504 + default_get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + case.get_baseline_dir.return_value = str( + baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" + ) + case.get_value.side_effect = ( - str(baseline_root), - "master/ERIO.ne30_g16_rx1.A.docker_gnu", + "/tmp/components/cpl", 0.05, ) @@ -264,15 +321,15 @@ def test_compare_throughput( == "TPUTCOMP: Computation time changed by -0.80% relative to baseline" ) - @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.read_baseline_mem") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_memory_no_baseline( - self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage + self, get_latest_cpl_logs, read_baseline_mem, get_cpl_mem_usage ): read_baseline_mem.return_value = None - get_mem_usage.return_value = [ + get_cpl_mem_usage.return_value = [ (1, 1000.0), (2, 1001.0), (3, 1002.0), @@ -282,9 +339,12 @@ def test_compare_memory_no_baseline( with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + case.get_baseline_dir.return_value = str( + baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" + ) + case.get_value.side_effect = ( - str(baseline_root), - "master/ERIO.ne30_g16_rx1.A.docker_gnu", + "/tmp/components/cpl", 0.05, ) @@ -296,15 +356,15 @@ def test_compare_memory_no_baseline( == "MEMCOMP: Memory usage highwater has changed by 0.00% relative to baseline" ) - @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.read_baseline_mem") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_memory_not_enough_samples( - self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage + self, get_latest_cpl_logs, read_baseline_mem, get_cpl_mem_usage ): read_baseline_mem.return_value = 1000.0 - get_mem_usage.return_value = [ + get_cpl_mem_usage.return_value = [ (1, 1000.0), (2, 1001.0), ] @@ -315,6 +375,7 @@ def test_compare_memory_not_enough_samples( case.get_value.side_effect = ( str(baseline_root), "master/ERIO.ne30_g16_rx1.A.docker_gnu", + "/tmp/components/cpl", 0.05, ) @@ -323,15 +384,15 @@ def test_compare_memory_not_enough_samples( assert below_tolerance is None assert comment == "Found 2 memory usage samples, need atleast 4" - @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.read_baseline_mem") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_memory_no_baseline_file( - self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage + self, get_latest_cpl_logs, read_baseline_mem, get_cpl_mem_usage ): read_baseline_mem.side_effect = FileNotFoundError - get_mem_usage.return_value = [ + get_cpl_mem_usage.return_value = [ (1, 1000.0), (2, 1001.0), (3, 1002.0), @@ -344,6 +405,7 @@ def test_compare_memory_no_baseline_file( case.get_value.side_effect = ( str(baseline_root), "master/ERIO.ne30_g16_rx1.A.docker_gnu", + "/tmp/components/cpl", 0.05, ) @@ -352,15 +414,15 @@ def test_compare_memory_no_baseline_file( assert below_tolerance is None assert comment == "Could not read baseline memory usage: " - @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.read_baseline_mem") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_memory_no_tolerance( - self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage + self, get_latest_cpl_logs, read_baseline_mem, get_cpl_mem_usage ): read_baseline_mem.return_value = 1000.0 - get_mem_usage.return_value = [ + get_cpl_mem_usage.return_value = [ (1, 1000.0), (2, 1001.0), (3, 1002.0), @@ -370,9 +432,12 @@ def test_compare_memory_no_tolerance( with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + case.get_baseline_dir.return_value = str( + baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" + ) + case.get_value.side_effect = ( - str(baseline_root), - "master/ERIO.ne30_g16_rx1.A.docker_gnu", + "/tmp/components/cpl", None, ) @@ -384,15 +449,15 @@ def test_compare_memory_no_tolerance( == "MEMCOMP: Memory usage highwater has changed by 0.30% relative to baseline" ) - @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.read_baseline_mem") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_memory_above_threshold( - self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage + self, get_latest_cpl_logs, read_baseline_mem, get_cpl_mem_usage ): read_baseline_mem.return_value = 1000.0 - get_mem_usage.return_value = [ + get_cpl_mem_usage.return_value = [ (1, 2000.0), (2, 2001.0), (3, 2002.0), @@ -402,9 +467,12 @@ def test_compare_memory_above_threshold( with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + case.get_baseline_dir.return_value = str( + baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" + ) + case.get_value.side_effect = ( - str(baseline_root), - "master/ERIO.ne30_g16_rx1.A.docker_gnu", + "/tmp/components/cpl", 0.05, ) @@ -416,15 +484,15 @@ def test_compare_memory_above_threshold( == "Error: Memory usage increase >5% from baseline's 1000.000000 to 2003.000000" ) - @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.read_baseline_mem") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_memory( - self, get_latest_cpl_logs, read_baseline_mem, get_mem_usage + self, get_latest_cpl_logs, read_baseline_mem, get_cpl_mem_usage ): read_baseline_mem.return_value = 1000.0 - get_mem_usage.return_value = [ + get_cpl_mem_usage.return_value = [ (1, 1000.0), (2, 1001.0), (3, 1002.0), @@ -434,9 +502,12 @@ def test_compare_memory( with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + case.get_baseline_dir.return_value = str( + baseline_root / "master" / "ERIO.ne30_g16_rx1.A.docker_gnu" + ) + case.get_value.side_effect = ( - str(baseline_root), - "master/ERIO.ne30_g16_rx1.A.docker_gnu", + "/tmp/components/cpl", 0.05, ) diff --git a/CIME/tests/test_unit_system_tests.py b/CIME/tests/test_unit_system_tests.py index 066d4fbd193..9ea088411bd 100644 --- a/CIME/tests/test_unit_system_tests.py +++ b/CIME/tests/test_unit_system_tests.py @@ -259,6 +259,11 @@ def test_generate_baseline(self): "master/ERIO.ne30_g16_rx1.A.docker_gnu", str(run_dir), "mct", + "/tmp/components/cpl", + str(run_dir), + "mct", + str(run_dir), + "mct", ] if Config.instance().create_bless_log: @@ -280,14 +285,14 @@ def test_generate_baseline(self): with open(baseline_dir / "cpl-tput.log") as fd: lines = fd.readlines() - assert len(lines) == 3 - assert lines[-1] == "719.635" + assert len(lines) == 1 + assert lines[0] == "719.635" with open(baseline_dir / "cpl-mem.log") as fd: lines = fd.readlines() - assert len(lines) == 3 - assert lines[-1] == "1673.89" + assert len(lines) == 1 + assert lines[0] == "1673.89" def test_kwargs(self): case = mock.MagicMock() From ec661ca64eaa95f6ec90a2c018f68b9b9eb1a07c Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 6 Oct 2023 14:47:06 -0700 Subject: [PATCH 13/31] Fixes failing tests --- CIME/SystemTests/system_tests_common.py | 66 +++++++++++++------------ CIME/baselines.py | 32 +++++++----- CIME/tests/test_unit_baselines.py | 2 +- 3 files changed, 55 insertions(+), 45 deletions(-) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index 38263cd7cea..45e7ae932d3 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -634,44 +634,46 @@ def _check_for_memleak(self): with self._test_status: latestcpllogs = get_latest_cpl_logs(self._case) for cpllog in latestcpllogs: - memlist = default_get_mem_usage(self._case, cpllog) - - if len(memlist) < 3: + try: + memlist = default_get_mem_usage(self._case, cpllog) + except RuntimeError: self._test_status.set_status( MEMLEAK_PHASE, TEST_PASS_STATUS, comments="insuffiencient data for memleak test", ) + + continue + + finaldate = int(memlist[-1][0]) + originaldate = int( + memlist[1][0] + ) # skip first day mem record, it can be too low while initializing + finalmem = float(memlist[-1][1]) + originalmem = float(memlist[1][1]) + memdiff = -1 + if originalmem > 0: + memdiff = (finalmem - originalmem) / originalmem + tolerance = self._case.get_value("TEST_MEMLEAK_TOLERANCE") + if tolerance is None: + tolerance = 0.1 + expect(tolerance > 0.0, "Bad value for memleak tolerance in test") + if memdiff < 0: + self._test_status.set_status( + MEMLEAK_PHASE, + TEST_PASS_STATUS, + comments="data for memleak test is insuffiencient", + ) + elif memdiff < tolerance: + self._test_status.set_status(MEMLEAK_PHASE, TEST_PASS_STATUS) else: - finaldate = int(memlist[-1][0]) - originaldate = int( - memlist[1][0] - ) # skip first day mem record, it can be too low while initializing - finalmem = float(memlist[-1][1]) - originalmem = float(memlist[1][1]) - memdiff = -1 - if originalmem > 0: - memdiff = (finalmem - originalmem) / originalmem - tolerance = self._case.get_value("TEST_MEMLEAK_TOLERANCE") - if tolerance is None: - tolerance = 0.1 - expect(tolerance > 0.0, "Bad value for memleak tolerance in test") - if memdiff < 0: - self._test_status.set_status( - MEMLEAK_PHASE, - TEST_PASS_STATUS, - comments="data for memleak test is insuffiencient", - ) - elif memdiff < tolerance: - self._test_status.set_status(MEMLEAK_PHASE, TEST_PASS_STATUS) - else: - comment = "memleak detected, memory went from {:f} to {:f} in {:d} days".format( - originalmem, finalmem, finaldate - originaldate - ) - append_testlog(comment, self._orig_caseroot) - self._test_status.set_status( - MEMLEAK_PHASE, TEST_FAIL_STATUS, comments=comment - ) + comment = "memleak detected, memory went from {:f} to {:f} in {:d} days".format( + originalmem, finalmem, finaldate - originaldate + ) + append_testlog(comment, self._orig_caseroot) + self._test_status.set_status( + MEMLEAK_PHASE, TEST_FAIL_STATUS, comments=comment + ) def compare_env_run(self, expected=None): """ diff --git a/CIME/baselines.py b/CIME/baselines.py index 0f7d1c16b19..e74483d5a6d 100644 --- a/CIME/baselines.py +++ b/CIME/baselines.py @@ -66,7 +66,7 @@ def compare_memory(case, baseline_dir=None): try: current = config.get_mem_usage(case) except AttributeError: - current = default_get_mem_usage(case) + current = default_get_mem_usage(case)[-1][1] except RuntimeError as e: return None, str(e) @@ -208,19 +208,25 @@ def write_baseline(case, basegen_dir, throughput=True, memory=True): if throughput: try: - tput = config.get_throughput(case) - except AttributeError: - tput = str(default_get_throughput(case)) - - write_baseline_tput(basegen_dir, tput) + try: + tput = config.get_throughput(case) + except AttributeError: + tput = str(default_get_throughput(case)) + except RuntimeError: + pass + else: + write_baseline_tput(basegen_dir, tput) if memory: try: - mem = config.get_mem_usage(case) - except AttributeError: - mem = str(default_get_mem_usage(case)) - - write_baseline_mem(basegen_dir, mem) + try: + mem = config.get_mem_usage(case) + except AttributeError: + mem = str(default_get_mem_usage(case)[-1][1]) + except RuntimeError: + pass + else: + write_baseline_mem(basegen_dir, mem) def default_get_throughput(case): @@ -264,6 +270,8 @@ def default_get_mem_usage(case, cpllog=None): """ if cpllog is None: cpllog = get_latest_cpl_logs(case) + else: + cpllog = [cpllog,] try: memlist = get_cpl_mem_usage(cpllog[0]) @@ -275,7 +283,7 @@ def default_get_mem_usage(case, cpllog=None): f"Found {len(memlist)} memory usage samples, need atleast 4" ) - return memlist[-1][1] + return memlist def write_baseline_tput(baseline_dir, tput): diff --git a/CIME/tests/test_unit_baselines.py b/CIME/tests/test_unit_baselines.py index 644c6b28dcb..91e7b72d8a3 100644 --- a/CIME/tests/test_unit_baselines.py +++ b/CIME/tests/test_unit_baselines.py @@ -161,7 +161,7 @@ def test_default_get_mem_usage(self, get_latest_cpl_logs, get_cpl_mem_usage): mem = baselines.default_get_mem_usage(case) - assert mem == None + assert mem == [(None, None)] @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.get_latest_cpl_logs") From 4bf66054ccdbaf26bfc1323e239788f175c6da78 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 6 Oct 2023 15:16:28 -0700 Subject: [PATCH 14/31] Fixes black formatting --- CIME/baselines.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CIME/baselines.py b/CIME/baselines.py index e74483d5a6d..e084fe54b3a 100644 --- a/CIME/baselines.py +++ b/CIME/baselines.py @@ -271,7 +271,9 @@ def default_get_mem_usage(case, cpllog=None): if cpllog is None: cpllog = get_latest_cpl_logs(case) else: - cpllog = [cpllog,] + cpllog = [ + cpllog, + ] try: memlist = get_cpl_mem_usage(cpllog[0]) From e61d6412640f92f41029948ef7149700f55f158e Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Mon, 9 Oct 2023 14:27:59 -0700 Subject: [PATCH 15/31] Updates permissions --- .github/workflows/testing.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 2f0b35042f5..7010406cae4 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -25,6 +25,7 @@ concurrency: permissions: contents: read # to fetch code (actions/checkout) + packages: read jobs: pre-commit: From b353d1d30402565a81d1e24f2bf0a213dcf13c92 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Mon, 9 Oct 2023 14:30:20 -0700 Subject: [PATCH 16/31] Removes --pull --- .github/workflows/testing.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 7010406cae4..53cead36411 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -59,7 +59,6 @@ jobs: credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - options: '--pull=always' strategy: matrix: python-version: ['3.8', '3.9', '3.10'] @@ -99,7 +98,6 @@ jobs: credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - options: '--pull=always' strategy: matrix: model: ["e3sm", "cesm"] From ecab15438bc673e548ebcae7ae6bb245efa407b0 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Wed, 11 Oct 2023 09:32:41 -0700 Subject: [PATCH 17/31] Fixes BASECMP_CASE name --- CIME/case/case.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CIME/case/case.py b/CIME/case/case.py index 14bb0069024..2bf14540205 100644 --- a/CIME/case/case.py +++ b/CIME/case/case.py @@ -210,7 +210,7 @@ def __init__(self, case_root=None, read_only=True, record=False, non_local=False def get_baseline_dir(self): baseline_root = self.get_value("BASELINE_ROOT") - baseline_name = self.get_value("BASECMP_self") + baseline_name = self.get_value("BASECMP_CASE") return os.path.join(baseline_root, baseline_name) From a786b10b35e429f8071e0603d2116b4f58eed03a Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Wed, 11 Oct 2023 23:35:38 -0700 Subject: [PATCH 18/31] Cleans up baselines --- CIME/SystemTests/system_tests_common.py | 8 +- CIME/baselines.py | 556 +++++++++++++----------- CIME/tests/test_unit_baselines.py | 189 ++++---- CIME/tests/test_unit_system_tests.py | 181 +++++++- 4 files changed, 584 insertions(+), 350 deletions(-) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index 45e7ae932d3..43c08d45c7f 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -28,7 +28,7 @@ from CIME.locked_files import LOCKED_DIR, lock_file, is_locked from CIME.baselines import ( get_latest_cpl_logs, - default_get_mem_usage, + get_default_mem_usage, compare_memory, compare_throughput, write_baseline, @@ -635,12 +635,12 @@ def _check_for_memleak(self): latestcpllogs = get_latest_cpl_logs(self._case) for cpllog in latestcpllogs: try: - memlist = default_get_mem_usage(self._case, cpllog) + memlist = get_default_mem_usage(self._case, cpllog) except RuntimeError: self._test_status.set_status( MEMLEAK_PHASE, TEST_PASS_STATUS, - comments="insuffiencient data for memleak test", + comments="insufficient data for memleak test", ) continue @@ -662,7 +662,7 @@ def _check_for_memleak(self): self._test_status.set_status( MEMLEAK_PHASE, TEST_PASS_STATUS, - comments="data for memleak test is insuffiencient", + comments="data for memleak test is insufficient", ) elif memdiff < tolerance: self._test_status.set_status(MEMLEAK_PHASE, TEST_PASS_STATUS) diff --git a/CIME/baselines.py b/CIME/baselines.py index e084fe54b3a..47f74a698fd 100644 --- a/CIME/baselines.py +++ b/CIME/baselines.py @@ -9,69 +9,89 @@ logger = logging.getLogger(__name__) -def get_latest_cpl_logs(case): +def compare_throughput(case, baseline_dir=None): """ - find and return the latest cpl log file in the run directory + Compares model throughput. + + Parameters + ---------- + case : CIME.case.case.Case + Current case object. + baseline_dir : str + Overrides the baseline directory. + + Returns + ------- + below_tolerance : bool + Whether the comparison was below the tolerance. + comment : str + Provides explanation from comparison. """ - coupler_log_path = case.get_value("RUNDIR") + if baseline_dir is None: + baseline_dir = case.get_baseline_dir() - cpllog_name = "drv" if case.get_value("COMP_INTERFACE") == "nuopc" else "cpl" + config = load_coupler_customization(case) - cpllogs = glob.glob(os.path.join(coupler_log_path, "{}*.log.*".format(cpllog_name))) + baseline_file = os.path.join(baseline_dir, "cpl-tput.log") - lastcpllogs = [] + try: + baseline = read_baseline_file(baseline_file) + except FileNotFoundError as e: + comment = f"Could not read baseline throughput file: {e!s}" - if cpllogs: - lastcpllogs.append(max(cpllogs, key=os.path.getctime)) + logger.debug(comment) - basename = os.path.basename(lastcpllogs[0]) + return None, comment - suffix = basename.split(".", 1)[1] + tolerance = case.get_value("TEST_TPUT_TOLERANCE") - for log in cpllogs: - if log in lastcpllogs: - continue + if tolerance is None: + tolerance = 0.1 - if log.endswith(suffix): - lastcpllogs.append(log) + expect( + tolerance > 0.0, + "Bad value for throughput tolerance in test", + ) - return lastcpllogs + try: + below_tolerance, comment = config.compare_baseline_throughput( + case, baseline, tolerance + ) + except AttributeError: + below_tolerance, comment = compare_baseline_throughput( + case, baseline, tolerance + ) + + return below_tolerance, comment def compare_memory(case, baseline_dir=None): """ - Compare current memory usage to baseline. + Compares model highwater memory usage. Parameters ---------- case : CIME.case.case.Case - Case object. - baseline_dir : str or None - Path to the baseline directory. + Current case object. + baseline_dir : str + Overrides the baseline directory. Returns ------- below_tolerance : bool - Whether the current memory usage is below the baseline. - comments : str - Comments about baseline comparison. + Whether the comparison was below the tolerance. + comment : str + Provides explanation from comparison. """ if baseline_dir is None: baseline_dir = case.get_baseline_dir() config = load_coupler_customization(case) - # TODO need better handling - try: - try: - current = config.get_mem_usage(case) - except AttributeError: - current = default_get_mem_usage(case)[-1][1] - except RuntimeError as e: - return None, str(e) + baseline_file = os.path.join(baseline_dir, "cpl-mem.log") try: - baseline = read_baseline_mem(baseline_dir) + baseline = read_baseline_file(baseline_file) except FileNotFoundError as e: comment = f"Could not read baseline memory usage: {e!s}" @@ -79,94 +99,67 @@ def compare_memory(case, baseline_dir=None): return None, comment - if baseline is None: - baseline = 0.0 - tolerance = case.get_value("TEST_MEMLEAK_TOLERANCE") if tolerance is None: tolerance = 0.1 try: - below_tolerance, comments = config.compare_memory_baseline( - current, baseline, tolerance + below_tolerance, comments = config.compare_baseline_memory( + case, baseline, tolerance ) except AttributeError: - below_tolerance, comments = compare_memory_baseline( - current, baseline, tolerance - ) + below_tolerance, comments = compare_baseline_memory(case, baseline, tolerance) return below_tolerance, comments -def compare_memory_baseline(current, baseline, tolerance): - try: - diff = (current - baseline) / baseline - except ZeroDivisionError: - diff = 0.0 - - # Should we check if tolerance is above 0 - below_tolerance = None - comment = "" - - if diff is not None: - below_tolerance = diff < tolerance - - if below_tolerance: - comment = "MEMCOMP: Memory usage highwater has changed by {:.2f}% relative to baseline".format( - diff * 100 - ) - else: - comment = "Error: Memory usage increase >{:d}% from baseline's {:f} to {:f}".format( - int(tolerance * 100), baseline, current - ) - - return below_tolerance, comment - - -def compare_throughput(case, baseline_dir=None): - if baseline_dir is None: - baseline_dir = case.get_baseline_dir() +def write_baseline(case, basegen_dir, throughput=True, memory=True): + """ + Writes the baseline performance files. + Parameters + ---------- + case : CIME.case.case.Case + Current case object. + basegen_dir : str + Path to baseline directory. + throughput : bool + If true, write throughput baseline. + memory : bool + If true, write memory baseline. + """ config = load_coupler_customization(case) - try: - current = config.get_throughput(case) - except AttributeError: - current = default_get_throughput(case) - - try: - baseline = read_baseline_tput(baseline_dir) - except FileNotFoundError as e: - comment = f"Could not read baseline throughput file: {e!s}" + if throughput: + tput = get_throughput(case, config) - logger.debug(comment) + baseline_file = os.path.join(basegen_dir, "cpl-tput.log") - return None, comment + write_baseline_file(baseline_file, tput) - tolerance = case.get_value("TEST_TPUT_TOLERANCE") + if memory: + mem = get_mem_usage(case, config) - if tolerance is None: - tolerance = 0.1 + baseline_file = os.path.join(basegen_dir, "cpl-mem.log") - expect( - tolerance > 0.0, - "Bad value for throughput tolerance in test", - ) + write_baseline_file(baseline_file, mem) - try: - below_tolerance, comment = config.compare_baseline_throughput( - current, baseline, tolerance - ) - except AttributeError: - below_tolerance, comment = compare_baseline_throughput( - current, baseline, tolerance - ) - return below_tolerance, comment +def load_coupler_customization(case): + """ + Loads customizations from the coupler `cime_config` directory. + Parameters + ---------- + case : CIME.case.case.Case + Current case object. -def load_coupler_customization(case): + Returns + ------- + CIME.config.Config + Runtime configuration. + """ comp_root_dir_cpl = case.get_value("COMP_ROOT_DIR_CPL") cpl_customize = os.path.join(comp_root_dir_cpl, "cime_config", "customize") @@ -174,94 +167,92 @@ def load_coupler_customization(case): return Config.load(cpl_customize) -def compare_baseline_throughput(current, baseline, tolerance): - try: - # comparing ypd so bigger is better - diff = (baseline - current) / baseline - except (ValueError, TypeError): - # Should we default the diff to 0.0 as with _compare_current_memory? - comment = f"Could not determine diff with baseline {baseline!r} and current {current!r}" +def get_throughput(case, config): + """ + Gets the model throughput. - logger.debug(comment) + First attempts to use a coupler define method to retrieve the + models throughput. If this is not defined then the default + method of parsing the coupler log is used. - diff = None + Parameters + ---------- + case : CIME.case.case.Case + Current case object. - below_tolerance = None + Returns + ------- + str or None + Model throughput. + """ + try: + tput = config.get_throughput(case) + except AttributeError: + tput = str(get_default_throughput(case)) - if diff is not None: - below_tolerance = diff < tolerance + return tput - if below_tolerance: - comment = "TPUTCOMP: Computation time changed by {:.2f}% relative to baseline".format( - diff * 100 - ) - else: - comment = "Error: TPUTCOMP: Computation time increase > {:d}% from baseline".format( - int(tolerance * 100) - ) - return below_tolerance, comment +def get_mem_usage(case, config): + """ + Gets the model memory usage. + First attempts to use a coupler defined method to retrieve the + models memory usage. If this is not defined then the default + method of parsing the coupler log is used. -def write_baseline(case, basegen_dir, throughput=True, memory=True): - config = load_coupler_customization(case) + Parameters + ---------- + case : CIME.case.case.Case + Current case object. - if throughput: - try: - try: - tput = config.get_throughput(case) - except AttributeError: - tput = str(default_get_throughput(case)) - except RuntimeError: - pass - else: - write_baseline_tput(basegen_dir, tput) + Returns + ------- + str or None + Model memory usage. + """ + try: + mem = config.get_mem_usage(case) + except AttributeError: + mem = get_default_mem_usage(case) - if memory: - try: - try: - mem = config.get_mem_usage(case) - except AttributeError: - mem = str(default_get_mem_usage(case)[-1][1]) - except RuntimeError: - pass - else: - write_baseline_mem(basegen_dir, mem) + mem = str(mem[-1][1]) + + return mem -def default_get_throughput(case): +def write_baseline_file(baseline_file, value): """ + Writes value to `baseline_file`. + Parameters ---------- - cpllog : str - Path to the coupler log. - - Returns - ------- - str - Last recorded highwater memory usage. + baseline_file : str + Path to the baseline file. + value : str + Value to write. """ - cpllog = get_latest_cpl_logs(case) + with open(baseline_file, "w") as fd: + fd.write(value) - try: - tput = get_cpl_throughput(cpllog[0]) - except (FileNotFoundError, IndexError): - tput = None - return tput +def get_default_mem_usage(case, cpllog=None): + """ + Default function to retrieve memory usage from the coupler log. + If the usage is not available from the log then `None` is returned. -def default_get_mem_usage(case, cpllog=None): - """ Parameters ---------- + case : CIME.case.case.Case + Current case object. cpllog : str - Path to the coupler log. + Overrides the default coupler log. Returns ------- - str - Last recorded highwater memory usage. + str or None + Model memory usage or `None`. Raises ------ @@ -278,7 +269,7 @@ def default_get_mem_usage(case, cpllog=None): try: memlist = get_cpl_mem_usage(cpllog[0]) except (FileNotFoundError, IndexError): - memlist = [(None, None)] + memlist = None else: if len(memlist) <= 3: raise RuntimeError( @@ -288,113 +279,59 @@ def default_get_mem_usage(case, cpllog=None): return memlist -def write_baseline_tput(baseline_dir, tput): +def get_default_throughput(case): """ - Writes throughput to baseline file. + Default function to retrieve throughput from the coupler log. - The format is arbitrary, it's the callers responsibilty - to decode the data. + If the throughput is not available from the log then `None` is returned. Parameters ---------- - baseline_dir : str - Path to the baseline directory. - tput : str - Model throughput. - """ - tput_file = os.path.join(baseline_dir, "cpl-tput.log") - - with open(tput_file, "w") as fd: - fd.write(tput) - - -def write_baseline_mem(baseline_dir, mem): - """ - Writes memory usage to baseline file. - - The format is arbitrary, it's the callers responsibilty - to decode the data. + case : CIME.case.case.Case + Current case object. - Parameters - ---------- - baseline_dir : str - Path to the baseline directory. - mem : str - Model memory usage. + Returns + ------- + str or None + Model throughput or `None`. """ - mem_file = os.path.join(baseline_dir, "cpl-mem.log") - - with open(mem_file, "w") as fd: - fd.write(mem) - + cpllog = get_latest_cpl_logs(case) -def read_baseline_tput(baseline_dir): - """ - Reads the raw lines of the throughput baseline file. + try: + tput = get_cpl_throughput(cpllog[0]) + except (FileNotFoundError, IndexError): + tput = None - Parameters - ---------- - baseline_dir : str - Path to the baseline directory. + return tput - Returns - ------- - list - Contents of the throughput baseline file. - Raises - ------ - FileNotFoundError - If baseline file does not exist. +def get_latest_cpl_logs(case): """ - return read_baseline_value(os.path.join(baseline_dir, "cpl-tput.log")) - - -def read_baseline_mem(baseline_dir): + find and return the latest cpl log file in the run directory """ - Read the raw lines of the memory baseline file. + coupler_log_path = case.get_value("RUNDIR") - Parameters - ---------- - baseline_dir : str - Path to the baseline directory. + cpllog_name = "drv" if case.get_value("COMP_INTERFACE") == "nuopc" else "cpl" - Returns - ------- - list - Contents of the memory baseline file. + cpllogs = glob.glob(os.path.join(coupler_log_path, "{}*.log.*".format(cpllog_name))) - Raises - ------ - FileNotFoundError - If baseline file does not exist. - """ - return read_baseline_value(os.path.join(baseline_dir, "cpl-mem.log")) + lastcpllogs = [] + if cpllogs: + lastcpllogs.append(max(cpllogs, key=os.path.getctime)) -def read_baseline_value(baseline_file): - """ - Generic read function, ignores lines prepended by `#`. + basename = os.path.basename(lastcpllogs[0]) - Parameters - ---------- - baseline_file : str - Path to baseline file. + suffix = basename.split(".", 1)[1] - Returns - ------- - list - Lines contained in the baseline file without comments. + for log in cpllogs: + if log in lastcpllogs: + continue - Raises - ------ - FileNotFoundError - If ``baseline_file`` is not found. - """ - with open(baseline_file) as fd: - lines = [x for x in fd.readlines() if not x.startswith("#")] + if log.endswith(suffix): + lastcpllogs.append(log) - return lines + return lastcpllogs def get_cpl_mem_usage(cpllog): @@ -458,3 +395,134 @@ def get_cpl_throughput(cpllog): if m: return float(m.group(1)) return None + + +def read_baseline_file(baseline_file): + """ + Reads value from `baseline_file`. + + Parameters + ---------- + baseline_file : str + Path to the baseline file. + + Returns + ------- + str + Value stored in baseline file without comments. + """ + with open(baseline_file) as fd: + lines = [x for x in fd.readlines() if not x.startswith("#")] + + return lines + + +def compare_baseline_throughput(case, baseline, tolerance): + """ + Default throughput baseline comparison. + + Compares the throughput from the coupler to the baseline value. + + Parameters + ---------- + case : CIME.case.case.Case + Current case object. + baseline : list + Lines contained in the baseline file. + tolerance : float + Allowed tolerance for comparison. + + Returns + ------- + below_tolerance : bool + Whether the comparison was below the tolerance. + comment : str + provides explanation from comparison. + """ + current = get_default_throughput(case) + + try: + # default baseline is stored as single float + baseline = float(baseline[0]) + except IndexError: + comment = "Could not compare throughput to baseline, as basline had no value." + + return None, comment + + # comparing ypd so bigger is better + diff = (baseline - current) / baseline + + below_tolerance = None + + if diff is not None: + below_tolerance = diff < tolerance + + if below_tolerance: + comment = "TPUTCOMP: Computation time changed by {:.2f}% relative to baseline".format( + diff * 100 + ) + else: + comment = "Error: TPUTCOMP: Computation time increase > {:d}% from baseline".format( + int(tolerance * 100) + ) + + return below_tolerance, comment + + +def compare_baseline_memory(case, baseline, tolerance): + """ + Default memory usage baseline comparison. + + Compares the highwater memory usage from the coupler to the baseline value. + + Parameters + ---------- + case : CIME.case.case.Case + Current case object. + baseline : list + Lines contained in the baseline file. + tolerance : float + Allowed tolerance for comparison. + + Returns + ------- + below_tolerance : bool + Whether the comparison was below the tolerance. + comment : str + provides explanation from comparison. + """ + try: + current = get_default_mem_usage(case) + except RuntimeError as e: + return None, str(e) + else: + current = current[-1][1] + + try: + # default baseline is stored as single float + baseline = float(baseline[0]) + except IndexError: + baseline = 0.0 + + try: + diff = (current - baseline) / baseline + except ZeroDivisionError: + diff = 0.0 + + # Should we check if tolerance is above 0 + below_tolerance = None + comment = "" + + if diff is not None: + below_tolerance = diff < tolerance + + if below_tolerance: + comment = "MEMCOMP: Memory usage highwater has changed by {:.2f}% relative to baseline".format( + diff * 100 + ) + else: + comment = "Error: Memory usage increase >{:d}% from baseline's {:f} to {:f}".format( + int(tolerance * 100), baseline, current + ) + + return below_tolerance, comment diff --git a/CIME/tests/test_unit_baselines.py b/CIME/tests/test_unit_baselines.py index 91e7b72d8a3..ae98b2aa79a 100644 --- a/CIME/tests/test_unit_baselines.py +++ b/CIME/tests/test_unit_baselines.py @@ -75,93 +75,75 @@ def test_get_cpl_mem_usage(self, isfile): (10105.0, 1673.89), ] - def test_read_baseline_mem_empty(self): - with mock.patch("builtins.open", mock.mock_open(read_data="")) as mock_file: - baseline = baselines.read_baseline_mem("/tmp/cpl-mem.log") - - assert baseline == [] - - def test_read_baseline_mem_none(self): - with mock.patch("builtins.open", mock.mock_open(read_data="-1")) as mock_file: - baseline = baselines.read_baseline_mem("/tmp/cpl-mem.log") + def test_read_baseline_file_multi_line(self): + with mock.patch( + "builtins.open", mock.mock_open(read_data="1000.0\n2000.0\n") + ) as mock_file: + baseline = baselines.read_baseline_file("/tmp/cpl-mem.log") - assert baseline == ["-1"] + mock_file.assert_called_with("/tmp/cpl-mem.log") + assert baseline == ["1000.0\n", "2000.0\n"] - def test_read_baseline_mem(self): - with mock.patch("builtins.open", mock.mock_open(read_data="200")) as mock_file: - baseline = baselines.read_baseline_mem("/tmp/cpl-mem.log") + def test_read_baseline_file_content(self): + with mock.patch( + "builtins.open", mock.mock_open(read_data="1000.0") + ) as mock_file: + baseline = baselines.read_baseline_file("/tmp/cpl-mem.log") - assert baseline == ["200"] + mock_file.assert_called_with("/tmp/cpl-mem.log") + assert baseline == ["1000.0"] - def test_read_baseline_tput_empty(self): + def test_read_baseline_file(self): with mock.patch("builtins.open", mock.mock_open(read_data="")) as mock_file: - baseline = baselines.read_baseline_tput("/tmp/cpl-tput.log") + baseline = baselines.read_baseline_file("/tmp/cpl-mem.log") + mock_file.assert_called_with("/tmp/cpl-mem.log") assert baseline == [] - def test_read_baseline_tput_none(self): - with mock.patch("builtins.open", mock.mock_open(read_data="-1")) as mock_file: - baseline = baselines.read_baseline_tput("/tmp/cpl-tput.log") - - assert baseline == ["-1"] - - def test_read_baseline_tput(self): - with mock.patch("builtins.open", mock.mock_open(read_data="200")) as mock_file: - baseline = baselines.read_baseline_tput("/tmp/cpl-tput.log") - - assert baseline == ["200"] - - def test_write_baseline_mem_no_value(self): - with mock.patch("builtins.open", mock.mock_open()) as mock_file: - baselines.write_baseline_mem("/tmp", "-1") - - mock_file.assert_called_with("/tmp/cpl-mem.log", "w") - mock_file.return_value.write.assert_called_with("-1") - - def test_write_baseline_mem(self): + def test_write_baseline_file(self): with mock.patch("builtins.open", mock.mock_open()) as mock_file: - baselines.write_baseline_mem("/tmp", "200") - - mock_file.assert_called_with("/tmp/cpl-mem.log", "w") - mock_file.return_value.write.assert_called_with("200") - - def test_write_baseline_tput_no_value(self): - with mock.patch("builtins.open", mock.mock_open()) as mock_file: - baselines.write_baseline_tput("/tmp", "-1") + baselines.write_baseline_file("/tmp/cpl-tput.log", "1000") mock_file.assert_called_with("/tmp/cpl-tput.log", "w") - mock_file.return_value.write.assert_called_with("-1") - - def test_write_baseline_tput(self): - with mock.patch("builtins.open", mock.mock_open()) as mock_file: - baselines.write_baseline_tput("/tmp", "200") - - mock_file.assert_called_with("/tmp/cpl-tput.log", "w") - mock_file.return_value.write.assert_called_with("200") + mock_file.return_value.write.assert_called_with("1000") @mock.patch("CIME.baselines.get_cpl_throughput") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_default_get_throughput(self, get_latest_cpl_logs, get_cpl_throughput): + def test_get_default_throughput(self, get_latest_cpl_logs, get_cpl_throughput): get_cpl_throughput.side_effect = FileNotFoundError() with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) - tput = baselines.default_get_throughput(case) + tput = baselines.get_default_throughput(case) assert tput == None @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_default_get_mem_usage(self, get_latest_cpl_logs, get_cpl_mem_usage): + def test_get_default_mem_usage_override( + self, get_latest_cpl_logs, get_cpl_mem_usage + ): + get_cpl_mem_usage.side_effect = FileNotFoundError() + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + + mem = baselines.get_default_mem_usage(case, "/tmp/override") + + assert mem == None + + @mock.patch("CIME.baselines.get_cpl_mem_usage") + @mock.patch("CIME.baselines.get_latest_cpl_logs") + def test_get_default_mem_usage(self, get_latest_cpl_logs, get_cpl_mem_usage): get_cpl_mem_usage.side_effect = FileNotFoundError() with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) - mem = baselines.default_get_mem_usage(case) + mem = baselines.get_default_mem_usage(case) - assert mem == [(None, None)] + assert mem == None @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.get_latest_cpl_logs") @@ -180,15 +162,15 @@ def test_write_baseline(self, get_latest_cpl_logs, get_cpl_mem_usage): case, baseline_root, get_latest_cpl_logs.return_value ) - @mock.patch("CIME.baselines.get_cpl_throughput") - @mock.patch("CIME.baselines.read_baseline_tput") + @mock.patch("CIME.baselines.get_default_throughput") + @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_throughput_no_baseline_file( - self, get_latest_cpl_logs, read_baseline_tput, get_cpl_throughput + self, get_latest_cpl_logs, read_baseline_file, get_default_throughput ): - read_baseline_tput.side_effect = FileNotFoundError + read_baseline_file.side_effect = FileNotFoundError - get_cpl_throughput.return_value = 504 + get_default_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -205,15 +187,15 @@ def test_compare_throughput_no_baseline_file( assert below_tolerance is None assert comment == "Could not read baseline throughput file: " - @mock.patch("CIME.baselines.default_get_throughput") - @mock.patch("CIME.baselines.read_baseline_tput") + @mock.patch("CIME.baselines.get_default_throughput") + @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_throughput_no_baseline( - self, get_latest_cpl_logs, read_baseline_tput, default_get_throughput + self, get_latest_cpl_logs, read_baseline_file, get_default_throughput ): - read_baseline_tput.return_value = None + read_baseline_file.return_value = [] - default_get_throughput.return_value = 504 + get_default_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -230,17 +212,22 @@ def test_compare_throughput_no_baseline( (below_tolerance, comment) = baselines.compare_throughput(case) assert below_tolerance is None - assert comment == "Could not determine diff with baseline None and current 504" + assert ( + comment + == "Could not compare throughput to baseline, as basline had no value." + ) - @mock.patch("CIME.baselines.default_get_throughput") - @mock.patch("CIME.baselines.read_baseline_tput") + @mock.patch("CIME.baselines.get_default_throughput") + @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_throughput_no_tolerance( - self, get_latest_cpl_logs, read_baseline_tput, default_get_throughput + self, get_latest_cpl_logs, read_baseline_file, get_default_throughput ): - read_baseline_tput.return_value = 500 + read_baseline_file.return_value = [ + "500", + ] - default_get_throughput.return_value = 504 + get_default_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -262,15 +249,15 @@ def test_compare_throughput_no_tolerance( == "TPUTCOMP: Computation time changed by -0.80% relative to baseline" ) - @mock.patch("CIME.baselines.default_get_throughput") - @mock.patch("CIME.baselines.read_baseline_tput") + @mock.patch("CIME.baselines.get_default_throughput") + @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_throughput_above_threshold( - self, get_latest_cpl_logs, read_baseline_tput, default_get_throughput + self, get_latest_cpl_logs, read_baseline_file, get_default_throughput ): - read_baseline_tput.return_value = 1000 + read_baseline_file.return_value = ["1000"] - default_get_throughput.return_value = 504 + get_default_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -291,15 +278,15 @@ def test_compare_throughput_above_threshold( comment == "Error: TPUTCOMP: Computation time increase > 5% from baseline" ) - @mock.patch("CIME.baselines.default_get_throughput") - @mock.patch("CIME.baselines.read_baseline_tput") + @mock.patch("CIME.baselines.get_default_throughput") + @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_throughput( - self, get_latest_cpl_logs, read_baseline_tput, default_get_throughput + self, get_latest_cpl_logs, read_baseline_file, get_default_throughput ): - read_baseline_tput.return_value = 500 + read_baseline_file.return_value = ["500"] - default_get_throughput.return_value = 504 + get_default_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -322,12 +309,12 @@ def test_compare_throughput( ) @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.read_baseline_mem") + @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_memory_no_baseline( - self, get_latest_cpl_logs, read_baseline_mem, get_cpl_mem_usage + self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): - read_baseline_mem.return_value = None + read_baseline_file.return_value = [] get_cpl_mem_usage.return_value = [ (1, 1000.0), @@ -357,12 +344,12 @@ def test_compare_memory_no_baseline( ) @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.read_baseline_mem") + @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_memory_not_enough_samples( - self, get_latest_cpl_logs, read_baseline_mem, get_cpl_mem_usage + self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): - read_baseline_mem.return_value = 1000.0 + read_baseline_file.return_value = ["1000.0"] get_cpl_mem_usage.return_value = [ (1, 1000.0), @@ -385,12 +372,12 @@ def test_compare_memory_not_enough_samples( assert comment == "Found 2 memory usage samples, need atleast 4" @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.read_baseline_mem") + @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_memory_no_baseline_file( - self, get_latest_cpl_logs, read_baseline_mem, get_cpl_mem_usage + self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): - read_baseline_mem.side_effect = FileNotFoundError + read_baseline_file.side_effect = FileNotFoundError get_cpl_mem_usage.return_value = [ (1, 1000.0), @@ -415,12 +402,12 @@ def test_compare_memory_no_baseline_file( assert comment == "Could not read baseline memory usage: " @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.read_baseline_mem") + @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_memory_no_tolerance( - self, get_latest_cpl_logs, read_baseline_mem, get_cpl_mem_usage + self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): - read_baseline_mem.return_value = 1000.0 + read_baseline_file.return_value = ["1000.0"] get_cpl_mem_usage.return_value = [ (1, 1000.0), @@ -450,12 +437,12 @@ def test_compare_memory_no_tolerance( ) @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.read_baseline_mem") + @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_memory_above_threshold( - self, get_latest_cpl_logs, read_baseline_mem, get_cpl_mem_usage + self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): - read_baseline_mem.return_value = 1000.0 + read_baseline_file.return_value = ["1000.0"] get_cpl_mem_usage.return_value = [ (1, 2000.0), @@ -485,12 +472,12 @@ def test_compare_memory_above_threshold( ) @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.read_baseline_mem") + @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_compare_memory( - self, get_latest_cpl_logs, read_baseline_mem, get_cpl_mem_usage + self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): - read_baseline_mem.return_value = 1000.0 + read_baseline_file.return_value = ["1000.0"] get_cpl_mem_usage.return_value = [ (1, 1000.0), diff --git a/CIME/tests/test_unit_system_tests.py b/CIME/tests/test_unit_system_tests.py index 9ea088411bd..fc684321cc0 100644 --- a/CIME/tests/test_unit_system_tests.py +++ b/CIME/tests/test_unit_system_tests.py @@ -66,7 +66,186 @@ def create_mock_case(tempdir, idx=None, cpllog_data=None): return case, caseroot, baseline_root, run_dir -class TestCaseSubmit(unittest.TestCase): +class TestUnitSystemTests(unittest.TestCase): + @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") + @mock.patch("CIME.SystemTests.system_tests_common.get_default_mem_usage") + @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") + def test_check_for_memleak_runtime_error( + self, get_latest_cpl_logs, get_default_mem_usage, append_testlog + ): + get_default_mem_usage.side_effect = RuntimeError + + with tempfile.TemporaryDirectory() as tempdir: + caseroot = Path(tempdir) / "caseroot" + caseroot.mkdir(parents=True, exist_ok=False) + + rundir = caseroot / "run" + rundir.mkdir(parents=True, exist_ok=False) + + cpllog = rundir / "cpl.log.gz" + + get_latest_cpl_logs.return_value = [ + str(cpllog), + ] + + case = mock.MagicMock() + case.get_value.side_effect = ( + str(caseroot), + "ERIO.ne30_g16_rx1.A.docker_gnu", + "mct", + 0.01, + ) + + common = SystemTestsCommon(case) + + common._test_status = mock.MagicMock() + + common._check_for_memleak() + + common._test_status.set_status.assert_any_call( + "MEMLEAK", "PASS", comments="insufficient data for memleak test" + ) + + append_testlog.assert_not_called() + + @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") + @mock.patch("CIME.SystemTests.system_tests_common.get_default_mem_usage") + @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") + def test_check_for_memleak_not_enough_samples( + self, get_latest_cpl_logs, get_default_mem_usage, append_testlog + ): + get_default_mem_usage.return_value = [ + (1, 1000.0), + (2, 0), + ] + + with tempfile.TemporaryDirectory() as tempdir: + caseroot = Path(tempdir) / "caseroot" + caseroot.mkdir(parents=True, exist_ok=False) + + rundir = caseroot / "run" + rundir.mkdir(parents=True, exist_ok=False) + + cpllog = rundir / "cpl.log.gz" + + get_latest_cpl_logs.return_value = [ + str(cpllog), + ] + + case = mock.MagicMock() + case.get_value.side_effect = ( + str(caseroot), + "ERIO.ne30_g16_rx1.A.docker_gnu", + "mct", + 0.01, + ) + + common = SystemTestsCommon(case) + + common._test_status = mock.MagicMock() + + common._check_for_memleak() + + common._test_status.set_status.assert_any_call( + "MEMLEAK", "PASS", comments="data for memleak test is insufficient" + ) + + append_testlog.assert_not_called() + + @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") + @mock.patch("CIME.SystemTests.system_tests_common.get_default_mem_usage") + @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") + def test_check_for_memleak_found( + self, get_latest_cpl_logs, get_default_mem_usage, append_testlog + ): + get_default_mem_usage.return_value = [ + (1, 1000.0), + (2, 2000.0), + (3, 3000.0), + (4, 3000.0), + ] + + with tempfile.TemporaryDirectory() as tempdir: + caseroot = Path(tempdir) / "caseroot" + caseroot.mkdir(parents=True, exist_ok=False) + + rundir = caseroot / "run" + rundir.mkdir(parents=True, exist_ok=False) + + cpllog = rundir / "cpl.log.gz" + + get_latest_cpl_logs.return_value = [ + str(cpllog), + ] + + case = mock.MagicMock() + case.get_value.side_effect = ( + str(caseroot), + "ERIO.ne30_g16_rx1.A.docker_gnu", + "mct", + 0.01, + ) + + common = SystemTestsCommon(case) + + common._test_status = mock.MagicMock() + + common._check_for_memleak() + + expected_comment = "memleak detected, memory went from 2000.000000 to 3000.000000 in 2 days" + + common._test_status.set_status.assert_any_call( + "MEMLEAK", "FAIL", comments=expected_comment + ) + + append_testlog.assert_any_call(expected_comment, str(caseroot)) + + @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") + @mock.patch("CIME.SystemTests.system_tests_common.get_default_mem_usage") + @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") + def test_check_for_memleak( + self, get_latest_cpl_logs, get_default_mem_usage, append_testlog + ): + get_default_mem_usage.return_value = [ + (1, 3040.0), + (2, 3002.0), + (3, 3030.0), + (4, 3008.0), + ] + + with tempfile.TemporaryDirectory() as tempdir: + caseroot = Path(tempdir) / "caseroot" + caseroot.mkdir(parents=True, exist_ok=False) + + rundir = caseroot / "run" + rundir.mkdir(parents=True, exist_ok=False) + + cpllog = rundir / "cpl.log.gz" + + get_latest_cpl_logs.return_value = [ + str(cpllog), + ] + + case = mock.MagicMock() + case.get_value.side_effect = ( + str(caseroot), + "ERIO.ne30_g16_rx1.A.docker_gnu", + "mct", + 0.01, + ) + + common = SystemTestsCommon(case) + + common._test_status = mock.MagicMock() + + common._check_for_memleak() + + print(common._test_status.set_status.call_args_list) + + common._test_status.set_status.assert_any_call("MEMLEAK", "PASS") + + append_testlog.assert_not_called() + @mock.patch("CIME.SystemTests.system_tests_common.compare_throughput") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") def test_compare_throughput(self, append_testlog, compare_throughput): From 1c6d7a15057d5f86b1679cee0c9fef9c62cb12f6 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 12 Oct 2023 14:19:38 -0700 Subject: [PATCH 19/31] Fixes handling non performance metrics --- CIME/baselines.py | 32 +++++-- CIME/tests/test_unit_baselines.py | 149 +++++++++++++++++++++++++++--- 2 files changed, 158 insertions(+), 23 deletions(-) diff --git a/CIME/baselines.py b/CIME/baselines.py index 47f74a698fd..1011bfc815b 100644 --- a/CIME/baselines.py +++ b/CIME/baselines.py @@ -132,18 +132,24 @@ def write_baseline(case, basegen_dir, throughput=True, memory=True): config = load_coupler_customization(case) if throughput: - tput = get_throughput(case, config) - - baseline_file = os.path.join(basegen_dir, "cpl-tput.log") + try: + tput = get_throughput(case, config) + except RuntimeError: + pass + else: + baseline_file = os.path.join(basegen_dir, "cpl-tput.log") - write_baseline_file(baseline_file, tput) + write_baseline_file(baseline_file, tput) if memory: - mem = get_mem_usage(case, config) - - baseline_file = os.path.join(basegen_dir, "cpl-mem.log") + try: + mem = get_mem_usage(case, config) + except RuntimeError: + pass + else: + baseline_file = os.path.join(basegen_dir, "cpl-mem.log") - write_baseline_file(baseline_file, mem) + write_baseline_file(baseline_file, mem) def load_coupler_customization(case): @@ -188,7 +194,12 @@ def get_throughput(case, config): try: tput = config.get_throughput(case) except AttributeError: - tput = str(get_default_throughput(case)) + tput = get_default_throughput(case) + + if tput is None: + raise RuntimeError("Could not get default throughput") from None + + tput = str(tput) return tput @@ -216,6 +227,9 @@ def get_mem_usage(case, config): except AttributeError: mem = get_default_mem_usage(case) + if mem is None: + raise RuntimeError("Could not get default memory usage") from None + mem = str(mem[-1][1]) return mem diff --git a/CIME/tests/test_unit_baselines.py b/CIME/tests/test_unit_baselines.py index ae98b2aa79a..a5af4c4a9a2 100644 --- a/CIME/tests/test_unit_baselines.py +++ b/CIME/tests/test_unit_baselines.py @@ -10,12 +10,13 @@ from CIME.tests.test_unit_system_tests import CPLLOG -def create_mock_case(tempdir, get_latest_cpl_logs): +def create_mock_case(tempdir, get_latest_cpl_logs=None): caseroot = Path(tempdir, "0", "caseroot") rundir = caseroot / "run" - get_latest_cpl_logs.return_value = (str(rundir / "cpl.log.gz"),) + if get_latest_cpl_logs is not None: + get_latest_cpl_logs.return_value = (str(rundir / "cpl.log.gz"),) baseline_root = Path(tempdir, "baselines") @@ -27,6 +28,82 @@ def create_mock_case(tempdir, get_latest_cpl_logs): class TestUnitBaseline(unittest.TestCase): + @mock.patch("CIME.baselines.get_default_mem_usage") + def test_get_mem_usage_default_no_value(self, get_default_mem_usage): + get_default_mem_usage.return_value = None + + case = mock.MagicMock() + + config = mock.MagicMock() + + config.get_mem_usage.side_effect = AttributeError + + with self.assertRaises(RuntimeError): + baselines.get_mem_usage(case, config) + + @mock.patch("CIME.baselines.get_default_mem_usage") + def test_get_mem_usage_default(self, get_default_mem_usage): + get_default_mem_usage.return_value = [(1, 1000)] + + case = mock.MagicMock() + + config = mock.MagicMock() + + config.get_mem_usage.side_effect = AttributeError + + mem = baselines.get_mem_usage(case, config) + + assert mem == "1000" + + def test_get_mem_usage(self): + case = mock.MagicMock() + + config = mock.MagicMock() + + config.get_mem_usage.return_value = "1000" + + mem = baselines.get_mem_usage(case, config) + + assert mem == "1000" + + @mock.patch("CIME.baselines.get_default_throughput") + def test_get_throughput_default_no_value(self, get_default_throughput): + get_default_throughput.return_value = None + + case = mock.MagicMock() + + config = mock.MagicMock() + + config.get_throughput.side_effect = AttributeError + + with self.assertRaises(RuntimeError): + baselines.get_throughput(case, config) + + @mock.patch("CIME.baselines.get_default_throughput") + def test_get_throughput_default(self, get_default_throughput): + get_default_throughput.return_value = 100 + + case = mock.MagicMock() + + config = mock.MagicMock() + + config.get_throughput.side_effect = AttributeError + + tput = baselines.get_throughput(case, config) + + assert tput == "100" + + def test_get_throughput(self): + case = mock.MagicMock() + + config = mock.MagicMock() + + config.get_throughput.return_value = "100" + + tput = baselines.get_throughput(case, config) + + assert tput == "100" + def test_get_cpl_throughput_no_file(self): throughput = baselines.get_cpl_throughput("/tmp/cpl.log") @@ -145,23 +222,67 @@ def test_get_default_mem_usage(self, get_latest_cpl_logs, get_cpl_mem_usage): assert mem == None - @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_write_baseline(self, get_latest_cpl_logs, get_cpl_mem_usage): - with tempfile.TemporaryDirectory() as tempdir: - case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) + @mock.patch("CIME.baselines.write_baseline_file") + @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.get_throughput") + def test_write_baseline_skip( + self, get_throughput, get_mem_usage, write_baseline_file + ): + get_throughput.return_value = "100" + + get_mem_usage.return_value = "1000" - get_cpl_mem_usage.return_value = [ - (1, 1000.0), - (2, 1001.0), - (3, 1002.0), - (4, 1003.0), - ] + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir) baselines.write_baseline( - case, baseline_root, get_latest_cpl_logs.return_value + case, + baseline_root, + False, + False, ) + get_throughput.assert_not_called() + get_mem_usage.assert_not_called() + write_baseline_file.assert_not_called() + + @mock.patch("CIME.baselines.write_baseline_file") + @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.get_throughput") + def test_write_baseline_runtimeerror( + self, get_throughput, get_mem_usage, write_baseline_file + ): + get_throughput.side_effect = RuntimeError + + get_mem_usage.side_effect = RuntimeError + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir) + + baselines.write_baseline(case, baseline_root) + + get_throughput.assert_called() + get_mem_usage.assert_called() + write_baseline_file.assert_not_called() + + @mock.patch("CIME.baselines.write_baseline_file") + @mock.patch("CIME.baselines.get_mem_usage") + @mock.patch("CIME.baselines.get_throughput") + def test_write_baseline(self, get_throughput, get_mem_usage, write_baseline_file): + get_throughput.return_value = "100" + + get_mem_usage.return_value = "1000" + + with tempfile.TemporaryDirectory() as tempdir: + case, _, _, baseline_root = create_mock_case(tempdir) + + baselines.write_baseline(case, baseline_root) + + get_throughput.assert_called() + get_mem_usage.assert_called() + write_baseline_file.assert_any_call(str(baseline_root / "cpl-tput.log"), "100") + write_baseline_file.assert_any_call(str(baseline_root / "cpl-mem.log"), "1000") + @mock.patch("CIME.baselines.get_default_throughput") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") From 3596b5526580a10017e9a464664afdd3bc20fb6a Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 12 Oct 2023 15:09:59 -0700 Subject: [PATCH 20/31] Updates _check_for_memleak to be customized by the coupler --- CIME/SystemTests/system_tests_common.py | 97 ++++++++++++++----------- CIME/tests/test_unit_system_tests.py | 50 +++++++++++-- 2 files changed, 99 insertions(+), 48 deletions(-) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index 43c08d45c7f..0fd90c5cfa0 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -32,6 +32,7 @@ compare_memory, compare_throughput, write_baseline, + load_coupler_customization, ) import CIME.build as build @@ -631,49 +632,22 @@ def _check_for_memleak(self): Examine memory usage as recorded in the cpl log file and look for unexpected increases. """ + config = load_coupler_customization(self._case) + with self._test_status: - latestcpllogs = get_latest_cpl_logs(self._case) - for cpllog in latestcpllogs: - try: - memlist = get_default_mem_usage(self._case, cpllog) - except RuntimeError: - self._test_status.set_status( - MEMLEAK_PHASE, - TEST_PASS_STATUS, - comments="insufficient data for memleak test", - ) + try: + memleak, comment = config.detect_memory_leak(self._case) + except AttributeError: + memleak, comment = default_detect_memory_leak(self._case) - continue - - finaldate = int(memlist[-1][0]) - originaldate = int( - memlist[1][0] - ) # skip first day mem record, it can be too low while initializing - finalmem = float(memlist[-1][1]) - originalmem = float(memlist[1][1]) - memdiff = -1 - if originalmem > 0: - memdiff = (finalmem - originalmem) / originalmem - tolerance = self._case.get_value("TEST_MEMLEAK_TOLERANCE") - if tolerance is None: - tolerance = 0.1 - expect(tolerance > 0.0, "Bad value for memleak tolerance in test") - if memdiff < 0: - self._test_status.set_status( - MEMLEAK_PHASE, - TEST_PASS_STATUS, - comments="data for memleak test is insufficient", - ) - elif memdiff < tolerance: - self._test_status.set_status(MEMLEAK_PHASE, TEST_PASS_STATUS) - else: - comment = "memleak detected, memory went from {:f} to {:f} in {:d} days".format( - originalmem, finalmem, finaldate - originaldate - ) - append_testlog(comment, self._orig_caseroot) - self._test_status.set_status( - MEMLEAK_PHASE, TEST_FAIL_STATUS, comments=comment - ) + if memleak: + append_testlog(comment, self._orig_caseroot) + + status = TEST_FAIL_STATUS + else: + status = TEST_PASS_STATUS + + self._test_status.set_status(MEMLEAK_PHASE, status, comments=comment) def compare_env_run(self, expected=None): """ @@ -798,6 +772,47 @@ def _generate_baseline(self): write_baseline(self._case, basegen_dir, cpllog) +def default_detect_memory_leak(case): + leak = False + comment = "" + + latestcpllogs = get_latest_cpl_logs(case) + + for cpllog in latestcpllogs: + try: + memlist = get_default_mem_usage(case, cpllog) + except RuntimeError: + return False, "insufficient data for memleak test" + + # last day - second day, skip first day, can be too low while initializing + elapsed_days = int(memlist[-1][0]) - int(memlist[1][0]) + + finalmem, originalmem = float(memlist[-1][1]), float(memlist[1][1]) + + memdiff = -1 if originalmem <= 0 else (finalmem - originalmem) / originalmem + + # default to 0.1 + tolerance = case.get_value("TEST_MEMLEAK_TOLERANCE") or 0.1 + + expect(tolerance > 0.0, "Bad value for memleak tolerance in test") + + if memdiff < 0: + leak = False + comment = "data for memleak test is insufficient" + elif memdiff < tolerance: + leak = False + comment = "" + else: + leak = True + comment = ( + "memleak detected, memory went from {:f} to {:f} in {:d} days".format( + originalmem, finalmem, elapsed_days + ) + ) + + return leak, comment + + class FakeTest(SystemTestsCommon): """ Inheriters of the FakeTest Class are intended to test the code. diff --git a/CIME/tests/test_unit_system_tests.py b/CIME/tests/test_unit_system_tests.py index fc684321cc0..2f9f7d9918b 100644 --- a/CIME/tests/test_unit_system_tests.py +++ b/CIME/tests/test_unit_system_tests.py @@ -67,12 +67,21 @@ def create_mock_case(tempdir, idx=None, cpllog_data=None): class TestUnitSystemTests(unittest.TestCase): + @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") @mock.patch("CIME.SystemTests.system_tests_common.get_default_mem_usage") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak_runtime_error( - self, get_latest_cpl_logs, get_default_mem_usage, append_testlog + self, + get_latest_cpl_logs, + get_default_mem_usage, + append_testlog, + load_coupler_customization, ): + load_coupler_customization.return_value.detect_memory_leak.side_effect = ( + AttributeError + ) + get_default_mem_usage.side_effect = RuntimeError with tempfile.TemporaryDirectory() as tempdir: @@ -108,12 +117,21 @@ def test_check_for_memleak_runtime_error( append_testlog.assert_not_called() + @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") @mock.patch("CIME.SystemTests.system_tests_common.get_default_mem_usage") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak_not_enough_samples( - self, get_latest_cpl_logs, get_default_mem_usage, append_testlog + self, + get_latest_cpl_logs, + get_default_mem_usage, + append_testlog, + load_coupler_customization, ): + load_coupler_customization.return_value.detect_memory_leak.side_effect = ( + AttributeError + ) + get_default_mem_usage.return_value = [ (1, 1000.0), (2, 0), @@ -152,12 +170,21 @@ def test_check_for_memleak_not_enough_samples( append_testlog.assert_not_called() + @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") @mock.patch("CIME.SystemTests.system_tests_common.get_default_mem_usage") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak_found( - self, get_latest_cpl_logs, get_default_mem_usage, append_testlog + self, + get_latest_cpl_logs, + get_default_mem_usage, + append_testlog, + load_coupler_customization, ): + load_coupler_customization.return_value.detect_memory_leak.side_effect = ( + AttributeError + ) + get_default_mem_usage.return_value = [ (1, 1000.0), (2, 2000.0), @@ -200,12 +227,21 @@ def test_check_for_memleak_found( append_testlog.assert_any_call(expected_comment, str(caseroot)) + @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") @mock.patch("CIME.SystemTests.system_tests_common.get_default_mem_usage") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak( - self, get_latest_cpl_logs, get_default_mem_usage, append_testlog + self, + get_latest_cpl_logs, + get_default_mem_usage, + append_testlog, + load_coupler_customization, ): + load_coupler_customization.return_value.detect_memory_leak.side_effect = ( + AttributeError + ) + get_default_mem_usage.return_value = [ (1, 3040.0), (2, 3002.0), @@ -240,9 +276,9 @@ def test_check_for_memleak( common._check_for_memleak() - print(common._test_status.set_status.call_args_list) - - common._test_status.set_status.assert_any_call("MEMLEAK", "PASS") + common._test_status.set_status.assert_any_call( + "MEMLEAK", "PASS", comments="" + ) append_testlog.assert_not_called() From 789002130f2294cbd6d8861667f50e49fef99c45 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 13 Oct 2023 12:41:10 -0700 Subject: [PATCH 21/31] Renames methods --- CIME/SystemTests/system_tests_common.py | 16 +-- CIME/baselines.py | 32 +++--- CIME/bless_test_results.py | 18 ++-- CIME/tests/test_unit_baselines.py | 132 +++++++++++++----------- CIME/tests/test_unit_system_tests.py | 68 ++++++------ 5 files changed, 145 insertions(+), 121 deletions(-) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index 0fd90c5cfa0..7fca1010a60 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -28,10 +28,10 @@ from CIME.locked_files import LOCKED_DIR, lock_file, is_locked from CIME.baselines import ( get_latest_cpl_logs, - get_default_mem_usage, - compare_memory, - compare_throughput, - write_baseline, + _get_mem_usage, + perf_compare_memory_baseline, + perf_compare_throughput_baseline, + perf_write_baseline, load_coupler_customization, ) import CIME.build as build @@ -679,7 +679,7 @@ def _compare_memory(self): Compares current test memory usage to baseline. """ with self._test_status: - below_tolerance, comment = compare_memory(self._case) + below_tolerance, comment = perf_compare_memory_baseline(self._case) if below_tolerance is not None: append_testlog(comment, self._orig_caseroot) @@ -699,7 +699,7 @@ def _compare_throughput(self): Compares current test throughput to baseline. """ with self._test_status: - below_tolerance, comment = compare_throughput(self._case) + below_tolerance, comment = perf_compare_throughput_baseline(self._case) if below_tolerance is not None: append_testlog(comment, self._orig_caseroot) @@ -769,7 +769,7 @@ def _generate_baseline(self): preserve_meta=False, ) - write_baseline(self._case, basegen_dir, cpllog) + perf_write_baseline(self._case, basegen_dir, cpllog) def default_detect_memory_leak(case): @@ -780,7 +780,7 @@ def default_detect_memory_leak(case): for cpllog in latestcpllogs: try: - memlist = get_default_mem_usage(case, cpllog) + memlist = _get_mem_usage(case, cpllog) except RuntimeError: return False, "insufficient data for memleak test" diff --git a/CIME/baselines.py b/CIME/baselines.py index 1011bfc815b..a8c1aecd7ac 100644 --- a/CIME/baselines.py +++ b/CIME/baselines.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) -def compare_throughput(case, baseline_dir=None): +def perf_compare_throughput_baseline(case, baseline_dir=None): """ Compares model throughput. @@ -54,18 +54,18 @@ def compare_throughput(case, baseline_dir=None): ) try: - below_tolerance, comment = config.compare_baseline_throughput( + below_tolerance, comment = config.perf_compare_throughput_baseline( case, baseline, tolerance ) except AttributeError: - below_tolerance, comment = compare_baseline_throughput( + below_tolerance, comment = _perf_compare_throughput_baseline( case, baseline, tolerance ) return below_tolerance, comment -def compare_memory(case, baseline_dir=None): +def perf_compare_memory_baseline(case, baseline_dir=None): """ Compares model highwater memory usage. @@ -105,16 +105,18 @@ def compare_memory(case, baseline_dir=None): tolerance = 0.1 try: - below_tolerance, comments = config.compare_baseline_memory( + below_tolerance, comments = config.perf_compare_memory_baseline( case, baseline, tolerance ) except AttributeError: - below_tolerance, comments = compare_baseline_memory(case, baseline, tolerance) + below_tolerance, comments = _perf_compare_memory_baseline( + case, baseline, tolerance + ) return below_tolerance, comments -def write_baseline(case, basegen_dir, throughput=True, memory=True): +def perf_write_baseline(case, basegen_dir, throughput=True, memory=True): """ Writes the baseline performance files. @@ -194,7 +196,7 @@ def get_throughput(case, config): try: tput = config.get_throughput(case) except AttributeError: - tput = get_default_throughput(case) + tput = _get_throughput(case) if tput is None: raise RuntimeError("Could not get default throughput") from None @@ -225,7 +227,7 @@ def get_mem_usage(case, config): try: mem = config.get_mem_usage(case) except AttributeError: - mem = get_default_mem_usage(case) + mem = _get_mem_usage(case) if mem is None: raise RuntimeError("Could not get default memory usage") from None @@ -250,7 +252,7 @@ def write_baseline_file(baseline_file, value): fd.write(value) -def get_default_mem_usage(case, cpllog=None): +def _get_mem_usage(case, cpllog=None): """ Default function to retrieve memory usage from the coupler log. @@ -293,7 +295,7 @@ def get_default_mem_usage(case, cpllog=None): return memlist -def get_default_throughput(case): +def _get_throughput(case): """ Default function to retrieve throughput from the coupler log. @@ -431,7 +433,7 @@ def read_baseline_file(baseline_file): return lines -def compare_baseline_throughput(case, baseline, tolerance): +def _perf_compare_throughput_baseline(case, baseline, tolerance): """ Default throughput baseline comparison. @@ -453,7 +455,7 @@ def compare_baseline_throughput(case, baseline, tolerance): comment : str provides explanation from comparison. """ - current = get_default_throughput(case) + current = _get_throughput(case) try: # default baseline is stored as single float @@ -483,7 +485,7 @@ def compare_baseline_throughput(case, baseline, tolerance): return below_tolerance, comment -def compare_baseline_memory(case, baseline, tolerance): +def _perf_compare_memory_baseline(case, baseline, tolerance): """ Default memory usage baseline comparison. @@ -506,7 +508,7 @@ def compare_baseline_memory(case, baseline, tolerance): provides explanation from comparison. """ try: - current = get_default_mem_usage(case) + current = _get_mem_usage(case) except RuntimeError as e: return None, str(e) else: diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index d2a88d4b2c8..c2ba6fbb1c9 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -12,9 +12,9 @@ from CIME.case import Case from CIME.test_utils import get_test_status_files from CIME.baselines import ( - compare_throughput, - compare_memory, - write_baseline, + perf_compare_throughput_baseline, + perf_compare_memory_baseline, + perf_write_baseline, ) import os, time @@ -36,7 +36,9 @@ def bless_throughput( baseline_root, baseline_name, case.get_value("CASEBASEID") ) - below_threshold, comment = compare_throughput(case, baseline_dir=baseline_dir) + below_threshold, comment = perf_compare_throughput_baseline( + case, baseline_dir=baseline_dir + ) if below_threshold: logger.info("Diff appears to have been already resolved.") @@ -47,7 +49,7 @@ def bless_throughput( force or input("Update this diff (y/n)? ").upper() in ["Y", "YES"] ): try: - write_baseline(case, baseline_dir, memory=False) + perf_write_baseline(case, baseline_dir, memory=False) except Exception as e: success = False @@ -71,7 +73,9 @@ def bless_memory( baseline_root, baseline_name, case.get_value("CASEBASEID") ) - below_threshold, comment = compare_memory(case, baseline_dir=baseline_dir) + below_threshold, comment = perf_compare_memory_baseline( + case, baseline_dir=baseline_dir + ) if below_threshold: logger.info("Diff appears to have been already resolved.") @@ -82,7 +86,7 @@ def bless_memory( force or input("Update this diff (y/n)? ").upper() in ["Y", "YES"] ): try: - write_baseline(case, baseline_dir, throughput=False) + perf_write_baseline(case, baseline_dir, throughput=False) except Exception as e: success = False diff --git a/CIME/tests/test_unit_baselines.py b/CIME/tests/test_unit_baselines.py index a5af4c4a9a2..0b357e51253 100644 --- a/CIME/tests/test_unit_baselines.py +++ b/CIME/tests/test_unit_baselines.py @@ -28,9 +28,9 @@ def create_mock_case(tempdir, get_latest_cpl_logs=None): class TestUnitBaseline(unittest.TestCase): - @mock.patch("CIME.baselines.get_default_mem_usage") - def test_get_mem_usage_default_no_value(self, get_default_mem_usage): - get_default_mem_usage.return_value = None + @mock.patch("CIME.baselines._get_mem_usage") + def test_get_mem_usage_default_no_value(self, _get_mem_usage): + _get_mem_usage.return_value = None case = mock.MagicMock() @@ -41,9 +41,9 @@ def test_get_mem_usage_default_no_value(self, get_default_mem_usage): with self.assertRaises(RuntimeError): baselines.get_mem_usage(case, config) - @mock.patch("CIME.baselines.get_default_mem_usage") - def test_get_mem_usage_default(self, get_default_mem_usage): - get_default_mem_usage.return_value = [(1, 1000)] + @mock.patch("CIME.baselines._get_mem_usage") + def test_get_mem_usage_default(self, _get_mem_usage): + _get_mem_usage.return_value = [(1, 1000)] case = mock.MagicMock() @@ -66,9 +66,9 @@ def test_get_mem_usage(self): assert mem == "1000" - @mock.patch("CIME.baselines.get_default_throughput") - def test_get_throughput_default_no_value(self, get_default_throughput): - get_default_throughput.return_value = None + @mock.patch("CIME.baselines._get_throughput") + def test_get_throughput_default_no_value(self, _get_throughput): + _get_throughput.return_value = None case = mock.MagicMock() @@ -79,9 +79,9 @@ def test_get_throughput_default_no_value(self, get_default_throughput): with self.assertRaises(RuntimeError): baselines.get_throughput(case, config) - @mock.patch("CIME.baselines.get_default_throughput") - def test_get_throughput_default(self, get_default_throughput): - get_default_throughput.return_value = 100 + @mock.patch("CIME.baselines._get_throughput") + def test_get_throughput_default(self, _get_throughput): + _get_throughput.return_value = 100 case = mock.MagicMock() @@ -186,39 +186,37 @@ def test_write_baseline_file(self): @mock.patch("CIME.baselines.get_cpl_throughput") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_get_default_throughput(self, get_latest_cpl_logs, get_cpl_throughput): + def test__get_throughput(self, get_latest_cpl_logs, get_cpl_throughput): get_cpl_throughput.side_effect = FileNotFoundError() with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) - tput = baselines.get_default_throughput(case) + tput = baselines._get_throughput(case) assert tput == None @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_get_default_mem_usage_override( - self, get_latest_cpl_logs, get_cpl_mem_usage - ): + def test__get_mem_usage_override(self, get_latest_cpl_logs, get_cpl_mem_usage): get_cpl_mem_usage.side_effect = FileNotFoundError() with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) - mem = baselines.get_default_mem_usage(case, "/tmp/override") + mem = baselines._get_mem_usage(case, "/tmp/override") assert mem == None @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_get_default_mem_usage(self, get_latest_cpl_logs, get_cpl_mem_usage): + def test__get_mem_usage(self, get_latest_cpl_logs, get_cpl_mem_usage): get_cpl_mem_usage.side_effect = FileNotFoundError() with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) - mem = baselines.get_default_mem_usage(case) + mem = baselines._get_mem_usage(case) assert mem == None @@ -235,7 +233,7 @@ def test_write_baseline_skip( with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir) - baselines.write_baseline( + baselines.perf_write_baseline( case, baseline_root, False, @@ -259,7 +257,7 @@ def test_write_baseline_runtimeerror( with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir) - baselines.write_baseline(case, baseline_root) + baselines.perf_write_baseline(case, baseline_root) get_throughput.assert_called() get_mem_usage.assert_called() @@ -268,7 +266,9 @@ def test_write_baseline_runtimeerror( @mock.patch("CIME.baselines.write_baseline_file") @mock.patch("CIME.baselines.get_mem_usage") @mock.patch("CIME.baselines.get_throughput") - def test_write_baseline(self, get_throughput, get_mem_usage, write_baseline_file): + def test_perf_write_baseline( + self, get_throughput, get_mem_usage, write_baseline_file + ): get_throughput.return_value = "100" get_mem_usage.return_value = "1000" @@ -276,22 +276,22 @@ def test_write_baseline(self, get_throughput, get_mem_usage, write_baseline_file with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir) - baselines.write_baseline(case, baseline_root) + baselines.perf_write_baseline(case, baseline_root) get_throughput.assert_called() get_mem_usage.assert_called() write_baseline_file.assert_any_call(str(baseline_root / "cpl-tput.log"), "100") write_baseline_file.assert_any_call(str(baseline_root / "cpl-mem.log"), "1000") - @mock.patch("CIME.baselines.get_default_throughput") + @mock.patch("CIME.baselines._get_throughput") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_throughput_no_baseline_file( - self, get_latest_cpl_logs, read_baseline_file, get_default_throughput + def test_perf_compare_throughput_baseline_no_baseline_file( + self, get_latest_cpl_logs, read_baseline_file, _get_throughput ): read_baseline_file.side_effect = FileNotFoundError - get_default_throughput.return_value = 504 + _get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -303,20 +303,22 @@ def test_compare_throughput_no_baseline_file( 0.05, ) - (below_tolerance, comment) = baselines.compare_throughput(case) + (below_tolerance, comment) = baselines.perf_compare_throughput_baseline( + case + ) assert below_tolerance is None assert comment == "Could not read baseline throughput file: " - @mock.patch("CIME.baselines.get_default_throughput") + @mock.patch("CIME.baselines._get_throughput") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_throughput_no_baseline( - self, get_latest_cpl_logs, read_baseline_file, get_default_throughput + def test_perf_compare_throughput_baseline_no_baseline( + self, get_latest_cpl_logs, read_baseline_file, _get_throughput ): read_baseline_file.return_value = [] - get_default_throughput.return_value = 504 + _get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -330,7 +332,9 @@ def test_compare_throughput_no_baseline( 0.05, ) - (below_tolerance, comment) = baselines.compare_throughput(case) + (below_tolerance, comment) = baselines.perf_compare_throughput_baseline( + case + ) assert below_tolerance is None assert ( @@ -338,17 +342,17 @@ def test_compare_throughput_no_baseline( == "Could not compare throughput to baseline, as basline had no value." ) - @mock.patch("CIME.baselines.get_default_throughput") + @mock.patch("CIME.baselines._get_throughput") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_throughput_no_tolerance( - self, get_latest_cpl_logs, read_baseline_file, get_default_throughput + def test_perf_compare_throughput_baseline_no_tolerance( + self, get_latest_cpl_logs, read_baseline_file, _get_throughput ): read_baseline_file.return_value = [ "500", ] - get_default_throughput.return_value = 504 + _get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -362,7 +366,9 @@ def test_compare_throughput_no_tolerance( None, ) - (below_tolerance, comment) = baselines.compare_throughput(case) + (below_tolerance, comment) = baselines.perf_compare_throughput_baseline( + case + ) assert below_tolerance assert ( @@ -370,15 +376,15 @@ def test_compare_throughput_no_tolerance( == "TPUTCOMP: Computation time changed by -0.80% relative to baseline" ) - @mock.patch("CIME.baselines.get_default_throughput") + @mock.patch("CIME.baselines._get_throughput") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_throughput_above_threshold( - self, get_latest_cpl_logs, read_baseline_file, get_default_throughput + def test_perf_compare_throughput_baseline_above_threshold( + self, get_latest_cpl_logs, read_baseline_file, _get_throughput ): read_baseline_file.return_value = ["1000"] - get_default_throughput.return_value = 504 + _get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -392,22 +398,24 @@ def test_compare_throughput_above_threshold( 0.05, ) - (below_tolerance, comment) = baselines.compare_throughput(case) + (below_tolerance, comment) = baselines.perf_compare_throughput_baseline( + case + ) assert not below_tolerance assert ( comment == "Error: TPUTCOMP: Computation time increase > 5% from baseline" ) - @mock.patch("CIME.baselines.get_default_throughput") + @mock.patch("CIME.baselines._get_throughput") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_throughput( - self, get_latest_cpl_logs, read_baseline_file, get_default_throughput + def test_perf_compare_throughput_baseline( + self, get_latest_cpl_logs, read_baseline_file, _get_throughput ): read_baseline_file.return_value = ["500"] - get_default_throughput.return_value = 504 + _get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -421,7 +429,9 @@ def test_compare_throughput( 0.05, ) - (below_tolerance, comment) = baselines.compare_throughput(case) + (below_tolerance, comment) = baselines.perf_compare_throughput_baseline( + case + ) assert below_tolerance assert ( @@ -432,7 +442,7 @@ def test_compare_throughput( @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_memory_no_baseline( + def test_perf_compare_memory_baseline_no_baseline( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): read_baseline_file.return_value = [] @@ -456,7 +466,7 @@ def test_compare_memory_no_baseline( 0.05, ) - (below_tolerance, comment) = baselines.compare_memory(case) + (below_tolerance, comment) = baselines.perf_compare_memory_baseline(case) assert below_tolerance assert ( @@ -467,7 +477,7 @@ def test_compare_memory_no_baseline( @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_memory_not_enough_samples( + def test_perf_compare_memory_baseline_not_enough_samples( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): read_baseline_file.return_value = ["1000.0"] @@ -487,7 +497,7 @@ def test_compare_memory_not_enough_samples( 0.05, ) - (below_tolerance, comment) = baselines.compare_memory(case) + (below_tolerance, comment) = baselines.perf_compare_memory_baseline(case) assert below_tolerance is None assert comment == "Found 2 memory usage samples, need atleast 4" @@ -495,7 +505,7 @@ def test_compare_memory_not_enough_samples( @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_memory_no_baseline_file( + def test_perf_compare_memory_baseline_no_baseline_file( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): read_baseline_file.side_effect = FileNotFoundError @@ -517,7 +527,7 @@ def test_compare_memory_no_baseline_file( 0.05, ) - (below_tolerance, comment) = baselines.compare_memory(case) + (below_tolerance, comment) = baselines.perf_compare_memory_baseline(case) assert below_tolerance is None assert comment == "Could not read baseline memory usage: " @@ -525,7 +535,7 @@ def test_compare_memory_no_baseline_file( @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_memory_no_tolerance( + def test_perf_compare_memory_baseline_no_tolerance( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): read_baseline_file.return_value = ["1000.0"] @@ -549,7 +559,7 @@ def test_compare_memory_no_tolerance( None, ) - (below_tolerance, comment) = baselines.compare_memory(case) + (below_tolerance, comment) = baselines.perf_compare_memory_baseline(case) assert below_tolerance assert ( @@ -560,7 +570,7 @@ def test_compare_memory_no_tolerance( @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_memory_above_threshold( + def test_perf_compare_memory_baseline_above_threshold( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): read_baseline_file.return_value = ["1000.0"] @@ -584,7 +594,7 @@ def test_compare_memory_above_threshold( 0.05, ) - (below_tolerance, comment) = baselines.compare_memory(case) + (below_tolerance, comment) = baselines.perf_compare_memory_baseline(case) assert not below_tolerance assert ( @@ -595,7 +605,7 @@ def test_compare_memory_above_threshold( @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test_compare_memory( + def test_perf_compare_memory_baseline( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): read_baseline_file.return_value = ["1000.0"] @@ -619,7 +629,7 @@ def test_compare_memory( 0.05, ) - (below_tolerance, comment) = baselines.compare_memory(case) + (below_tolerance, comment) = baselines.perf_compare_memory_baseline(case) assert below_tolerance assert ( diff --git a/CIME/tests/test_unit_system_tests.py b/CIME/tests/test_unit_system_tests.py index 2f9f7d9918b..f61fffee7cf 100644 --- a/CIME/tests/test_unit_system_tests.py +++ b/CIME/tests/test_unit_system_tests.py @@ -69,12 +69,12 @@ def create_mock_case(tempdir, idx=None, cpllog_data=None): class TestUnitSystemTests(unittest.TestCase): @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - @mock.patch("CIME.SystemTests.system_tests_common.get_default_mem_usage") + @mock.patch("CIME.SystemTests.system_tests_common._get_mem_usage") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak_runtime_error( self, get_latest_cpl_logs, - get_default_mem_usage, + _get_mem_usage, append_testlog, load_coupler_customization, ): @@ -82,7 +82,7 @@ def test_check_for_memleak_runtime_error( AttributeError ) - get_default_mem_usage.side_effect = RuntimeError + _get_mem_usage.side_effect = RuntimeError with tempfile.TemporaryDirectory() as tempdir: caseroot = Path(tempdir) / "caseroot" @@ -119,12 +119,12 @@ def test_check_for_memleak_runtime_error( @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - @mock.patch("CIME.SystemTests.system_tests_common.get_default_mem_usage") + @mock.patch("CIME.SystemTests.system_tests_common._get_mem_usage") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak_not_enough_samples( self, get_latest_cpl_logs, - get_default_mem_usage, + _get_mem_usage, append_testlog, load_coupler_customization, ): @@ -132,7 +132,7 @@ def test_check_for_memleak_not_enough_samples( AttributeError ) - get_default_mem_usage.return_value = [ + _get_mem_usage.return_value = [ (1, 1000.0), (2, 0), ] @@ -172,12 +172,12 @@ def test_check_for_memleak_not_enough_samples( @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - @mock.patch("CIME.SystemTests.system_tests_common.get_default_mem_usage") + @mock.patch("CIME.SystemTests.system_tests_common._get_mem_usage") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak_found( self, get_latest_cpl_logs, - get_default_mem_usage, + _get_mem_usage, append_testlog, load_coupler_customization, ): @@ -185,7 +185,7 @@ def test_check_for_memleak_found( AttributeError ) - get_default_mem_usage.return_value = [ + _get_mem_usage.return_value = [ (1, 1000.0), (2, 2000.0), (3, 3000.0), @@ -229,12 +229,12 @@ def test_check_for_memleak_found( @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - @mock.patch("CIME.SystemTests.system_tests_common.get_default_mem_usage") + @mock.patch("CIME.SystemTests.system_tests_common._get_mem_usage") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak( self, get_latest_cpl_logs, - get_default_mem_usage, + _get_mem_usage, append_testlog, load_coupler_customization, ): @@ -242,7 +242,7 @@ def test_check_for_memleak( AttributeError ) - get_default_mem_usage.return_value = [ + _get_mem_usage.return_value = [ (1, 3040.0), (2, 3002.0), (3, 3030.0), @@ -282,10 +282,10 @@ def test_check_for_memleak( append_testlog.assert_not_called() - @mock.patch("CIME.SystemTests.system_tests_common.compare_throughput") + @mock.patch("CIME.SystemTests.system_tests_common.perf_compare_throughput_baseline") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - def test_compare_throughput(self, append_testlog, compare_throughput): - compare_throughput.return_value = ( + def test_compare_throughput(self, append_testlog, perf_compare_throughput_baseline): + perf_compare_throughput_baseline.return_value = ( True, "TPUTCOMP: Computation time changed by 2.00% relative to baseline", ) @@ -312,10 +312,12 @@ def test_compare_throughput(self, append_testlog, compare_throughput): str(caseroot), ) - @mock.patch("CIME.SystemTests.system_tests_common.compare_throughput") + @mock.patch("CIME.SystemTests.system_tests_common.perf_compare_throughput_baseline") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - def test_compare_throughput_error_diff(self, append_testlog, compare_throughput): - compare_throughput.return_value = (None, "Error diff value") + def test_compare_throughput_error_diff( + self, append_testlog, perf_compare_throughput_baseline + ): + perf_compare_throughput_baseline.return_value = (None, "Error diff value") with tempfile.TemporaryDirectory() as tempdir: caseroot = Path(tempdir) / "caseroot" @@ -336,10 +338,12 @@ def test_compare_throughput_error_diff(self, append_testlog, compare_throughput) append_testlog.assert_not_called() - @mock.patch("CIME.SystemTests.system_tests_common.compare_throughput") + @mock.patch("CIME.SystemTests.system_tests_common.perf_compare_throughput_baseline") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - def test_compare_throughput_fail(self, append_testlog, compare_throughput): - compare_throughput.return_value = ( + def test_compare_throughput_fail( + self, append_testlog, perf_compare_throughput_baseline + ): + perf_compare_throughput_baseline.return_value = ( False, "Error: TPUTCOMP: Computation time increase > 5% from baseline", ) @@ -366,10 +370,10 @@ def test_compare_throughput_fail(self, append_testlog, compare_throughput): str(caseroot), ) - @mock.patch("CIME.SystemTests.system_tests_common.compare_memory") + @mock.patch("CIME.SystemTests.system_tests_common.perf_compare_memory_baseline") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - def test_compare_memory(self, append_testlog, compare_memory): - compare_memory.return_value = ( + def test_compare_memory(self, append_testlog, perf_compare_memory_baseline): + perf_compare_memory_baseline.return_value = ( True, "MEMCOMP: Memory usage highwater has changed by 2.00% relative to baseline", ) @@ -396,10 +400,12 @@ def test_compare_memory(self, append_testlog, compare_memory): str(caseroot), ) - @mock.patch("CIME.SystemTests.system_tests_common.compare_memory") + @mock.patch("CIME.SystemTests.system_tests_common.perf_compare_memory_baseline") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - def test_compare_memory_erorr_diff(self, append_testlog, compare_memory): - compare_memory.return_value = (None, "Error diff value") + def test_compare_memory_erorr_diff( + self, append_testlog, perf_compare_memory_baseline + ): + perf_compare_memory_baseline.return_value = (None, "Error diff value") with tempfile.TemporaryDirectory() as tempdir: caseroot = Path(tempdir) / "caseroot" @@ -420,10 +426,12 @@ def test_compare_memory_erorr_diff(self, append_testlog, compare_memory): append_testlog.assert_not_called() - @mock.patch("CIME.SystemTests.system_tests_common.compare_memory") + @mock.patch("CIME.SystemTests.system_tests_common.perf_compare_memory_baseline") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - def test_compare_memory_erorr_fail(self, append_testlog, compare_memory): - compare_memory.return_value = ( + def test_compare_memory_erorr_fail( + self, append_testlog, perf_compare_memory_baseline + ): + perf_compare_memory_baseline.return_value = ( False, "Error: Memory usage increase >5% from baseline's 1000.000000 to 1002.000000", ) From d76f407081b14599c4a5d28c8ec4c9b1d29f8a7f Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 13 Oct 2023 12:51:07 -0700 Subject: [PATCH 22/31] Fixes more naming --- CIME/SystemTests/system_tests_common.py | 4 +- CIME/baselines.py | 24 ++--- CIME/tests/test_unit_baselines.py | 136 ++++++++++++------------ CIME/tests/test_unit_system_tests.py | 24 ++--- 4 files changed, 94 insertions(+), 94 deletions(-) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index 7fca1010a60..7edfb5d30ac 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -28,7 +28,7 @@ from CIME.locked_files import LOCKED_DIR, lock_file, is_locked from CIME.baselines import ( get_latest_cpl_logs, - _get_mem_usage, + _perf_get_memory, perf_compare_memory_baseline, perf_compare_throughput_baseline, perf_write_baseline, @@ -780,7 +780,7 @@ def default_detect_memory_leak(case): for cpllog in latestcpllogs: try: - memlist = _get_mem_usage(case, cpllog) + memlist = _perf_get_memory(case, cpllog) except RuntimeError: return False, "insufficient data for memleak test" diff --git a/CIME/baselines.py b/CIME/baselines.py index a8c1aecd7ac..40a11a13db9 100644 --- a/CIME/baselines.py +++ b/CIME/baselines.py @@ -135,7 +135,7 @@ def perf_write_baseline(case, basegen_dir, throughput=True, memory=True): if throughput: try: - tput = get_throughput(case, config) + tput = perf_get_throughput(case, config) except RuntimeError: pass else: @@ -145,7 +145,7 @@ def perf_write_baseline(case, basegen_dir, throughput=True, memory=True): if memory: try: - mem = get_mem_usage(case, config) + mem = perf_get_memory(case, config) except RuntimeError: pass else: @@ -175,7 +175,7 @@ def load_coupler_customization(case): return Config.load(cpl_customize) -def get_throughput(case, config): +def perf_get_throughput(case, config): """ Gets the model throughput. @@ -194,9 +194,9 @@ def get_throughput(case, config): Model throughput. """ try: - tput = config.get_throughput(case) + tput = config.perf_get_throughput(case) except AttributeError: - tput = _get_throughput(case) + tput = _perf_get_throughput(case) if tput is None: raise RuntimeError("Could not get default throughput") from None @@ -206,7 +206,7 @@ def get_throughput(case, config): return tput -def get_mem_usage(case, config): +def perf_get_memory(case, config): """ Gets the model memory usage. @@ -225,9 +225,9 @@ def get_mem_usage(case, config): Model memory usage. """ try: - mem = config.get_mem_usage(case) + mem = config.perf_get_memory(case) except AttributeError: - mem = _get_mem_usage(case) + mem = _perf_get_memory(case) if mem is None: raise RuntimeError("Could not get default memory usage") from None @@ -252,7 +252,7 @@ def write_baseline_file(baseline_file, value): fd.write(value) -def _get_mem_usage(case, cpllog=None): +def _perf_get_memory(case, cpllog=None): """ Default function to retrieve memory usage from the coupler log. @@ -295,7 +295,7 @@ def _get_mem_usage(case, cpllog=None): return memlist -def _get_throughput(case): +def _perf_get_throughput(case): """ Default function to retrieve throughput from the coupler log. @@ -455,7 +455,7 @@ def _perf_compare_throughput_baseline(case, baseline, tolerance): comment : str provides explanation from comparison. """ - current = _get_throughput(case) + current = _perf_get_throughput(case) try: # default baseline is stored as single float @@ -508,7 +508,7 @@ def _perf_compare_memory_baseline(case, baseline, tolerance): provides explanation from comparison. """ try: - current = _get_mem_usage(case) + current = _perf_get_memory(case) except RuntimeError as e: return None, str(e) else: diff --git a/CIME/tests/test_unit_baselines.py b/CIME/tests/test_unit_baselines.py index 0b357e51253..dac5bc9ef10 100644 --- a/CIME/tests/test_unit_baselines.py +++ b/CIME/tests/test_unit_baselines.py @@ -28,79 +28,79 @@ def create_mock_case(tempdir, get_latest_cpl_logs=None): class TestUnitBaseline(unittest.TestCase): - @mock.patch("CIME.baselines._get_mem_usage") - def test_get_mem_usage_default_no_value(self, _get_mem_usage): - _get_mem_usage.return_value = None + @mock.patch("CIME.baselines._perf_get_memory") + def test_perf_get_memory_default_no_value(self, _perf_get_memory): + _perf_get_memory.return_value = None case = mock.MagicMock() config = mock.MagicMock() - config.get_mem_usage.side_effect = AttributeError + config.perf_get_memory.side_effect = AttributeError with self.assertRaises(RuntimeError): - baselines.get_mem_usage(case, config) + baselines.perf_get_memory(case, config) - @mock.patch("CIME.baselines._get_mem_usage") - def test_get_mem_usage_default(self, _get_mem_usage): - _get_mem_usage.return_value = [(1, 1000)] + @mock.patch("CIME.baselines._perf_get_memory") + def test_perf_get_memory_default(self, _perf_get_memory): + _perf_get_memory.return_value = [(1, 1000)] case = mock.MagicMock() config = mock.MagicMock() - config.get_mem_usage.side_effect = AttributeError + config.perf_get_memory.side_effect = AttributeError - mem = baselines.get_mem_usage(case, config) + mem = baselines.perf_get_memory(case, config) assert mem == "1000" - def test_get_mem_usage(self): + def test_perf_get_memory(self): case = mock.MagicMock() config = mock.MagicMock() - config.get_mem_usage.return_value = "1000" + config.perf_get_memory.return_value = "1000" - mem = baselines.get_mem_usage(case, config) + mem = baselines.perf_get_memory(case, config) assert mem == "1000" - @mock.patch("CIME.baselines._get_throughput") - def test_get_throughput_default_no_value(self, _get_throughput): - _get_throughput.return_value = None + @mock.patch("CIME.baselines._perf_get_throughput") + def test_perf_get_throughput_default_no_value(self, _perf_get_throughput): + _perf_get_throughput.return_value = None case = mock.MagicMock() config = mock.MagicMock() - config.get_throughput.side_effect = AttributeError + config.perf_get_throughput.side_effect = AttributeError with self.assertRaises(RuntimeError): - baselines.get_throughput(case, config) + baselines.perf_get_throughput(case, config) - @mock.patch("CIME.baselines._get_throughput") - def test_get_throughput_default(self, _get_throughput): - _get_throughput.return_value = 100 + @mock.patch("CIME.baselines._perf_get_throughput") + def test_perf_get_throughput_default(self, _perf_get_throughput): + _perf_get_throughput.return_value = 100 case = mock.MagicMock() config = mock.MagicMock() - config.get_throughput.side_effect = AttributeError + config.perf_get_throughput.side_effect = AttributeError - tput = baselines.get_throughput(case, config) + tput = baselines.perf_get_throughput(case, config) assert tput == "100" - def test_get_throughput(self): + def test_perf_get_throughput(self): case = mock.MagicMock() config = mock.MagicMock() - config.get_throughput.return_value = "100" + config.perf_get_throughput.return_value = "100" - tput = baselines.get_throughput(case, config) + tput = baselines.perf_get_throughput(case, config) assert tput == "100" @@ -186,49 +186,49 @@ def test_write_baseline_file(self): @mock.patch("CIME.baselines.get_cpl_throughput") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test__get_throughput(self, get_latest_cpl_logs, get_cpl_throughput): + def test__perf_get_throughput(self, get_latest_cpl_logs, get_cpl_throughput): get_cpl_throughput.side_effect = FileNotFoundError() with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) - tput = baselines._get_throughput(case) + tput = baselines._perf_get_throughput(case) assert tput == None @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test__get_mem_usage_override(self, get_latest_cpl_logs, get_cpl_mem_usage): + def test__perf_get_memory_override(self, get_latest_cpl_logs, get_cpl_mem_usage): get_cpl_mem_usage.side_effect = FileNotFoundError() with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) - mem = baselines._get_mem_usage(case, "/tmp/override") + mem = baselines._perf_get_memory(case, "/tmp/override") assert mem == None @mock.patch("CIME.baselines.get_cpl_mem_usage") @mock.patch("CIME.baselines.get_latest_cpl_logs") - def test__get_mem_usage(self, get_latest_cpl_logs, get_cpl_mem_usage): + def test__perf_get_memory(self, get_latest_cpl_logs, get_cpl_mem_usage): get_cpl_mem_usage.side_effect = FileNotFoundError() with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) - mem = baselines._get_mem_usage(case) + mem = baselines._perf_get_memory(case) assert mem == None @mock.patch("CIME.baselines.write_baseline_file") - @mock.patch("CIME.baselines.get_mem_usage") - @mock.patch("CIME.baselines.get_throughput") + @mock.patch("CIME.baselines.perf_get_memory") + @mock.patch("CIME.baselines.perf_get_throughput") def test_write_baseline_skip( - self, get_throughput, get_mem_usage, write_baseline_file + self, perf_get_throughput, perf_get_memory, write_baseline_file ): - get_throughput.return_value = "100" + perf_get_throughput.return_value = "100" - get_mem_usage.return_value = "1000" + perf_get_memory.return_value = "1000" with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir) @@ -240,58 +240,58 @@ def test_write_baseline_skip( False, ) - get_throughput.assert_not_called() - get_mem_usage.assert_not_called() + perf_get_throughput.assert_not_called() + perf_get_memory.assert_not_called() write_baseline_file.assert_not_called() @mock.patch("CIME.baselines.write_baseline_file") - @mock.patch("CIME.baselines.get_mem_usage") - @mock.patch("CIME.baselines.get_throughput") + @mock.patch("CIME.baselines.perf_get_memory") + @mock.patch("CIME.baselines.perf_get_throughput") def test_write_baseline_runtimeerror( - self, get_throughput, get_mem_usage, write_baseline_file + self, perf_get_throughput, perf_get_memory, write_baseline_file ): - get_throughput.side_effect = RuntimeError + perf_get_throughput.side_effect = RuntimeError - get_mem_usage.side_effect = RuntimeError + perf_get_memory.side_effect = RuntimeError with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir) baselines.perf_write_baseline(case, baseline_root) - get_throughput.assert_called() - get_mem_usage.assert_called() + perf_get_throughput.assert_called() + perf_get_memory.assert_called() write_baseline_file.assert_not_called() @mock.patch("CIME.baselines.write_baseline_file") - @mock.patch("CIME.baselines.get_mem_usage") - @mock.patch("CIME.baselines.get_throughput") + @mock.patch("CIME.baselines.perf_get_memory") + @mock.patch("CIME.baselines.perf_get_throughput") def test_perf_write_baseline( - self, get_throughput, get_mem_usage, write_baseline_file + self, perf_get_throughput, perf_get_memory, write_baseline_file ): - get_throughput.return_value = "100" + perf_get_throughput.return_value = "100" - get_mem_usage.return_value = "1000" + perf_get_memory.return_value = "1000" with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir) baselines.perf_write_baseline(case, baseline_root) - get_throughput.assert_called() - get_mem_usage.assert_called() + perf_get_throughput.assert_called() + perf_get_memory.assert_called() write_baseline_file.assert_any_call(str(baseline_root / "cpl-tput.log"), "100") write_baseline_file.assert_any_call(str(baseline_root / "cpl-mem.log"), "1000") - @mock.patch("CIME.baselines._get_throughput") + @mock.patch("CIME.baselines._perf_get_throughput") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_perf_compare_throughput_baseline_no_baseline_file( - self, get_latest_cpl_logs, read_baseline_file, _get_throughput + self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): read_baseline_file.side_effect = FileNotFoundError - _get_throughput.return_value = 504 + _perf_get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -310,15 +310,15 @@ def test_perf_compare_throughput_baseline_no_baseline_file( assert below_tolerance is None assert comment == "Could not read baseline throughput file: " - @mock.patch("CIME.baselines._get_throughput") + @mock.patch("CIME.baselines._perf_get_throughput") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_perf_compare_throughput_baseline_no_baseline( - self, get_latest_cpl_logs, read_baseline_file, _get_throughput + self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): read_baseline_file.return_value = [] - _get_throughput.return_value = 504 + _perf_get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -342,17 +342,17 @@ def test_perf_compare_throughput_baseline_no_baseline( == "Could not compare throughput to baseline, as basline had no value." ) - @mock.patch("CIME.baselines._get_throughput") + @mock.patch("CIME.baselines._perf_get_throughput") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_perf_compare_throughput_baseline_no_tolerance( - self, get_latest_cpl_logs, read_baseline_file, _get_throughput + self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): read_baseline_file.return_value = [ "500", ] - _get_throughput.return_value = 504 + _perf_get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -376,15 +376,15 @@ def test_perf_compare_throughput_baseline_no_tolerance( == "TPUTCOMP: Computation time changed by -0.80% relative to baseline" ) - @mock.patch("CIME.baselines._get_throughput") + @mock.patch("CIME.baselines._perf_get_throughput") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_perf_compare_throughput_baseline_above_threshold( - self, get_latest_cpl_logs, read_baseline_file, _get_throughput + self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): read_baseline_file.return_value = ["1000"] - _get_throughput.return_value = 504 + _perf_get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -407,15 +407,15 @@ def test_perf_compare_throughput_baseline_above_threshold( comment == "Error: TPUTCOMP: Computation time increase > 5% from baseline" ) - @mock.patch("CIME.baselines._get_throughput") + @mock.patch("CIME.baselines._perf_get_throughput") @mock.patch("CIME.baselines.read_baseline_file") @mock.patch("CIME.baselines.get_latest_cpl_logs") def test_perf_compare_throughput_baseline( - self, get_latest_cpl_logs, read_baseline_file, _get_throughput + self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): read_baseline_file.return_value = ["500"] - _get_throughput.return_value = 504 + _perf_get_throughput.return_value = 504 with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) diff --git a/CIME/tests/test_unit_system_tests.py b/CIME/tests/test_unit_system_tests.py index f61fffee7cf..4a206fb454e 100644 --- a/CIME/tests/test_unit_system_tests.py +++ b/CIME/tests/test_unit_system_tests.py @@ -69,12 +69,12 @@ def create_mock_case(tempdir, idx=None, cpllog_data=None): class TestUnitSystemTests(unittest.TestCase): @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - @mock.patch("CIME.SystemTests.system_tests_common._get_mem_usage") + @mock.patch("CIME.SystemTests.system_tests_common._perf_get_memory") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak_runtime_error( self, get_latest_cpl_logs, - _get_mem_usage, + _perf_get_memory, append_testlog, load_coupler_customization, ): @@ -82,7 +82,7 @@ def test_check_for_memleak_runtime_error( AttributeError ) - _get_mem_usage.side_effect = RuntimeError + _perf_get_memory.side_effect = RuntimeError with tempfile.TemporaryDirectory() as tempdir: caseroot = Path(tempdir) / "caseroot" @@ -119,12 +119,12 @@ def test_check_for_memleak_runtime_error( @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - @mock.patch("CIME.SystemTests.system_tests_common._get_mem_usage") + @mock.patch("CIME.SystemTests.system_tests_common._perf_get_memory") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak_not_enough_samples( self, get_latest_cpl_logs, - _get_mem_usage, + _perf_get_memory, append_testlog, load_coupler_customization, ): @@ -132,7 +132,7 @@ def test_check_for_memleak_not_enough_samples( AttributeError ) - _get_mem_usage.return_value = [ + _perf_get_memory.return_value = [ (1, 1000.0), (2, 0), ] @@ -172,12 +172,12 @@ def test_check_for_memleak_not_enough_samples( @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - @mock.patch("CIME.SystemTests.system_tests_common._get_mem_usage") + @mock.patch("CIME.SystemTests.system_tests_common._perf_get_memory") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak_found( self, get_latest_cpl_logs, - _get_mem_usage, + _perf_get_memory, append_testlog, load_coupler_customization, ): @@ -185,7 +185,7 @@ def test_check_for_memleak_found( AttributeError ) - _get_mem_usage.return_value = [ + _perf_get_memory.return_value = [ (1, 1000.0), (2, 2000.0), (3, 3000.0), @@ -229,12 +229,12 @@ def test_check_for_memleak_found( @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - @mock.patch("CIME.SystemTests.system_tests_common._get_mem_usage") + @mock.patch("CIME.SystemTests.system_tests_common._perf_get_memory") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak( self, get_latest_cpl_logs, - _get_mem_usage, + _perf_get_memory, append_testlog, load_coupler_customization, ): @@ -242,7 +242,7 @@ def test_check_for_memleak( AttributeError ) - _get_mem_usage.return_value = [ + _perf_get_memory.return_value = [ (1, 3040.0), (2, 3002.0), (3, 3030.0), From 13e0e8b65d9281c8ee4400484fda03696dae2348 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 13 Oct 2023 13:10:58 -0700 Subject: [PATCH 23/31] Fixes striping newlines when reading baseline files --- CIME/baselines.py | 2 +- CIME/tests/test_unit_baselines.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CIME/baselines.py b/CIME/baselines.py index 40a11a13db9..2a3fe7369c6 100644 --- a/CIME/baselines.py +++ b/CIME/baselines.py @@ -428,7 +428,7 @@ def read_baseline_file(baseline_file): Value stored in baseline file without comments. """ with open(baseline_file) as fd: - lines = [x for x in fd.readlines() if not x.startswith("#")] + lines = [x.strip() for x in fd.readlines() if not x.startswith("#")] return lines diff --git a/CIME/tests/test_unit_baselines.py b/CIME/tests/test_unit_baselines.py index dac5bc9ef10..8382339bdc7 100644 --- a/CIME/tests/test_unit_baselines.py +++ b/CIME/tests/test_unit_baselines.py @@ -159,7 +159,7 @@ def test_read_baseline_file_multi_line(self): baseline = baselines.read_baseline_file("/tmp/cpl-mem.log") mock_file.assert_called_with("/tmp/cpl-mem.log") - assert baseline == ["1000.0\n", "2000.0\n"] + assert baseline == ["1000.0", "2000.0"] def test_read_baseline_file_content(self): with mock.patch( From 7780b1fee43cb1892067bd917b2e786192f03d45 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 13 Oct 2023 13:11:42 -0700 Subject: [PATCH 24/31] Fixes passing correct paths to memory/throughput bless functions --- CIME/bless_test_results.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index c2ba6fbb1c9..5802d685b71 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -397,8 +397,8 @@ def bless_test_results( success, reason = bless_throughput( case, test_name, - baseline_root, - baseline_name, + baseline_root_resolved, + baseline_name_resolved, report_only, force, ) @@ -410,8 +410,8 @@ def bless_test_results( success, reason = bless_memory( case, test_name, - baseline_root, - baseline_name, + baseline_root_resolved, + baseline_name_resolved, report_only, force, ) From 52ca0a1c791e14c3756d6e6d32f52628a37f2f72 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 13 Oct 2023 13:18:57 -0700 Subject: [PATCH 25/31] Renames memory leak functions --- CIME/SystemTests/system_tests_common.py | 6 +++--- CIME/tests/test_unit_system_tests.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index 7edfb5d30ac..e63c4c6d502 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -636,9 +636,9 @@ def _check_for_memleak(self): with self._test_status: try: - memleak, comment = config.detect_memory_leak(self._case) + memleak, comment = config.perf_check_for_memory_leak(self._case) except AttributeError: - memleak, comment = default_detect_memory_leak(self._case) + memleak, comment = perf_check_for_memory_leak(self._case) if memleak: append_testlog(comment, self._orig_caseroot) @@ -772,7 +772,7 @@ def _generate_baseline(self): perf_write_baseline(self._case, basegen_dir, cpllog) -def default_detect_memory_leak(case): +def perf_check_for_memory_leak(case): leak = False comment = "" diff --git a/CIME/tests/test_unit_system_tests.py b/CIME/tests/test_unit_system_tests.py index 4a206fb454e..99486178f36 100644 --- a/CIME/tests/test_unit_system_tests.py +++ b/CIME/tests/test_unit_system_tests.py @@ -78,7 +78,7 @@ def test_check_for_memleak_runtime_error( append_testlog, load_coupler_customization, ): - load_coupler_customization.return_value.detect_memory_leak.side_effect = ( + load_coupler_customization.return_value.perf_check_for_memory_leak.side_effect = ( AttributeError ) @@ -128,7 +128,7 @@ def test_check_for_memleak_not_enough_samples( append_testlog, load_coupler_customization, ): - load_coupler_customization.return_value.detect_memory_leak.side_effect = ( + load_coupler_customization.return_value.perf_check_for_memory_leak.side_effect = ( AttributeError ) @@ -181,7 +181,7 @@ def test_check_for_memleak_found( append_testlog, load_coupler_customization, ): - load_coupler_customization.return_value.detect_memory_leak.side_effect = ( + load_coupler_customization.return_value.perf_check_for_memory_leak.side_effect = ( AttributeError ) @@ -238,7 +238,7 @@ def test_check_for_memleak( append_testlog, load_coupler_customization, ): - load_coupler_customization.return_value.detect_memory_leak.side_effect = ( + load_coupler_customization.return_value.perf_check_for_memory_leak.side_effect = ( AttributeError ) From 69f7e367bc8e20e170bcce3b5f65df2f7082e4f0 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 13 Oct 2023 13:36:39 -0700 Subject: [PATCH 26/31] Fixes black formatting --- CIME/SystemTests/system_tests_common.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index e63c4c6d502..233de1e2660 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -634,11 +634,18 @@ def _check_for_memleak(self): """ config = load_coupler_customization(self._case) + # default to 0.1 + tolerance = self._case.get_value("TEST_MEMLEAK_TOLERANCE") or 0.1 + + expect(tolerance > 0.0, "Bad value for memleak tolerance in test") + with self._test_status: try: - memleak, comment = config.perf_check_for_memory_leak(self._case) + memleak, comment = config.perf_check_for_memory_leak( + self._case, tolerance + ) except AttributeError: - memleak, comment = perf_check_for_memory_leak(self._case) + memleak, comment = perf_check_for_memory_leak(self._case, tolerance) if memleak: append_testlog(comment, self._orig_caseroot) @@ -772,7 +779,7 @@ def _generate_baseline(self): perf_write_baseline(self._case, basegen_dir, cpllog) -def perf_check_for_memory_leak(case): +def perf_check_for_memory_leak(case, tolerance): leak = False comment = "" @@ -791,11 +798,6 @@ def perf_check_for_memory_leak(case): memdiff = -1 if originalmem <= 0 else (finalmem - originalmem) / originalmem - # default to 0.1 - tolerance = case.get_value("TEST_MEMLEAK_TOLERANCE") or 0.1 - - expect(tolerance > 0.0, "Bad value for memleak tolerance in test") - if memdiff < 0: leak = False comment = "data for memleak test is insufficient" From ccb3b4ea59e854fb26d31499492f170bd9766d02 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 13 Oct 2023 15:51:38 -0700 Subject: [PATCH 27/31] Fixes baseline module organization --- CIME/SystemTests/system_tests_common.py | 2 +- CIME/baselines/__init__.py | 0 .../performance.py} | 0 CIME/bless_test_results.py | 2 +- ....py => test_unit_baselines_performance.py} | 178 +++++++++--------- 5 files changed, 91 insertions(+), 91 deletions(-) create mode 100644 CIME/baselines/__init__.py rename CIME/{baselines.py => baselines/performance.py} (100%) rename CIME/tests/{test_unit_baselines.py => test_unit_baselines_performance.py} (74%) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index 233de1e2660..a05616882df 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -26,7 +26,7 @@ from CIME.config import Config from CIME.provenance import save_test_time, get_test_success from CIME.locked_files import LOCKED_DIR, lock_file, is_locked -from CIME.baselines import ( +from CIME.baselines.performance import ( get_latest_cpl_logs, _perf_get_memory, perf_compare_memory_baseline, diff --git a/CIME/baselines/__init__.py b/CIME/baselines/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/CIME/baselines.py b/CIME/baselines/performance.py similarity index 100% rename from CIME/baselines.py rename to CIME/baselines/performance.py diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index 5802d685b71..0f09b91eb0b 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -11,7 +11,7 @@ from CIME.hist_utils import generate_baseline, compare_baseline from CIME.case import Case from CIME.test_utils import get_test_status_files -from CIME.baselines import ( +from CIME.baselines.performance import ( perf_compare_throughput_baseline, perf_compare_memory_baseline, perf_write_baseline, diff --git a/CIME/tests/test_unit_baselines.py b/CIME/tests/test_unit_baselines_performance.py similarity index 74% rename from CIME/tests/test_unit_baselines.py rename to CIME/tests/test_unit_baselines_performance.py index 8382339bdc7..a6eebbb2bbb 100644 --- a/CIME/tests/test_unit_baselines.py +++ b/CIME/tests/test_unit_baselines_performance.py @@ -6,7 +6,7 @@ from unittest import mock from pathlib import Path -from CIME import baselines +from CIME.baselines import performance from CIME.tests.test_unit_system_tests import CPLLOG @@ -27,8 +27,8 @@ def create_mock_case(tempdir, get_latest_cpl_logs=None): return case, caseroot, rundir, baseline_root -class TestUnitBaseline(unittest.TestCase): - @mock.patch("CIME.baselines._perf_get_memory") +class TestUnitBaselinesPerformance(unittest.TestCase): + @mock.patch("CIME.baselines.performance._perf_get_memory") def test_perf_get_memory_default_no_value(self, _perf_get_memory): _perf_get_memory.return_value = None @@ -39,9 +39,9 @@ def test_perf_get_memory_default_no_value(self, _perf_get_memory): config.perf_get_memory.side_effect = AttributeError with self.assertRaises(RuntimeError): - baselines.perf_get_memory(case, config) + performance.perf_get_memory(case, config) - @mock.patch("CIME.baselines._perf_get_memory") + @mock.patch("CIME.baselines.performance._perf_get_memory") def test_perf_get_memory_default(self, _perf_get_memory): _perf_get_memory.return_value = [(1, 1000)] @@ -51,7 +51,7 @@ def test_perf_get_memory_default(self, _perf_get_memory): config.perf_get_memory.side_effect = AttributeError - mem = baselines.perf_get_memory(case, config) + mem = performance.perf_get_memory(case, config) assert mem == "1000" @@ -62,11 +62,11 @@ def test_perf_get_memory(self): config.perf_get_memory.return_value = "1000" - mem = baselines.perf_get_memory(case, config) + mem = performance.perf_get_memory(case, config) assert mem == "1000" - @mock.patch("CIME.baselines._perf_get_throughput") + @mock.patch("CIME.baselines.performance._perf_get_throughput") def test_perf_get_throughput_default_no_value(self, _perf_get_throughput): _perf_get_throughput.return_value = None @@ -77,9 +77,9 @@ def test_perf_get_throughput_default_no_value(self, _perf_get_throughput): config.perf_get_throughput.side_effect = AttributeError with self.assertRaises(RuntimeError): - baselines.perf_get_throughput(case, config) + performance.perf_get_throughput(case, config) - @mock.patch("CIME.baselines._perf_get_throughput") + @mock.patch("CIME.baselines.performance._perf_get_throughput") def test_perf_get_throughput_default(self, _perf_get_throughput): _perf_get_throughput.return_value = 100 @@ -89,7 +89,7 @@ def test_perf_get_throughput_default(self, _perf_get_throughput): config.perf_get_throughput.side_effect = AttributeError - tput = baselines.perf_get_throughput(case, config) + tput = performance.perf_get_throughput(case, config) assert tput == "100" @@ -100,12 +100,12 @@ def test_perf_get_throughput(self): config.perf_get_throughput.return_value = "100" - tput = baselines.perf_get_throughput(case, config) + tput = performance.perf_get_throughput(case, config) assert tput == "100" def test_get_cpl_throughput_no_file(self): - throughput = baselines.get_cpl_throughput("/tmp/cpl.log") + throughput = performance.get_cpl_throughput("/tmp/cpl.log") assert throughput is None @@ -116,7 +116,7 @@ def test_get_cpl_throughput(self): with gzip.open(cpl_log_path, "w") as fd: fd.write(CPLLOG.encode("utf-8")) - throughput = baselines.get_cpl_throughput(str(cpl_log_path)) + throughput = performance.get_cpl_throughput(str(cpl_log_path)) assert throughput == 719.635 @@ -127,7 +127,7 @@ def test_get_cpl_mem_usage_gz(self): with gzip.open(cpl_log_path, "w") as fd: fd.write(CPLLOG.encode("utf-8")) - mem_usage = baselines.get_cpl_mem_usage(str(cpl_log_path)) + mem_usage = performance.get_cpl_mem_usage(str(cpl_log_path)) assert mem_usage == [ (10102.0, 1673.89), @@ -136,14 +136,14 @@ def test_get_cpl_mem_usage_gz(self): (10105.0, 1673.89), ] - @mock.patch("CIME.baselines.os.path.isfile") + @mock.patch("CIME.baselines.performance.os.path.isfile") def test_get_cpl_mem_usage(self, isfile): isfile.return_value = True with mock.patch( "builtins.open", mock.mock_open(read_data=CPLLOG.encode("utf-8")) ) as mock_file: - mem_usage = baselines.get_cpl_mem_usage("/tmp/cpl.log") + mem_usage = performance.get_cpl_mem_usage("/tmp/cpl.log") assert mem_usage == [ (10102.0, 1673.89), @@ -156,7 +156,7 @@ def test_read_baseline_file_multi_line(self): with mock.patch( "builtins.open", mock.mock_open(read_data="1000.0\n2000.0\n") ) as mock_file: - baseline = baselines.read_baseline_file("/tmp/cpl-mem.log") + baseline = performance.read_baseline_file("/tmp/cpl-mem.log") mock_file.assert_called_with("/tmp/cpl-mem.log") assert baseline == ["1000.0", "2000.0"] @@ -165,64 +165,64 @@ def test_read_baseline_file_content(self): with mock.patch( "builtins.open", mock.mock_open(read_data="1000.0") ) as mock_file: - baseline = baselines.read_baseline_file("/tmp/cpl-mem.log") + baseline = performance.read_baseline_file("/tmp/cpl-mem.log") mock_file.assert_called_with("/tmp/cpl-mem.log") assert baseline == ["1000.0"] def test_read_baseline_file(self): with mock.patch("builtins.open", mock.mock_open(read_data="")) as mock_file: - baseline = baselines.read_baseline_file("/tmp/cpl-mem.log") + baseline = performance.read_baseline_file("/tmp/cpl-mem.log") mock_file.assert_called_with("/tmp/cpl-mem.log") assert baseline == [] def test_write_baseline_file(self): with mock.patch("builtins.open", mock.mock_open()) as mock_file: - baselines.write_baseline_file("/tmp/cpl-tput.log", "1000") + performance.write_baseline_file("/tmp/cpl-tput.log", "1000") mock_file.assert_called_with("/tmp/cpl-tput.log", "w") mock_file.return_value.write.assert_called_with("1000") - @mock.patch("CIME.baselines.get_cpl_throughput") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance.get_cpl_throughput") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test__perf_get_throughput(self, get_latest_cpl_logs, get_cpl_throughput): get_cpl_throughput.side_effect = FileNotFoundError() with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) - tput = baselines._perf_get_throughput(case) + tput = performance._perf_get_throughput(case) assert tput == None - @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test__perf_get_memory_override(self, get_latest_cpl_logs, get_cpl_mem_usage): get_cpl_mem_usage.side_effect = FileNotFoundError() with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) - mem = baselines._perf_get_memory(case, "/tmp/override") + mem = performance._perf_get_memory(case, "/tmp/override") assert mem == None - @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test__perf_get_memory(self, get_latest_cpl_logs, get_cpl_mem_usage): get_cpl_mem_usage.side_effect = FileNotFoundError() with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) - mem = baselines._perf_get_memory(case) + mem = performance._perf_get_memory(case) assert mem == None - @mock.patch("CIME.baselines.write_baseline_file") - @mock.patch("CIME.baselines.perf_get_memory") - @mock.patch("CIME.baselines.perf_get_throughput") + @mock.patch("CIME.baselines.performance.write_baseline_file") + @mock.patch("CIME.baselines.performance.perf_get_memory") + @mock.patch("CIME.baselines.performance.perf_get_throughput") def test_write_baseline_skip( self, perf_get_throughput, perf_get_memory, write_baseline_file ): @@ -233,7 +233,7 @@ def test_write_baseline_skip( with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir) - baselines.perf_write_baseline( + performance.perf_write_baseline( case, baseline_root, False, @@ -244,9 +244,9 @@ def test_write_baseline_skip( perf_get_memory.assert_not_called() write_baseline_file.assert_not_called() - @mock.patch("CIME.baselines.write_baseline_file") - @mock.patch("CIME.baselines.perf_get_memory") - @mock.patch("CIME.baselines.perf_get_throughput") + @mock.patch("CIME.baselines.performance.write_baseline_file") + @mock.patch("CIME.baselines.performance.perf_get_memory") + @mock.patch("CIME.baselines.performance.perf_get_throughput") def test_write_baseline_runtimeerror( self, perf_get_throughput, perf_get_memory, write_baseline_file ): @@ -257,15 +257,15 @@ def test_write_baseline_runtimeerror( with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir) - baselines.perf_write_baseline(case, baseline_root) + performance.perf_write_baseline(case, baseline_root) perf_get_throughput.assert_called() perf_get_memory.assert_called() write_baseline_file.assert_not_called() - @mock.patch("CIME.baselines.write_baseline_file") - @mock.patch("CIME.baselines.perf_get_memory") - @mock.patch("CIME.baselines.perf_get_throughput") + @mock.patch("CIME.baselines.performance.write_baseline_file") + @mock.patch("CIME.baselines.performance.perf_get_memory") + @mock.patch("CIME.baselines.performance.perf_get_throughput") def test_perf_write_baseline( self, perf_get_throughput, perf_get_memory, write_baseline_file ): @@ -276,16 +276,16 @@ def test_perf_write_baseline( with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir) - baselines.perf_write_baseline(case, baseline_root) + performance.perf_write_baseline(case, baseline_root) perf_get_throughput.assert_called() perf_get_memory.assert_called() write_baseline_file.assert_any_call(str(baseline_root / "cpl-tput.log"), "100") write_baseline_file.assert_any_call(str(baseline_root / "cpl-mem.log"), "1000") - @mock.patch("CIME.baselines._perf_get_throughput") - @mock.patch("CIME.baselines.read_baseline_file") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance._perf_get_throughput") + @mock.patch("CIME.baselines.performance.read_baseline_file") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test_perf_compare_throughput_baseline_no_baseline_file( self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): @@ -303,16 +303,16 @@ def test_perf_compare_throughput_baseline_no_baseline_file( 0.05, ) - (below_tolerance, comment) = baselines.perf_compare_throughput_baseline( + (below_tolerance, comment) = performance.perf_compare_throughput_baseline( case ) assert below_tolerance is None assert comment == "Could not read baseline throughput file: " - @mock.patch("CIME.baselines._perf_get_throughput") - @mock.patch("CIME.baselines.read_baseline_file") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance._perf_get_throughput") + @mock.patch("CIME.baselines.performance.read_baseline_file") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test_perf_compare_throughput_baseline_no_baseline( self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): @@ -332,7 +332,7 @@ def test_perf_compare_throughput_baseline_no_baseline( 0.05, ) - (below_tolerance, comment) = baselines.perf_compare_throughput_baseline( + (below_tolerance, comment) = performance.perf_compare_throughput_baseline( case ) @@ -342,9 +342,9 @@ def test_perf_compare_throughput_baseline_no_baseline( == "Could not compare throughput to baseline, as basline had no value." ) - @mock.patch("CIME.baselines._perf_get_throughput") - @mock.patch("CIME.baselines.read_baseline_file") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance._perf_get_throughput") + @mock.patch("CIME.baselines.performance.read_baseline_file") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test_perf_compare_throughput_baseline_no_tolerance( self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): @@ -366,7 +366,7 @@ def test_perf_compare_throughput_baseline_no_tolerance( None, ) - (below_tolerance, comment) = baselines.perf_compare_throughput_baseline( + (below_tolerance, comment) = performance.perf_compare_throughput_baseline( case ) @@ -376,9 +376,9 @@ def test_perf_compare_throughput_baseline_no_tolerance( == "TPUTCOMP: Computation time changed by -0.80% relative to baseline" ) - @mock.patch("CIME.baselines._perf_get_throughput") - @mock.patch("CIME.baselines.read_baseline_file") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance._perf_get_throughput") + @mock.patch("CIME.baselines.performance.read_baseline_file") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test_perf_compare_throughput_baseline_above_threshold( self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): @@ -398,7 +398,7 @@ def test_perf_compare_throughput_baseline_above_threshold( 0.05, ) - (below_tolerance, comment) = baselines.perf_compare_throughput_baseline( + (below_tolerance, comment) = performance.perf_compare_throughput_baseline( case ) @@ -407,9 +407,9 @@ def test_perf_compare_throughput_baseline_above_threshold( comment == "Error: TPUTCOMP: Computation time increase > 5% from baseline" ) - @mock.patch("CIME.baselines._perf_get_throughput") - @mock.patch("CIME.baselines.read_baseline_file") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance._perf_get_throughput") + @mock.patch("CIME.baselines.performance.read_baseline_file") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test_perf_compare_throughput_baseline( self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): @@ -429,7 +429,7 @@ def test_perf_compare_throughput_baseline( 0.05, ) - (below_tolerance, comment) = baselines.perf_compare_throughput_baseline( + (below_tolerance, comment) = performance.perf_compare_throughput_baseline( case ) @@ -439,9 +439,9 @@ def test_perf_compare_throughput_baseline( == "TPUTCOMP: Computation time changed by -0.80% relative to baseline" ) - @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.read_baseline_file") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") + @mock.patch("CIME.baselines.performance.read_baseline_file") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test_perf_compare_memory_baseline_no_baseline( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): @@ -466,7 +466,7 @@ def test_perf_compare_memory_baseline_no_baseline( 0.05, ) - (below_tolerance, comment) = baselines.perf_compare_memory_baseline(case) + (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) assert below_tolerance assert ( @@ -474,9 +474,9 @@ def test_perf_compare_memory_baseline_no_baseline( == "MEMCOMP: Memory usage highwater has changed by 0.00% relative to baseline" ) - @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.read_baseline_file") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") + @mock.patch("CIME.baselines.performance.read_baseline_file") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test_perf_compare_memory_baseline_not_enough_samples( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): @@ -497,14 +497,14 @@ def test_perf_compare_memory_baseline_not_enough_samples( 0.05, ) - (below_tolerance, comment) = baselines.perf_compare_memory_baseline(case) + (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) assert below_tolerance is None assert comment == "Found 2 memory usage samples, need atleast 4" - @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.read_baseline_file") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") + @mock.patch("CIME.baselines.performance.read_baseline_file") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test_perf_compare_memory_baseline_no_baseline_file( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): @@ -527,14 +527,14 @@ def test_perf_compare_memory_baseline_no_baseline_file( 0.05, ) - (below_tolerance, comment) = baselines.perf_compare_memory_baseline(case) + (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) assert below_tolerance is None assert comment == "Could not read baseline memory usage: " - @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.read_baseline_file") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") + @mock.patch("CIME.baselines.performance.read_baseline_file") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test_perf_compare_memory_baseline_no_tolerance( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): @@ -559,7 +559,7 @@ def test_perf_compare_memory_baseline_no_tolerance( None, ) - (below_tolerance, comment) = baselines.perf_compare_memory_baseline(case) + (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) assert below_tolerance assert ( @@ -567,9 +567,9 @@ def test_perf_compare_memory_baseline_no_tolerance( == "MEMCOMP: Memory usage highwater has changed by 0.30% relative to baseline" ) - @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.read_baseline_file") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") + @mock.patch("CIME.baselines.performance.read_baseline_file") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test_perf_compare_memory_baseline_above_threshold( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): @@ -594,7 +594,7 @@ def test_perf_compare_memory_baseline_above_threshold( 0.05, ) - (below_tolerance, comment) = baselines.perf_compare_memory_baseline(case) + (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) assert not below_tolerance assert ( @@ -602,9 +602,9 @@ def test_perf_compare_memory_baseline_above_threshold( == "Error: Memory usage increase >5% from baseline's 1000.000000 to 2003.000000" ) - @mock.patch("CIME.baselines.get_cpl_mem_usage") - @mock.patch("CIME.baselines.read_baseline_file") - @mock.patch("CIME.baselines.get_latest_cpl_logs") + @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") + @mock.patch("CIME.baselines.performance.read_baseline_file") + @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") def test_perf_compare_memory_baseline( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): @@ -629,7 +629,7 @@ def test_perf_compare_memory_baseline( 0.05, ) - (below_tolerance, comment) = baselines.perf_compare_memory_baseline(case) + (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) assert below_tolerance assert ( @@ -654,7 +654,7 @@ def test_get_latest_cpl_logs_found_multiple(self): "mct", ) - latest_cpl_logs = baselines.get_latest_cpl_logs(case) + latest_cpl_logs = performance.get_latest_cpl_logs(case) assert len(latest_cpl_logs) == 2 assert sorted(latest_cpl_logs) == sorted( @@ -675,7 +675,7 @@ def test_get_latest_cpl_logs_found_single(self): "mct", ) - latest_cpl_logs = baselines.get_latest_cpl_logs(case) + latest_cpl_logs = performance.get_latest_cpl_logs(case) assert len(latest_cpl_logs) == 1 assert latest_cpl_logs[0] == str(cpl_log_path) @@ -687,6 +687,6 @@ def test_get_latest_cpl_logs(self): "mct", ) - latest_cpl_logs = baselines.get_latest_cpl_logs(case) + latest_cpl_logs = performance.get_latest_cpl_logs(case) assert len(latest_cpl_logs) == 0 From b856aa19213ec50e6945b10269c167340f7b76e5 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 13 Oct 2023 18:04:49 -0700 Subject: [PATCH 28/31] Fixes returning raw string from baseline --- CIME/baselines/performance.py | 12 +++++---- CIME/tests/test_unit_baselines_performance.py | 27 +++++++++---------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/CIME/baselines/performance.py b/CIME/baselines/performance.py index 2a3fe7369c6..b2ede20dce5 100644 --- a/CIME/baselines/performance.py +++ b/CIME/baselines/performance.py @@ -417,6 +417,8 @@ def read_baseline_file(baseline_file): """ Reads value from `baseline_file`. + Strips comments and returns the raw content to be decoded. + Parameters ---------- baseline_file : str @@ -430,7 +432,7 @@ def read_baseline_file(baseline_file): with open(baseline_file) as fd: lines = [x.strip() for x in fd.readlines() if not x.startswith("#")] - return lines + return "\n".join(lines) def _perf_compare_throughput_baseline(case, baseline, tolerance): @@ -459,8 +461,8 @@ def _perf_compare_throughput_baseline(case, baseline, tolerance): try: # default baseline is stored as single float - baseline = float(baseline[0]) - except IndexError: + baseline = float(baseline) + except ValueError: comment = "Could not compare throughput to baseline, as basline had no value." return None, comment @@ -516,8 +518,8 @@ def _perf_compare_memory_baseline(case, baseline, tolerance): try: # default baseline is stored as single float - baseline = float(baseline[0]) - except IndexError: + baseline = float(baseline) + except ValueError: baseline = 0.0 try: diff --git a/CIME/tests/test_unit_baselines_performance.py b/CIME/tests/test_unit_baselines_performance.py index a6eebbb2bbb..c73c2c15bd3 100644 --- a/CIME/tests/test_unit_baselines_performance.py +++ b/CIME/tests/test_unit_baselines_performance.py @@ -154,12 +154,13 @@ def test_get_cpl_mem_usage(self, isfile): def test_read_baseline_file_multi_line(self): with mock.patch( - "builtins.open", mock.mock_open(read_data="1000.0\n2000.0\n") + "builtins.open", + mock.mock_open(read_data="#comment about data\n1000.0\n2000.0\n"), ) as mock_file: baseline = performance.read_baseline_file("/tmp/cpl-mem.log") mock_file.assert_called_with("/tmp/cpl-mem.log") - assert baseline == ["1000.0", "2000.0"] + assert baseline == "1000.0\n2000.0" def test_read_baseline_file_content(self): with mock.patch( @@ -168,14 +169,14 @@ def test_read_baseline_file_content(self): baseline = performance.read_baseline_file("/tmp/cpl-mem.log") mock_file.assert_called_with("/tmp/cpl-mem.log") - assert baseline == ["1000.0"] + assert baseline == "1000.0" def test_read_baseline_file(self): with mock.patch("builtins.open", mock.mock_open(read_data="")) as mock_file: baseline = performance.read_baseline_file("/tmp/cpl-mem.log") mock_file.assert_called_with("/tmp/cpl-mem.log") - assert baseline == [] + assert baseline == "" def test_write_baseline_file(self): with mock.patch("builtins.open", mock.mock_open()) as mock_file: @@ -316,7 +317,7 @@ def test_perf_compare_throughput_baseline_no_baseline_file( def test_perf_compare_throughput_baseline_no_baseline( self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): - read_baseline_file.return_value = [] + read_baseline_file.return_value = "" _perf_get_throughput.return_value = 504 @@ -348,9 +349,7 @@ def test_perf_compare_throughput_baseline_no_baseline( def test_perf_compare_throughput_baseline_no_tolerance( self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): - read_baseline_file.return_value = [ - "500", - ] + read_baseline_file.return_value = "500" _perf_get_throughput.return_value = 504 @@ -382,7 +381,7 @@ def test_perf_compare_throughput_baseline_no_tolerance( def test_perf_compare_throughput_baseline_above_threshold( self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): - read_baseline_file.return_value = ["1000"] + read_baseline_file.return_value = "1000" _perf_get_throughput.return_value = 504 @@ -413,7 +412,7 @@ def test_perf_compare_throughput_baseline_above_threshold( def test_perf_compare_throughput_baseline( self, get_latest_cpl_logs, read_baseline_file, _perf_get_throughput ): - read_baseline_file.return_value = ["500"] + read_baseline_file.return_value = "500" _perf_get_throughput.return_value = 504 @@ -445,7 +444,7 @@ def test_perf_compare_throughput_baseline( def test_perf_compare_memory_baseline_no_baseline( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): - read_baseline_file.return_value = [] + read_baseline_file.return_value = "" get_cpl_mem_usage.return_value = [ (1, 1000.0), @@ -538,7 +537,7 @@ def test_perf_compare_memory_baseline_no_baseline_file( def test_perf_compare_memory_baseline_no_tolerance( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): - read_baseline_file.return_value = ["1000.0"] + read_baseline_file.return_value = "1000.0" get_cpl_mem_usage.return_value = [ (1, 1000.0), @@ -573,7 +572,7 @@ def test_perf_compare_memory_baseline_no_tolerance( def test_perf_compare_memory_baseline_above_threshold( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): - read_baseline_file.return_value = ["1000.0"] + read_baseline_file.return_value = "1000.0" get_cpl_mem_usage.return_value = [ (1, 2000.0), @@ -608,7 +607,7 @@ def test_perf_compare_memory_baseline_above_threshold( def test_perf_compare_memory_baseline( self, get_latest_cpl_logs, read_baseline_file, get_cpl_mem_usage ): - read_baseline_file.return_value = ["1000.0"] + read_baseline_file.return_value = "1000.0" get_cpl_mem_usage.return_value = [ (1, 1000.0), From 192eb3236fb377325cf1c5e1524166bb84943c2a Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 13 Oct 2023 18:05:37 -0700 Subject: [PATCH 29/31] Updates baseline documentation --- doc/source/users_guide/testing.rst | 151 ++++++++++++++++++++++++++--- 1 file changed, 135 insertions(+), 16 deletions(-) diff --git a/doc/source/users_guide/testing.rst b/doc/source/users_guide/testing.rst index 8ea7c29467c..40868d2bbdd 100644 --- a/doc/source/users_guide/testing.rst +++ b/doc/source/users_guide/testing.rst @@ -371,29 +371,148 @@ Interpreting test output is pretty easy, looking at an example:: You can see that `create_test <../Tools_user/create_test.html>`_ informs the user of the case directory and of the progress and duration of the various test phases. -=================== -Managing baselines -=================== -.. _`Managing baselines`: +========= +Baselines +========= +.. _`Baselines`: -A big part of testing is managing your baselines (sometimes called gold results). We have provided -tools to help the user do this without having to repeat full runs of test cases with `create_test <../Tools_user/create_test.html>`_ . +A big part of testing is managing your baselines (sometimes called gold results). We have provided tools to help the user do this without having to repeat full runs of test cases with `create_test <../Tools_user/create_test.html>`_ . -bless_test_results: Takes a batch of cases of tests that have already been run and copy their -results to a baseline area. +------------------- +Creating a baseline +------------------- +.. _`Creating a baseline`: -compare_test_results: Takes a batch of cases of tests that have already been run and compare their -results to a baseline area. +A baseline can be generated by passing ``-g`` to `create_test <../Tools_user/create_test.html>`_. There are additional options to control generating baselines.:: -Take a batch of results for the jenkins user for the testid 'mytest' and copy the results to -baselines for 'master':: + ./scripts/create_test -b master -g SMS.ne30_f19_g16_rx1.A - ./bless_test_results -r /home/jenkins/e3sm/scratch/jenkins/ -t mytest -b master +-------------------- +Comparing a baseline +-------------------- +.. _`Comparing a baseline`: -Take a batch of results for the jenkins user for the testid 'mytest' and compare the results to -baselines for 'master':: +Comparing the output of a test to a baseline is achieved by passing ``-c`` to `create_test <../Tools_user/create_test.html>`_.:: + + ./scripts/create_test -b master -c SMS.ne30_f19_g16_rx1.A + +------------------ +Managing baselines +------------------ +.. _`Managing baselines`: - ./compare_test_results -r /home/jenkins/e3sm/scratch/jenkins/ -t mytest -b master +Once a baseline has been generated it can be managed using the `bless_test_results <../Tools_user/bless_test_results.html>`_ tool. The tool provides the ability to bless different features of the baseline. The currently supported features are namelist files, history files, and performance metrics. The performance metrics are separated into throughput and memory usage. + +The following command can be used to compare a test to a baseline and bless an update to the history file.:: + + ./CIME/Tools/bless_test_results -b master --hist-only SMS.ne30_f19_g16_rx1.A + +The `compare_test_results <../Tools_user/compare_test_results.html>_` tool can be used to quickly compare tests to baselines and report any `diffs`.:: + + ./CIME/Tools/compare_test_results -b master SMS.ne30_f19_g16_rx1.A + +--------------------- +Performance baselines +--------------------- +.. _`Performance baselines`: +By default performance baselines are generated by parsing the coupler log and comparing the throughput in SYPD (Simulated Years Per Day) and the memory usage high water. + +This can be customized by creating a python module under ``$DRIVER_ROOT/cime_config/customize``. There are four hooks that can be used to customize the generation and comparison. + +- perf_get_throughput +- perf_get_memory +- perf_compare_throughput_baseline +- perf_compare_memory_baseline + +.. + TODO need to add api docs and link +The following pseudo code is an example of this customization.:: + + # $DRIVER/cime_config/customize/perf_baseline.py + + def perf_get_throughput(case): + """ + Parameters + ---------- + case : CIME.case.case.Case + Current case object. + + Returns + ------- + str + Storing throughput value. + """ + current = analyze_throughput(...) + + return json.dumps(current) + + def perf_get_memory(case): + """ + Parameters + ---------- + case : CIME.case.case.Case + Current case object. + + Returns + ------- + str + Storing memory value. + """ + current = analyze_memory(case) + + return json.dumps(current) + + def perf_compare_throughput_baseline(case, baseline, tolerance): + """ + Parameters + ---------- + case : CIME.case.case.Case + Current case object. + baseline : str + Baseline throughput value. + tolerance : float + Allowed difference tolerance. + + Returns + ------- + bool + Whether throughput diff is below tolerance. + str + Comments about the results. + """ + current = analyze_throughput(case) + + baseline = json.loads(baseline) + + diff, comments = generate_diff(...) + + return diff, comments + + def perf_compare_memory_baseline(case, baseline, tolerance): + """ + Parameters + ---------- + case : CIME.case.case.Case + Current case object. + baseline : str + Baseline memory value. + tolerance : float + Allowed difference tolerance. + + Returns + ------- + bool + Whether memory diff is below tolerance. + str + Comments about the results. + """ + current = analyze_memory(case) + + baseline = json.loads(baseline) + + diff, comments = generate_diff(...) + + return diff, comments ============= Adding tests From 3071dc359d03c7d24ba1d65581fd49f062eb4420 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 13 Oct 2023 18:40:06 -0700 Subject: [PATCH 30/31] Adds missing logging --- CIME/baselines/performance.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/CIME/baselines/performance.py b/CIME/baselines/performance.py index b2ede20dce5..f8f1fda77a1 100644 --- a/CIME/baselines/performance.py +++ b/CIME/baselines/performance.py @@ -136,8 +136,8 @@ def perf_write_baseline(case, basegen_dir, throughput=True, memory=True): if throughput: try: tput = perf_get_throughput(case, config) - except RuntimeError: - pass + except RuntimeError as e: + logger.debug("Could not get throughput: {0!s}".format(e)) else: baseline_file = os.path.join(basegen_dir, "cpl-tput.log") @@ -146,8 +146,8 @@ def perf_write_baseline(case, basegen_dir, throughput=True, memory=True): if memory: try: mem = perf_get_memory(case, config) - except RuntimeError: - pass + except RuntimeError as e: + logger.info("Could not get memory usage: {0!s}".format(e)) else: baseline_file = os.path.join(basegen_dir, "cpl-mem.log") @@ -286,6 +286,8 @@ def _perf_get_memory(case, cpllog=None): memlist = get_cpl_mem_usage(cpllog[0]) except (FileNotFoundError, IndexError): memlist = None + + logger.debug("Could not parse memory usage from coupler log") else: if len(memlist) <= 3: raise RuntimeError( @@ -318,6 +320,8 @@ def _perf_get_throughput(case): except (FileNotFoundError, IndexError): tput = None + logger.debug("Could not parse throughput from coupler log") + return tput From 7e307036e05904c448b523726b6274a946e4dae2 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 13 Oct 2023 18:56:01 -0700 Subject: [PATCH 31/31] Fixes commenting preview that was never pushed --- .github/workflows/docs.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 098cd4c8c01..70899438082 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -91,7 +91,9 @@ jobs: user_name: 'github-actions[bot]' user_email: 'github-actions[bot]@users.noreply.github.com' - name: Comment about previewing documentation - if: ${{ github.event_name == 'pull_request' }} + if: | + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository uses: actions/github-script@v6 with: script: |