Merge pull request #4586 from ESMCI/jgfouca/ignore_diffs_option

Add option to ignore diffs across the system

Useful for performance testing.

Test suite: btr and unit tests
Test baseline:
Test namelist changes:
Test status: bit for bit

User interface changes?:

Update gh-pages html (Y/N)?:

CIME/Tools/jenkins_generic_job

@@ -186,6 +186,12 @@ OR
         help="Do not fail if there are namelist diffs",
     )
 
+    parser.add_argument(
+        "--ignore-diffs",
+        action="store_true",
+        help="Do not fail if there are history diffs",
+    )
+
     parser.add_argument(
         "--save-timing",
         action="store_true",
@@ -272,6 +278,7 @@ OR
         args.check_memory,
         args.ignore_memleak,
         args.ignore_namelists,
+        args.ignore_diffs,
         args.save_timing,
         args.pes_file,
         args.jenkins_id,
@@ -304,6 +311,7 @@ def _main_func(description):
         check_memory,
         ignore_memleak,
         ignore_namelists,
+        ignore_diffs,
         save_timing,
         pes_file,
         jenkins_id,
@@ -334,6 +342,7 @@ def _main_func(description):
             check_memory,
             ignore_memleak,
             ignore_namelists,
+            ignore_diffs,
             save_timing,
             pes_file,
             jenkins_id,

CIME/Tools/wait_for_tests

@@ -74,6 +74,12 @@ OR
         help="Do not fail a test if the only problem is diffing namelists",
     )
 
+    parser.add_argument(
+        "--ignore-diffs",
+        action="store_true",
+        help="Do not fail a test if the only problem is diffing history files",
+    )
+
     parser.add_argument(
         "--ignore-memleak",
         action="store_true",
@@ -122,6 +128,7 @@ OR
         args.check_throughput,
         args.check_memory,
         args.ignore_namelist_diffs,
+        args.ignore_diffs,
         args.ignore_memleak,
         args.cdash_build_name,
         args.cdash_project,
@@ -142,6 +149,7 @@ def _main_func(description):
         check_throughput,
         check_memory,
         ignore_namelist_diffs,
+        ignore_diffs,
         ignore_memleak,
         cdash_build_name,
         cdash_project,
@@ -160,6 +168,7 @@ def _main_func(description):
             check_throughput=check_throughput,
             check_memory=check_memory,
             ignore_namelists=ignore_namelist_diffs,
+            ignore_diffs=ignore_diffs,
             ignore_memleak=ignore_memleak,
             cdash_build_name=cdash_build_name,
             cdash_project=cdash_project,

CIME/bless_test_results.py

@@ -216,7 +216,17 @@ def bless_test_results(
     bless_perf=False,
     **_,  # Capture all for extra
 ):
-    bless_all = not (namelists_only | hist_only | bless_tput | bless_mem | bless_perf)
+    if bless_perf:
+        bless_mem = True
+        bless_tput = True
+
+    bless_all_non_perf = not (namelists_only | hist_only | bless_tput | bless_mem)
+    is_perf_bless = bless_mem or bless_tput
+
+    expect(
+        not (is_perf_bless and hist_only) and not (is_perf_bless and namelists_only),
+        "Do not mix performance and non-performance blesses",
+    )
 
     test_status_files = get_test_status_files(test_root, compiler, test_id=test_id)
 
@@ -284,12 +294,13 @@ def bless_test_results(
         overall_result, phase = ts.get_overall_test_status(
             ignore_namelists=True,
             ignore_memleak=True,
-            check_throughput=False,
-            check_memory=False,
+            ignore_diffs=is_perf_bless,
+            check_throughput=bless_tput,
+            check_memory=bless_mem,
         )
 
         # See if we need to bless namelist
-        if namelists_only or bless_all:
+        if namelists_only or bless_all_non_perf:
             if no_skip_pass:
                 nl_bless = True
             else:
@@ -301,25 +312,18 @@ def bless_test_results(
 
         # Skip if test is build only i.e. testopts contains "B"
         if not build_only:
-            bless_needed = is_bless_needed(
+            hist_bless = is_hist_bless_needed(
                 test_name, ts, broken_blesses, overall_result, no_skip_pass, phase
+            ) and (hist_only or bless_all_non_perf)
+            tput_bless = (
+                bless_tput and ts.get_status(THROUGHPUT_PHASE) != TEST_PASS_STATUS
             )
+            mem_bless = bless_mem and ts.get_status(MEMCOMP_PHASE) != TEST_PASS_STATUS
 
-            # See if we need to bless baselines
-            if hist_only or bless_all:
-                hist_bless = bless_needed
-
-            if bless_tput or bless_perf:
-                tput_bless = bless_needed
-
-                if not tput_bless:
-                    tput_bless = ts.get_status(THROUGHPUT_PHASE) != TEST_PASS_STATUS
-
-            if bless_mem or bless_perf:
-                mem_bless = bless_needed
-
-                if not mem_bless:
-                    mem_bless = ts.get_status(MEMCOMP_PHASE) != TEST_PASS_STATUS
+        expect(
+            not ((nl_bless or hist_bless) and (tput_bless or mem_bless)),
+            "Do not mix performance and non-performance blessing",
+        )
 
         # Now, do the bless
         if not nl_bless and not hist_bless and not tput_bless and not mem_bless:
@@ -462,7 +466,9 @@ def bless_test_results(
     return success
 
 
-def is_bless_needed(test_name, ts, broken_blesses, overall_result, no_skip_pass, phase):
+def is_hist_bless_needed(
+    test_name, ts, broken_blesses, overall_result, no_skip_pass, phase
+):
     needed = False
 
     run_result = ts.get_status(RUN_PHASE)

CIME/jenkins_generic_job.py

@@ -280,6 +280,7 @@ def jenkins_generic_job(
     check_memory,
     ignore_memleak,
     ignore_namelists,
+    ignore_diffs,
     save_timing,
     pes_file,
     jenkins_id,
@@ -423,6 +424,7 @@ def jenkins_generic_job(
         check_throughput=check_throughput,
         check_memory=check_memory,
         ignore_namelists=ignore_namelists,
+        ignore_diffs=ignore_diffs,
         ignore_memleak=ignore_memleak,
         cdash_build_name=cdash_build_name,
         cdash_project=cdash_project,

CIME/scripts/create_test.py

@@ -392,6 +392,12 @@ def parse_command_line(args, description):
         help="Do not fail if there namelist diffs",
     )
 
+    parser.add_argument(
+        "--ignore-diffs",
+        action="store_true",
+        help="Do not fail if there history file diffs",
+    )
+
     parser.add_argument(
         "--ignore-memleak", action="store_true", help="Do not fail if there's a memleak"
     )
@@ -761,6 +767,7 @@ def parse_command_line(args, description):
         args.check_throughput,
         args.check_memory,
         args.ignore_namelists,
+        args.ignore_diffs,
         args.ignore_memleak,
         args.allow_pnl,
         args.non_local,
@@ -921,6 +928,7 @@ def create_test(
     check_throughput,
     check_memory,
     ignore_namelists,
+    ignore_diffs,
     ignore_memleak,
     allow_pnl,
     non_local,
@@ -976,6 +984,7 @@ def create_test(
         check_throughput=check_throughput,
         check_memory=check_memory,
         ignore_namelists=ignore_namelists,
+        ignore_diffs=ignore_diffs,
         ignore_memleak=ignore_memleak,
     )
 
@@ -1064,6 +1073,7 @@ def _main_func(description=None):
         check_throughput,
         check_memory,
         ignore_namelists,
+        ignore_diffs,
         ignore_memleak,
         allow_pnl,
         non_local,
@@ -1116,6 +1126,7 @@ def _main_func(description=None):
             check_throughput,
             check_memory,
             ignore_namelists,
+            ignore_diffs,
             ignore_memleak,
             allow_pnl,
             non_local,

CIME/test_scheduler.py

@@ -1429,6 +1429,7 @@ def run_tests(
         check_throughput=False,
         check_memory=False,
         ignore_namelists=False,
+        ignore_diffs=False,
         ignore_memleak=False,
     ):
         ###########################################################################
@@ -1484,6 +1485,7 @@ def run_tests(
             check_throughput=check_throughput,
             check_memory=check_memory,
             ignore_namelists=ignore_namelists,
+            ignore_diffs=ignore_diffs,
             ignore_memleak=ignore_memleak,
             no_run=self._no_run,
             expect_test_complete=expect_test_complete,

CIME/test_status.py

@@ -113,6 +113,7 @@ def _test_helper2(
     check_throughput=False,
     check_memory=False,
     ignore_namelists=False,
+    ignore_diffs=False,
     no_run=False,
     no_perm=False,
 ):
@@ -127,6 +128,7 @@ def _test_helper2(
             check_throughput=check_throughput,
             check_memory=check_memory,
             ignore_namelists=ignore_namelists,
+            ignore_diffs=ignore_diffs,
             no_run=no_run,
         )
         if rv is not None and the_status != rv:
@@ -410,6 +412,7 @@ def _get_overall_status_based_on_phases(
         check_throughput=False,
         check_memory=False,
         ignore_namelists=False,
+        ignore_diffs=False,
         ignore_memleak=False,
         no_run=False,
     ):
@@ -452,6 +455,7 @@ def _get_overall_status_based_on_phases(
                     (not check_throughput and phase == THROUGHPUT_PHASE)
                     or (not check_memory and phase == MEMCOMP_PHASE)
                     or (ignore_namelists and phase == NAMELIST_PHASE)
+                    or (ignore_diffs and phase == BASELINE_PHASE)
                     or (ignore_memleak and phase == MEMLEAK_PHASE)
                 ):
                     continue
@@ -493,6 +497,7 @@ def get_overall_test_status(
         check_throughput=False,
         check_memory=False,
         ignore_namelists=False,
+        ignore_diffs=False,
         ignore_memleak=False,
         no_run=False,
     ):
@@ -527,6 +532,10 @@ def get_overall_test_status(
         ('FAIL', 'COMPARE_2')
         >>> _test_helper2('FAIL ERS.foo.A BASELINE\nFAIL ERS.foo.A NLCOMP\nPASS ERS.foo.A COMPARE_2\nPASS ERS.foo.A RUN')
         ('DIFF', 'BASELINE')
+        >>> _test_helper2('FAIL ERS.foo.A BASELINE\nPASS ERS.foo.A NLCOMP\nPASS ERS.foo.A COMPARE_2\nPASS ERS.foo.A RUN', ignore_diffs=True)
+        ('PASS', 'RUN')
+        >>> _test_helper2('FAIL ERS.foo.A BASELINE\nFAIL ERS.foo.A NLCOMP\nPASS ERS.foo.A COMPARE_2\nPASS ERS.foo.A RUN', ignore_diffs=True)
+        ('NLFAIL', 'RUN')
         >>> _test_helper2('FAIL ERS.foo.A BASELINE\nFAIL ERS.foo.A NLCOMP\nFAIL ERS.foo.A COMPARE_2\nPASS ERS.foo.A RUN')
         ('FAIL', 'COMPARE_2')
         >>> _test_helper2('PEND ERS.foo.A COMPARE_2\nFAIL ERS.foo.A RUN')
@@ -585,6 +594,7 @@ def get_overall_test_status(
             check_throughput=check_throughput,
             check_memory=check_memory,
             ignore_namelists=ignore_namelists,
+            ignore_diffs=ignore_diffs,
             ignore_memleak=ignore_memleak,
             no_run=no_run,
         )
@@ -602,6 +612,7 @@ def get_overall_test_status(
                 check_throughput=check_throughput,
                 check_memory=check_memory,
                 ignore_namelists=ignore_namelists,
+                ignore_diffs=ignore_diffs,
                 ignore_memleak=ignore_memleak,
                 no_run=no_run,
             )

CIME/tests/test_unit_bless_test_results.py

@@ -10,7 +10,7 @@
     _bless_memory,
     bless_history,
     bless_namelists,
-    is_bless_needed,
+    is_hist_bless_needed,
 )
 
 
@@ -469,7 +469,7 @@ def test_bless_memory_only(
 
         ts = TestStatus.return_value
         ts.get_name.return_value = "SMS.f19_g16.S.docker_gnu"
-        ts.get_overall_test_status.return_value = ("PASS", "RUN")
+        ts.get_overall_test_status.return_value = ("DIFF", "MEMCOMP")
         ts.get_status.side_effect = ["PASS", "PASS", "FAIL", "FAIL"]
 
         case = Case.return_value.__enter__.return_value
@@ -508,7 +508,7 @@ def test_bless_throughput_only(
 
         ts = TestStatus.return_value
         ts.get_name.return_value = "SMS.f19_g16.S.docker_gnu"
-        ts.get_overall_test_status.return_value = ("PASS", "RUN")
+        ts.get_overall_test_status.return_value = ("DIFF", "TPUTCOMP")
         ts.get_status.side_effect = ["PASS", "PASS", "FAIL", "FAIL"]
 
         case = Case.return_value.__enter__.return_value
@@ -950,7 +950,7 @@ def test_is_bless_needed_no_skip_fail(self):
 
         broken_blesses = []
 
-        needed = is_bless_needed(
+        needed = is_hist_bless_needed(
             "SMS.f19_g16.A", ts, broken_blesses, "PASS", True, "RUN"
         )
 
@@ -965,7 +965,7 @@ def test_is_bless_needed_overall_fail(self):
 
         broken_blesses = []
 
-        needed = is_bless_needed(
+        needed = is_hist_bless_needed(
             "SMS.f19_g16.A", ts, broken_blesses, "FAIL", False, "RUN"
         )
 
@@ -978,7 +978,7 @@ def test_is_bless_needed_baseline_fail(self):
 
         broken_blesses = []
 
-        needed = is_bless_needed(
+        needed = is_hist_bless_needed(
             "SMS.f19_g16.A", ts, broken_blesses, "PASS", False, "RUN"
         )
 
@@ -993,7 +993,7 @@ def test_is_bless_needed_run_phase_fail(self):
 
         broken_blesses = []
 
-        needed = is_bless_needed(
+        needed = is_hist_bless_needed(
             "SMS.f19_g16.A", ts, broken_blesses, "PASS", False, "RUN"
         )
 
@@ -1006,7 +1006,7 @@ def test_is_bless_needed_no_run_phase(self):
 
         broken_blesses = []
 
-        needed = is_bless_needed(
+        needed = is_hist_bless_needed(
             "SMS.f19_g16.A", ts, broken_blesses, "PASS", False, "RUN"
         )
 
@@ -1019,7 +1019,7 @@ def test_is_bless_needed(self):
 
         broken_blesses = []
 
-        needed = is_bless_needed(
+        needed = is_hist_bless_needed(
             "SMS.f19_g16.A", ts, broken_blesses, "PASS", False, "RUN"
         )