From a1137ad0695a27c07a1b4e8fca4bdbd905708241 Mon Sep 17 00:00:00 2001 From: Pierre-Marie de Rodat Date: Fri, 23 Aug 2024 15:02:30 +0200 Subject: [PATCH] Fix test filtering when not all test finders have dedicated directories When one test finder TF1 guarantees dedicated directories for its tests but not TF2, and when a testsuite using these two test finders is run with a filtering pattern on the command line, e3.testsuite currently does not use the pattern to filter the directories to probe. This is expected for TF2: when tests do not have their own directories, filtering is done using the test matcher only. However, the pattern should still be matches against the test directory for TF1: most test finders with dedicated dirs do not provide a test matcher, so we need to rely on the test directory name to do the filtering. This change corrects the filtering logic so that the directory name is ignored only for tests found by TF2. --- NEWS.md | 1 + src/e3/testsuite/__init__.py | 20 ++++----- tests/tests/test_basics.py | 87 +++++++++++++++++++++++++++++++++++- 3 files changed, 97 insertions(+), 11 deletions(-) diff --git a/NEWS.md b/NEWS.md index 1b25be4..323f5bb 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,7 @@ 27.0 (Not released yet) ======================= +* Fix test filtering when not all test finders have dedicated directories. * `PatternSubstitute`: update annotations to accept replacement callbacks. * `e3.testsuite.report.xunit`: add a `XUnitImportApp` class to make it possible to create customized versions of the `e3-convert-xunit` script without code diff --git a/src/e3/testsuite/__init__.py b/src/e3/testsuite/__init__.py index 69b27dc..9cbb58f 100644 --- a/src/e3/testsuite/__init__.py +++ b/src/e3/testsuite/__init__.py @@ -653,9 +653,6 @@ def get_test_list(self, sublist: List[str]) -> List[ParsedTest]: # several patterns in "sublist" may yield the same testcase. testcases: Dict[str, ParsedTest] = {} test_finders = self.test_finders - dedicated_dirs_only = all( - tf.test_dedicated_directory for tf in test_finders - ) def matches_pattern( pattern: Optional[Pattern[str]], name: str @@ -711,19 +708,22 @@ def helper(spec: str) -> None: if vcsdir in dirnames: dirnames.remove(vcsdir) - # If all tests are guaranteed to have a dedicated directory, - # do not process directories that don't match the requested - # pattern. - if dedicated_dirs_only and not matches_pattern( - pattern, dirpath - ): - continue + # Compute whether the pattern matches the directory only once + # (now) instead of once per test finder (in the for loop + # below). + pattern_matches = matches_pattern(pattern, dirpath) # The first test finder that has a match "wins". When handling # test data, we want to deal only with absolute paths, so get # the absolute name now. dirpath = os.path.abspath(dirpath) for tf in test_finders: + # If this test finder guarantees that each testcase has a + # dedicated directory, do not process this directory if it + # does not match the requested pattern. + if tf.test_dedicated_directory and not pattern_matches: + continue + try: test_or_list = tf.probe( self, dirpath, dirnames, filenames diff --git a/tests/tests/test_basics.py b/tests/tests/test_basics.py index 3b61480..4dbc406 100644 --- a/tests/tests/test_basics.py +++ b/tests/tests/test_basics.py @@ -24,7 +24,11 @@ TestResultSummary as ResultSummary, TestStatus as Status, ) -from e3.testsuite.testcase_finder import TestFinder as Finder, ParsedTest +from e3.testsuite.testcase_finder import ( + TestFinder as Finder, + ParsedTest, + YAMLTestFinder, +) from .utils import ( MultiSchedulingSuite, @@ -1346,3 +1350,84 @@ def test(self): "t1-error": Status.ERROR, "t2-pass": Status.PASS, } + + +def test_filtering(tmp_path): + """Check support for filtering tests on the command line.""" + # Create the hierarchy of test directories in the "tests" directory + tests_dir = tmp_path / "tests" + for tc, filename in [ + ("foo/a", "test.yaml"), + ("foo/bar", "test.yaml"), + ("bar", "test.yaml"), + ("custom", "custom.yaml"), + ]: + tc_dir = tests_dir / tc + mkdir(str(tc_dir)) + with (tc_dir / filename).open("w"): + pass + + # This test finder, unlike YAMLTestFinder, does not guarantee that each + # tests has its own directory. + class CustomTestFinder(Finder): + test_dedicated_directory = False + + def probe(self, testsuite, dirpath, dirnames, filenames): + if "custom.yaml" in filenames: + return [ + ParsedTest( + f"custom.{t}", + MyDriver, + {}, + dirpath, + test_matcher=f"custom.{t}", + ) + for t in ["a", "b", "foo"] + ] + + # Dummy test driver that always passes + class MyDriver(BasicDriver): + def run(self, prev, slot): + pass + + def analyze(self, prev, slot): + self.result.set_status(Status.PASS) + self.push_result() + + class Mysuite(Suite): + test_driver_map = {"mydriver": MyDriver} + default_driver = "mydriver" + tests_subdir = str(tests_dir) + test_finders = [YAMLTestFinder(), CustomTestFinder()] + + def check(patterns, expected_results): + """Run the testsuite with the given patterns. + + Check that we get exactly the expected set of test results. + """ + suite = run_testsuite(Mysuite, patterns) + assert extract_results(suite) == { + name: Status.PASS for name in expected_results + } + + # No pattern: all tests should run + check( + [], {"foo__a", "foo__bar", "bar", "custom.a", "custom.b", "custom.foo"} + ) + + # Pattern that is actually a directory: run only tests from that directory + check([str(tests_dir / "bar")], {"bar"}) + + # Pattern that matches a directory: run all tests that matches that pattern + # anyway. + check(["foo"], {"foo__a", "foo__bar", "custom.foo"}) + + check(["custom"], {"custom.a", "custom.b", "custom.foo"}) + check([str(tests_dir / "custom")], {"custom.a", "custom.b", "custom.foo"}) + + # Pattern that matches a "test matcher" string: run only the corresponding + # test. + check(["custom.b"], {"custom.b"}) + + # Pattern that matches the middle of multiple tests: run only these tests + check(["a"], {"foo__a", "foo__bar", "bar", "custom.a"})