Merge branch 'mr/pmderodat/filter-no-dedicated-dir' into 'master'

Fix test filtering when not all test finders have dedicated directories

See merge request it/e3-testsuite!38

NEWS.md

@@ -1,6 +1,7 @@
 27.0 (Not released yet)
 =======================
 
+* Fix test filtering when not all test finders have dedicated directories.
 * `PatternSubstitute`: update annotations to accept replacement callbacks.
 * `e3.testsuite.report.xunit`: add a `XUnitImportApp` class to make it possible
  to create customized versions of the `e3-convert-xunit` script without code

src/e3/testsuite/__init__.py

@@ -653,9 +653,6 @@ def get_test_list(self, sublist: List[str]) -> List[ParsedTest]:
  # several patterns in "sublist" may yield the same testcase.
  testcases: Dict[str, ParsedTest] = {}
  test_finders = self.test_finders
- dedicated_dirs_only = all(
- tf.test_dedicated_directory for tf in test_finders
- )
 
  def matches_pattern(
  pattern: Optional[Pattern[str]], name: str
@@ -711,19 +708,22 @@ def helper(spec: str) -> None:
  if vcsdir in dirnames:
  dirnames.remove(vcsdir)
 
- # If all tests are guaranteed to have a dedicated directory,
- # do not process directories that don't match the requested
- # pattern.
- if dedicated_dirs_only and not matches_pattern(
- pattern, dirpath
- ):
- continue
+ # Compute whether the pattern matches the directory only once
+ # (now) instead of once per test finder (in the for loop
+ # below).
+ pattern_matches = matches_pattern(pattern, dirpath)
 
  # The first test finder that has a match "wins". When handling
  # test data, we want to deal only with absolute paths, so get
  # the absolute name now.
  dirpath = os.path.abspath(dirpath)
  for tf in test_finders:
+ # If this test finder guarantees that each testcase has a
+ # dedicated directory, do not process this directory if it
+ # does not match the requested pattern.
+ if tf.test_dedicated_directory and not pattern_matches:
+ continue
+
  try:
  test_or_list = tf.probe(
  self, dirpath, dirnames, filenames

tests/tests/test_basics.py

@@ -24,7 +24,11 @@
  TestResultSummary as ResultSummary,
  TestStatus as Status,
 )
-from e3.testsuite.testcase_finder import TestFinder as Finder, ParsedTest
+from e3.testsuite.testcase_finder import (
+ TestFinder as Finder,
+ ParsedTest,
+ YAMLTestFinder,
+)
 
 from .utils import (
  MultiSchedulingSuite,
@@ -1346,3 +1350,84 @@ def test(self):
  "t1-error": Status.ERROR,
  "t2-pass": Status.PASS,
  }
+
+
+def test_filtering(tmp_path):
+ """Check support for filtering tests on the command line."""
+ # Create the hierarchy of test directories in the "tests" directory
+ tests_dir = tmp_path / "tests"
+ for tc, filename in [
+ ("foo/a", "test.yaml"),
+ ("foo/bar", "test.yaml"),
+ ("bar", "test.yaml"),
+ ("custom", "custom.yaml"),
+ ]:
+ tc_dir = tests_dir / tc
+ mkdir(str(tc_dir))
+ with (tc_dir / filename).open("w"):
+ pass
+
+ # This test finder, unlike YAMLTestFinder, does not guarantee that each
+ # tests has its own directory.
+ class CustomTestFinder(Finder):
+ test_dedicated_directory = False
+
+ def probe(self, testsuite, dirpath, dirnames, filenames):
+ if "custom.yaml" in filenames:
+ return [
+ ParsedTest(
+ f"custom.{t}",
+ MyDriver,
+ {},
+ dirpath,
+ test_matcher=f"custom.{t}",
+ )
+ for t in ["a", "b", "foo"]
+ ]
+
+ # Dummy test driver that always passes
+ class MyDriver(BasicDriver):
+ def run(self, prev, slot):
+ pass
+
+ def analyze(self, prev, slot):
+ self.result.set_status(Status.PASS)
+ self.push_result()
+
+ class Mysuite(Suite):
+ test_driver_map = {"mydriver": MyDriver}
+ default_driver = "mydriver"
+ tests_subdir = str(tests_dir)
+ test_finders = [YAMLTestFinder(), CustomTestFinder()]
+
+ def check(patterns, expected_results):
+ """Run the testsuite with the given patterns.
+
+ Check that we get exactly the expected set of test results.
+ """
+ suite = run_testsuite(Mysuite, patterns)
+ assert extract_results(suite) == {
+ name: Status.PASS for name in expected_results
+ }
+
+ # No pattern: all tests should run
+ check(
+ [], {"foo__a", "foo__bar", "bar", "custom.a", "custom.b", "custom.foo"}
+ )
+
+ # Pattern that is actually a directory: run only tests from that directory
+ check([str(tests_dir / "bar")], {"bar"})
+
+ # Pattern that matches a directory: run all tests that matches that pattern
+ # anyway.
+ check(["foo"], {"foo__a", "foo__bar", "custom.foo"})
+
+ check(["custom"], {"custom.a", "custom.b", "custom.foo"})
+ check([str(tests_dir / "custom")], {"custom.a", "custom.b", "custom.foo"})
+
+ # Pattern that matches a "test matcher" string: run only the corresponding
+ # test.
+ check(["custom.b"], {"custom.b"})
+
+ # Pattern that matches the middle of multiple tests: run only these tests
+ check(["a"], {"foo__a", "foo__bar", "bar", "custom.a"})