Cleanup.

Updated description:

Fix stopping timed out processes run by a JobManager class by only
searching for and killing the command executable being run by clush,
orterun, mpirun, etc. Add a new harness/cmocka.py test to verify the
stopping of the processes with a test timeout.

Skip-unit-tests: true
Skip-fault-injection-test: true
Test-tag: pr daos_test dfuse_test test_load_mpi HarnessCmockaTest
Allow-unstable-test: true

Required-githooks: true

Signed-off-by: Phil Henderson <[email protected]>

src/tests/ftest/daos_test/dfs.py

@@ -4,6 +4,8 @@
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
 
+import os
+
 from daos_core_base import DaosCoreBase
 
 
@@ -32,7 +34,7 @@ def test_daos_dfs_unit(self):
         :avocado: tags=daos_test,dfs_test,dfs
         :avocado: tags=DaosCoreTestDfs,test_daos_dfs_unit
         """
-        self.run_subtest('dfs_test')
+        self.run_subtest(os.path.join(self.bin, "dfs_test"))
 
     def test_daos_dfs_parallel(self):
         """Jira ID: DAOS-5409.
@@ -48,7 +50,7 @@ def test_daos_dfs_parallel(self):
         :avocado: tags=daos_test,dfs_test,dfs
         :avocado: tags=DaosCoreTestDfs,test_daos_dfs_parallel
         """
-        self.run_subtest('dfs_test')
+        self.run_subtest(os.path.join(self.bin, "dfs_test"))
 
     def test_daos_dfs_sys(self):
         """Jira ID: DAOS-7759.
@@ -64,4 +66,4 @@ def test_daos_dfs_sys(self):
         :avocado: tags=daos_test,dfs_test,dfs
         :avocado: tags=DaosCoreTestDfs,test_daos_dfs_sys
         """
-        self.run_subtest('dfs_test')
+        self.run_subtest(os.path.join(self.bin, "dfs_test"))

src/tests/ftest/daos_test/dfuse.py

@@ -103,6 +103,7 @@ def run_test(self, il_lib=None):
                 daos_test_env['D_IL_MAX_EQ'] = '2'
                 daos_test_env['D_IL_NO_BYPASS'] = '1'
 
+        command = os.path.join(self.bin, 'dfuse_test')
         parameters = [
             '--test-dir',
             mount_dir,
@@ -125,8 +126,7 @@ def run_test(self, il_lib=None):
         if cache_mode != 'writeback':
             parameters.append('--metadata')
 
-        job = get_job_manager(
-            self, "Clush", get_cmocka_command(self.bin, 'dfuse_test', " ".join(parameters)))
+        job = get_job_manager(self, "Clush", get_cmocka_command(command, ' '.join(parameters)))
         job.assign_hosts(cmocka_utils.hosts)
         job.assign_environment(daos_test_env)
 

src/tests/ftest/harness/cmocka.py

@@ -24,10 +24,10 @@ def test_no_cmocka_xml(self):
 
         :avocado: tags=all
         :avocado: tags=vm
-        :avocado: tags=harness,harness_cmocka,failure_expected
+        :avocado: tags=harness,failure_expected
         :avocado: tags=HarnessCmockaTest,test_no_cmocka_xml
         """
-        self._run_cmocka_test(get_cmocka_command("", "hostname"), False, True)
+        self._run_cmocka_test(get_cmocka_command("hostname"), False, True)
         self.log.info("Test passed")
 
     def test_clush_manager_timeout(self):
@@ -37,7 +37,7 @@ def test_clush_manager_timeout(self):
 
         :avocado: tags=all
         :avocado: tags=vm
-        :avocado: tags=harness,harness_cmocka,failure_expected
+        :avocado: tags=harness,failure_expected
         :avocado: tags=HarnessCmockaTest,test_clush_manager_timeout
         """
         self._run_cmocka_test(self._get_manager_command("Clush", "sleep", "60"), True, True)
@@ -50,7 +50,7 @@ def test_orterun_manager_timeout(self):
 
         :avocado: tags=all
         :avocado: tags=vm
-        :avocado: tags=harness,harness_cmocka,failure_expected
+        :avocado: tags=harness,failure_expected
         :avocado: tags=HarnessCmockaTest,test_orterun_manager_timeout
         """
         self._run_cmocka_test(self._get_manager_command("Orterun", "sleep", "60"), True, True)
@@ -63,7 +63,7 @@ def test_mpirun_manager_timeout(self):
 
         :avocado: tags=all
         :avocado: tags=vm
-        :avocado: tags=harness,harness_cmocka,failure_expected
+        :avocado: tags=harness,failure_expected
         :avocado: tags=HarnessCmockaTest,test_mpirun_manager_timeout
         """
         self._run_cmocka_test(self._get_manager_command("Mpirun", "sleep", "60"), True, True)
@@ -100,7 +100,7 @@ def _get_manager_command(self, class_name, executable, parameters):
         Returns:
             JobManager: the requested JobManager class
         """
-        command = get_cmocka_command("", executable, parameters)
+        command = get_cmocka_command(executable, parameters)
         manager = get_job_manager(self, class_name, command)
         manager.assign_hosts(get_local_host())
         return manager

src/tests/ftest/util/cmocka_utils.py

@@ -13,23 +13,22 @@
 from run_utils import get_clush_command, run_local, run_remote
 
 
-def get_cmocka_command(path, executable, parameters=None):
+def get_cmocka_command(command, parameters=None):
     """Get an ExecutableCommand representing the provided command string.
 
     Adds detection of any bad keywords in the command output that, if found, will result in a
     command failure.
 
     Args:
-        path (str): the command path to use to create the CmockaCommand
-        executable (str): the command name to use to create the CmockaCommand
+        command (str): the command path to use to create the CmockaCommand
         parameters (str): parameters to use to create the CmockaCommand
 
     Returns:
         ExecutableCommand: the object setup to run the command
-
     """
     keywords = ["Process received signal", "stack smashing detected", "End of error message",
                 "buffer overflow detected"]
+    path, executable = os.path.split(command)
     command = CmockaCommand(path, executable, keywords)
     command.parameters.value = parameters
     return command

src/tests/ftest/util/daos_core_base.py

@@ -51,16 +51,14 @@ def get_test_param(self, name, default=None):
         path = "/".join(["/run/daos_tests", name, "*"])
         return self.params.get(self.get_test_name(), path, default)
 
-    def run_subtest(self, executable='daos_test', path=None):
+    def run_subtest(self, command=None):
         """Run the executable with a subtest argument.
 
         Args:
-            executable (str, optional): name of the executable. Defaults to 'daos_test'.
-            path (str, optional): path for the executable. Defaults to self.bin.
+            command (str, optional): command to run. Defaults to None which will yield daos_test.
         """
-        if path is None:
-            # path=None yields the default self.bin path; path="" yields no path
-            path = self.bin
+        if command is None:
+            command = os.path.join(self.bin, "daos_test")
 
         subtest = self.get_test_param("daos_test")
         num_clients = self.get_test_param("num_clients")
@@ -90,8 +88,7 @@ def run_subtest(self, executable='daos_test', path=None):
         daos_test_env["COVFILE"] = "/tmp/test.cov"
         daos_test_env["POOL_SCM_SIZE"] = str(scm_size)
         daos_test_env["POOL_NVME_SIZE"] = str(nvme_size)
-        daos_test_cmd = get_cmocka_command(
-            path, executable, f"-n {dmg_config_file} -{subtest} {str(args)}")
+        daos_test_cmd = get_cmocka_command(command, f"-n {dmg_config_file} -{subtest} {str(args)}")
         job = get_job_manager(self, "Orterun", daos_test_cmd, mpi_type="openmpi")
         job.assign_hosts(cmocka_utils.hosts, self.workdir, None)
         job.assign_processes(num_clients)