From b9aabf8d444447c8f086202de1b1d590593178ff Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 17 Oct 2024 23:46:18 +1300 Subject: [PATCH 01/12] Removed mixing, use a simple regex instead. --- source/fab/tools/__init__.py | 5 +- source/fab/tools/compiler.py | 133 ++++++++---------------- source/fab/tools/compiler_wrapper.py | 5 +- tests/conftest.py | 4 +- tests/unit_tests/tools/test_compiler.py | 23 ++-- tests/unit_tests/tools/test_tool_box.py | 6 +- 6 files changed, 61 insertions(+), 115 deletions(-) diff --git a/source/fab/tools/__init__.py b/source/fab/tools/__init__.py index 45eb666f..baa06c01 100644 --- a/source/fab/tools/__init__.py +++ b/source/fab/tools/__init__.py @@ -10,8 +10,7 @@ from fab.tools.ar import Ar from fab.tools.category import Category from fab.tools.compiler import (CCompiler, Compiler, FortranCompiler, Gcc, - Gfortran, GnuVersionHandling, Icc, Ifort, - IntelVersionHandling) + Gfortran, Icc, Ifort) from fab.tools.compiler_wrapper import CompilerWrapper, Mpicc, Mpif90 from fab.tools.flags import Flags from fab.tools.linker import Linker @@ -39,10 +38,8 @@ "Gcc", "Gfortran", "Git", - "GnuVersionHandling", "Icc", "Ifort", - "IntelVersionHandling", "Linker", "Mpif90", "Mpicc", diff --git a/source/fab/tools/compiler.py b/source/fab/tools/compiler.py index 6566a292..6935a3ff 100644 --- a/source/fab/tools/compiler.py +++ b/source/fab/tools/compiler.py @@ -30,6 +30,9 @@ class Compiler(CompilerSuiteTool): :param name: name of the compiler. :param exec_name: name of the executable to start. :param suite: name of the compiler suite this tool belongs to. + :param version_regex: A regular expression that allows extraction of + the version number from the version output of the compiler. The + version is taken from the first group of a match. :param category: the Category (C_COMPILER or FORTRAN_COMPILER). :param compile_flag: the compilation flag to use when only requesting compilation (not linking). @@ -46,6 +49,7 @@ class Compiler(CompilerSuiteTool): def __init__(self, name: str, exec_name: Union[str, Path], suite: str, + version_regex: str, category: Category, mpi: bool = False, compile_flag: Optional[str] = None, @@ -60,6 +64,7 @@ def __init__(self, name: str, self._output_flag = output_flag if output_flag else "-o" self._openmp_flag = openmp_flag if openmp_flag else "" self.flags.extend(os.getenv("FFLAGS", "").split()) + self._version_regex = version_regex @property def mpi(self) -> bool: @@ -149,7 +154,14 @@ def get_version(self) -> Tuple[int, ...]: # Run the compiler to get the version and parse the output # The implementations depend on vendor output = self.run_version_command() - version_string = self.parse_version_output(self.category, output) + + # Multiline is required in case that the version number is the end + # of the string, otherwise the $ would not match the end of line + matches = re.search(self._version_regex, output, re.MULTILINE) + if not matches: + raise RuntimeError(f"Unexpected version output format for " + f"compiler '{self.name}': {output}") + version_string = matches.groups()[0] # Expect the version to be dot-separated integers. try: @@ -188,15 +200,6 @@ def run_version_command( raise RuntimeError(f"Error asking for version of compiler " f"'{self.name}'") from err - def parse_version_output(self, category: Category, - version_output: str) -> str: - ''' - Extract the numerical part from the version output. - Implemented in specific compilers. - ''' - raise NotImplementedError("The method `parse_version_output` must be " - "provided using a mixin.") - def get_version_string(self) -> str: """ Get a string representing the version of the given compiler. @@ -219,6 +222,8 @@ class CCompiler(Compiler): :param name: name of the compiler. :param exec_name: name of the executable to start. :param suite: name of the compiler suite. + :param version_regex: A regular expression that allows extraction of + the version number from the version output of the compiler. :param mpi: whether the compiler or linker support MPI. :param compile_flag: the compilation flag to use when only requesting compilation (not linking). @@ -229,6 +234,7 @@ class CCompiler(Compiler): # pylint: disable=too-many-arguments def __init__(self, name: str, exec_name: str, suite: str, + version_regex: str, mpi: bool = False, compile_flag: Optional[str] = None, output_flag: Optional[str] = None, @@ -236,7 +242,8 @@ def __init__(self, name: str, exec_name: str, suite: str, super().__init__(name, exec_name, suite, category=Category.C_COMPILER, mpi=mpi, compile_flag=compile_flag, output_flag=output_flag, - openmp_flag=openmp_flag) + openmp_flag=openmp_flag, + version_regex=version_regex) # ============================================================================ @@ -248,6 +255,8 @@ class FortranCompiler(Compiler): :param name: name of the compiler. :param exec_name: name of the executable to start. :param suite: name of the compiler suite. + :param version_regex: A regular expression that allows extraction of + the version number from the version output of the compiler. :param mpi: whether MPI is supported by this compiler or not. :param compile_flag: the compilation flag to use when only requesting compilation (not linking). @@ -262,6 +271,7 @@ class FortranCompiler(Compiler): # pylint: disable=too-many-arguments def __init__(self, name: str, exec_name: str, suite: str, + version_regex: str, mpi: bool = False, compile_flag: Optional[str] = None, output_flag: Optional[str] = None, @@ -273,7 +283,8 @@ def __init__(self, name: str, exec_name: str, suite: str, super().__init__(name=name, exec_name=exec_name, suite=suite, category=Category.FORTRAN_COMPILER, mpi=mpi, compile_flag=compile_flag, - output_flag=output_flag, openmp_flag=openmp_flag) + output_flag=output_flag, openmp_flag=openmp_flag, + version_regex=version_regex) self._module_folder_flag = (module_folder_flag if module_folder_flag else "") self._syntax_only_flag = syntax_only_flag @@ -327,45 +338,7 @@ def compile_file(self, input_file: Path, # ============================================================================ -class GnuVersionHandling(): - '''Mixin to handle version information from GNU compilers''' - - def parse_version_output(self, category: Category, - version_output: str) -> str: - ''' - Extract the numerical part from a GNU compiler's version output - - :param name: the compiler's name - :param category: the compiler's Category - :param version_output: the full version output from the compiler - :returns: the actual version as a string - - :raises RuntimeError: if the output is not in an expected format. - ''' - - # Expect the version to appear after some in parentheses, e.g. - # "GNU Fortran (...) n.n[.n, ...]" or # "gcc (...) n.n[.n, ...]" - if category is Category.FORTRAN_COMPILER: - name = "GNU Fortran" - else: - name = "gcc" - # A version number is a digit, followed by a sequence of digits and - # '.'', ending with a digit. It must then be followed by either the - # end of the string, or a space (e.g. "... 5.6 123456"). We can't use - # \b to determine the end, since then "1.2." would be matched - # excluding the dot (so it would become a valid 1.2) - exp = name + r" \(.*?\) (\d[\d\.]+\d)(?:$| )" - # Multiline is required in case that the version number is the - # end of the string, otherwise the $ would not match the end of line - matches = re.search(exp, version_output, re.MULTILINE) - if not matches: - raise RuntimeError(f"Unexpected version output format for " - f"compiler '{name}': {version_output}") - return matches.groups()[0] - - -# ============================================================================ -class Gcc(GnuVersionHandling, CCompiler): +class Gcc(CCompiler): '''Class for GNU's gcc compiler. :param name: name of this compiler. @@ -375,12 +348,18 @@ def __init__(self, name: str = "gcc", exec_name: str = "gcc", mpi: bool = False): + # A version number is a digit, followed by a sequence of digits and + # '.'', ending with a digit. It must then be followed by either the + # end of the string, or a space (e.g. "... 5.6 123456"). We can't use + # \b to determine the end, since then "1.2." would be matched + # excluding the dot (so it would become a valid 1.2) super().__init__(name, exec_name, suite="gnu", mpi=mpi, - openmp_flag="-fopenmp") + openmp_flag="-fopenmp", + version_regex=r"gcc \(.*?\) (\d[\d\.]+\d)(?:$| )") # ============================================================================ -class Gfortran(GnuVersionHandling, FortranCompiler): +class Gfortran(FortranCompiler): '''Class for GNU's gfortran compiler. :param name: name of this compiler. @@ -392,45 +371,13 @@ def __init__(self, name: str = "gfortran", super().__init__(name, exec_name, suite="gnu", openmp_flag="-fopenmp", module_folder_flag="-J", - syntax_only_flag="-fsyntax-only") + syntax_only_flag="-fsyntax-only", + version_regex=(r"GNU Fortran \(.*?\) " + r"(\d[\d\.]+\d)(?:$| )")) # ============================================================================ -class IntelVersionHandling(): - '''Mixin to handle version information from Intel compilers''' - - def parse_version_output(self, category: Category, - version_output: str) -> str: - ''' - Extract the numerical part from an Intel compiler's version output - - :param name: the compiler's name - :param version_output: the full version output from the compiler - :returns: the actual version as a string - - :raises RuntimeError: if the output is not in an expected format. - ''' - - # Expect the version to appear after some in parentheses, e.g. - # "icc (...) n.n[.n, ...]" or "ifort (...) n.n[.n, ...]" - if category == Category.C_COMPILER: - name = "icc" - else: - name = "ifort" - - # A version number is a digit, followed by a sequence of digits and - # '.'', ending with a digit. It must then be followed by a space. - exp = name + r" \(.*?\) (\d[\d\.]+\d) " - matches = re.search(exp, version_output) - - if not matches: - raise RuntimeError(f"Unexpected version output format for " - f"compiler '{name}': {version_output}") - return matches.groups()[0] - - -# ============================================================================ -class Icc(IntelVersionHandling, CCompiler): +class Icc(CCompiler): '''Class for the Intel's icc compiler. :param name: name of this compiler. @@ -439,11 +386,12 @@ class Icc(IntelVersionHandling, CCompiler): def __init__(self, name: str = "icc", exec_name: str = "icc"): super().__init__(name, exec_name, suite="intel-classic", - openmp_flag="-qopenmp") + openmp_flag="-qopenmp", + version_regex=r"icc \(ICC\) (\d[\d\.]+\d) ") # ============================================================================ -class Ifort(IntelVersionHandling, FortranCompiler): +class Ifort(FortranCompiler): '''Class for Intel's ifort compiler. :param name: name of this compiler. @@ -454,4 +402,5 @@ def __init__(self, name: str = "ifort", exec_name: str = "ifort"): super().__init__(name, exec_name, suite="intel-classic", module_folder_flag="-module", openmp_flag="-qopenmp", - syntax_only_flag="-syntax-only") + syntax_only_flag="-syntax-only", + version_regex=r"ifort \(IFORT\) (\d[\d\.]+\d) ") diff --git a/source/fab/tools/compiler_wrapper.py b/source/fab/tools/compiler_wrapper.py index e54f98ea..4dc24199 100644 --- a/source/fab/tools/compiler_wrapper.py +++ b/source/fab/tools/compiler_wrapper.py @@ -36,12 +36,9 @@ def __init__(self, name: str, exec_name: str, name=name, exec_name=exec_name, category=self._compiler.category, suite=self._compiler.suite, + version_regex=self._compiler._version_regex, mpi=mpi, availability_option=self._compiler.availability_option) - # We need to have the right version to parse the version output - # So we set this function based on the function that the - # wrapped compiler uses: - setattr(self, "parse_version_output", compiler.parse_version_output) def __str__(self): return f"{type(self).__name__}({self._compiler.name})" diff --git a/tests/conftest.py b/tests/conftest.py index 559d4f3b..86de6476 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,7 +18,8 @@ @pytest.fixture(name="mock_c_compiler") def fixture_mock_c_compiler(): '''Provides a mock C-compiler.''' - mock_compiler = CCompiler("mock_c_compiler", "mock_exec", "suite") + mock_compiler = CCompiler("mock_c_compiler", "mock_exec", "suite", + version_regex="something") mock_compiler.run = mock.Mock() mock_compiler._version = (1, 2, 3) mock_compiler._name = "mock_c_compiler" @@ -32,6 +33,7 @@ def fixture_mock_fortran_compiler(): '''Provides a mock Fortran-compiler.''' mock_compiler = FortranCompiler("mock_fortran_compiler", "mock_exec", "suite", module_folder_flag="", + version_regex="something", syntax_only_flag=None, compile_flag=None, output_flag=None, openmp_flag=None) mock_compiler.run = mock.Mock() diff --git a/tests/unit_tests/tools/test_compiler.py b/tests/unit_tests/tools/test_compiler.py index ac948246..ff4ec01b 100644 --- a/tests/unit_tests/tools/test_compiler.py +++ b/tests/unit_tests/tools/test_compiler.py @@ -20,7 +20,8 @@ def test_compiler(): '''Test the compiler constructor.''' - cc = Compiler("gcc", "gcc", "gnu", category=Category.C_COMPILER, openmp_flag="-fopenmp") + cc = Compiler("gcc", "gcc", "gnu", version_regex="some_regex", + category=Category.C_COMPILER, openmp_flag="-fopenmp") assert cc.category == Category.C_COMPILER assert cc._compile_flag == "-c" assert cc._output_flag == "-o" @@ -29,13 +30,9 @@ def test_compiler(): assert cc.suite == "gnu" assert not cc.mpi assert cc.openmp_flag == "-fopenmp" - with pytest.raises(NotImplementedError) as err: - cc.parse_version_output(Category.FORTRAN_COMPILER, "NOT NEEDED") - assert ("The method `parse_version_output` must be provided using a mixin." - in str(err.value)) fc = FortranCompiler("gfortran", "gfortran", "gnu", openmp_flag="-fopenmp", - module_folder_flag="-J") + version_regex="something", module_folder_flag="-J") assert fc._compile_flag == "-c" assert fc._output_flag == "-o" assert fc.category == Category.FORTRAN_COMPILER @@ -44,10 +41,6 @@ def test_compiler(): assert fc.flags == [] assert not fc.mpi assert fc.openmp_flag == "-fopenmp" - with pytest.raises(NotImplementedError) as err: - fc.parse_version_output(Category.FORTRAN_COMPILER, "NOT NEEDED") - assert ("The method `parse_version_output` must be provided using a mixin." - in str(err.value)) def test_compiler_check_available(): @@ -121,16 +114,19 @@ def test_compiler_with_env_fflags(): def test_compiler_syntax_only(): '''Tests handling of syntax only flags.''' fc = FortranCompiler("gfortran", "gfortran", "gnu", + version_regex="something", openmp_flag="-fopenmp", module_folder_flag="-J") # Empty since no flag is defined assert not fc.has_syntax_only fc = FortranCompiler("gfortran", "gfortran", "gnu", openmp_flag="-fopenmp", - module_folder_flag="-J", syntax_only_flag=None) + version_regex="something", module_folder_flag="-J", + syntax_only_flag=None) # Empty since no flag is defined assert not fc.has_syntax_only fc = FortranCompiler("gfortran", "gfortran", "gnu", + version_regex="something", openmp_flag="-fopenmp", module_folder_flag="-J", syntax_only_flag="-fsyntax-only") @@ -141,6 +137,7 @@ def test_compiler_syntax_only(): def test_compiler_without_openmp(): '''Tests that the openmp flag is not used when openmp is not enabled. ''' fc = FortranCompiler("gfortran", "gfortran", "gnu", + version_regex="something", openmp_flag="-fopenmp", module_folder_flag="-J", syntax_only_flag="-fsyntax-only") @@ -157,6 +154,7 @@ def test_compiler_with_openmp(): '''Tests that the openmp flag is used as expected if openmp is enabled. ''' fc = FortranCompiler("gfortran", "gfortran", "gnu", + version_regex="something", openmp_flag="-fopenmp", module_folder_flag="-J", syntax_only_flag="-fsyntax-only") @@ -172,7 +170,7 @@ def test_compiler_with_openmp(): def test_compiler_module_output(): '''Tests handling of module output_flags.''' fc = FortranCompiler("gfortran", "gfortran", suite="gnu", - module_folder_flag="-J") + version_regex="something", module_folder_flag="-J") fc.set_module_output_path("/module_out") assert fc._module_output_path == "/module_out" fc.run = mock.MagicMock() @@ -185,6 +183,7 @@ def test_compiler_module_output(): def test_compiler_with_add_args(): '''Tests that additional arguments are handled as expected.''' fc = FortranCompiler("gfortran", "gfortran", suite="gnu", + version_regex="something", openmp_flag="-fopenmp", module_folder_flag="-J") fc.set_module_output_path("/module_out") diff --git a/tests/unit_tests/tools/test_tool_box.py b/tests/unit_tests/tools/test_tool_box.py index b8e2e903..965fe4f0 100644 --- a/tests/unit_tests/tools/test_tool_box.py +++ b/tests/unit_tests/tools/test_tool_box.py @@ -43,9 +43,11 @@ def test_tool_box_add_tool_replacement(): warning can be disabled.''' tb = ToolBox() - mock_compiler1 = CCompiler("mock_c_compiler1", "mock_exec1", "suite") + mock_compiler1 = CCompiler("mock_c_compiler1", "mock_exec1", "suite", + version_regex="something") mock_compiler1._is_available = True - mock_compiler2 = CCompiler("mock_c_compiler2", "mock_exec2", "suite") + mock_compiler2 = CCompiler("mock_c_compiler2", "mock_exec2", "suite", + version_regex="something") mock_compiler2._is_available = True tb.add_tool(mock_compiler1) From 8ee10e83437d77bb73d4693c1645b2e8ab8fbf08 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Fri, 18 Oct 2024 15:07:11 +1300 Subject: [PATCH 02/12] Added support for ifx/icx compiler as intel-llvm class. --- source/fab/tools/__init__.py | 4 +- source/fab/tools/compiler.py | 30 ++++++++++ tests/unit_tests/tools/test_compiler.py | 77 ++++++++++++++++++++++++- 3 files changed, 109 insertions(+), 2 deletions(-) diff --git a/source/fab/tools/__init__.py b/source/fab/tools/__init__.py index baa06c01..42357f2a 100644 --- a/source/fab/tools/__init__.py +++ b/source/fab/tools/__init__.py @@ -10,7 +10,7 @@ from fab.tools.ar import Ar from fab.tools.category import Category from fab.tools.compiler import (CCompiler, Compiler, FortranCompiler, Gcc, - Gfortran, Icc, Ifort) + Gfortran, Icc, Icx, Ifort, Ifx) from fab.tools.compiler_wrapper import CompilerWrapper, Mpicc, Mpif90 from fab.tools.flags import Flags from fab.tools.linker import Linker @@ -39,7 +39,9 @@ "Gfortran", "Git", "Icc", + "Icx", "Ifort", + "Ifx", "Linker", "Mpif90", "Mpicc", diff --git a/source/fab/tools/compiler.py b/source/fab/tools/compiler.py index 6935a3ff..587dfa1a 100644 --- a/source/fab/tools/compiler.py +++ b/source/fab/tools/compiler.py @@ -390,6 +390,20 @@ def __init__(self, name: str = "icc", exec_name: str = "icc"): version_regex=r"icc \(ICC\) (\d[\d\.]+\d) ") +# ============================================================================ +class Icx(CCompiler): + '''Class for the Intel's new llvm based icx compiler. + + :param name: name of this compiler. + :param exec_name: name of the executable. + ''' + def __init__(self, name: str = "icx", exec_name: str = "icx"): + super().__init__(name, exec_name, suite="intel-llvm", + openmp_flag="-qopenmp", + version_regex=(r"Intel\(R\) oneAPI DPC\+\+/C\+\+ " + r"Compiler (\d[\d\.]+\d) ")) + + # ============================================================================ class Ifort(FortranCompiler): '''Class for Intel's ifort compiler. @@ -404,3 +418,19 @@ def __init__(self, name: str = "ifort", exec_name: str = "ifort"): openmp_flag="-qopenmp", syntax_only_flag="-syntax-only", version_regex=r"ifort \(IFORT\) (\d[\d\.]+\d) ") + + +# ============================================================================ +class Ifx(FortranCompiler): + '''Class for Intel's new ifx compiler. + + :param name: name of this compiler. + :param exec_name: name of the executable. + ''' + + def __init__(self, name: str = "ifx", exec_name: str = "ifx"): + super().__init__(name, exec_name, suite="intel-llvm", + module_folder_flag="-module", + openmp_flag="-qopenmp", + syntax_only_flag="-syntax-only", + version_regex=r"ifx \(IFORT\) (\d[\d\.]+\d) ") diff --git a/tests/unit_tests/tools/test_compiler.py b/tests/unit_tests/tools/test_compiler.py index ff4ec01b..761825d6 100644 --- a/tests/unit_tests/tools/test_compiler.py +++ b/tests/unit_tests/tools/test_compiler.py @@ -15,7 +15,7 @@ import pytest from fab.tools import (Category, CCompiler, Compiler, FortranCompiler, - Gcc, Gfortran, Icc, Ifort) + Gcc, Gfortran, Icc, Icx, Ifort, Ifx) def test_compiler(): @@ -626,3 +626,78 @@ def test_ifort_get_version_invalid_version(version): with pytest.raises(RuntimeError) as err: icc.get_version() assert "Unexpected version output format for compiler" in str(err.value) + + +# ============================================================================ +def test_icx(): + '''Tests the icx class.''' + icx = Icx() + assert icx.name == "icx" + assert isinstance(icx, CCompiler) + assert icx.category == Category.C_COMPILER + assert not icx.mpi + + +def test_icx_get_version_2023(): + '''Test icx 2023.0.0 version detection.''' + full_output = dedent(""" +Intel(R) oneAPI DPC++/C++ Compiler 2023.0.0 (2023.0.0.20221201) +Target: x86_64-unknown-linux-gnu +Thread model: posix +InstalledDir: /opt/intel/oneapi/compiler/2023.0.0/linux/bin-llvm +Configuration file: /opt/intel/oneapi/compiler/2023.0.0/linux/bin-llvm/../bin/icx.cfg + + """) + icx = Icx() + with mock.patch.object(icx, "run", mock.Mock(return_value=full_output)): + assert icx.get_version() == (2023, 0, 0) + + +def test_icx_get_version_with_icc_string(): + '''Tests the icx class with an icc version output.''' + full_output = dedent(""" + icc (ICC) 2021.10.0 20230609 + Copyright (C) 1985-2023 Intel Corporation. All rights reserved. + + """) + icx = Icx() + with mock.patch.object(icx, "run", mock.Mock(return_value=full_output)): + with pytest.raises(RuntimeError) as err: + icx.get_version() + assert "Unexpected version output format for compiler" in str(err.value) + + +# ============================================================================ +def test_ifx(): + '''Tests the ifx class.''' + ifx = Ifx() + assert ifx.name == "ifx" + assert isinstance(ifx, FortranCompiler) + assert ifx.category == Category.FORTRAN_COMPILER + assert not ifx.mpi + + +def test_ifx_get_version_2023(): + '''Test ifx 2023.0.0 version detection.''' + full_output = dedent(""" +ifx (IFORT) 2023.0.0 20221201 +Copyright (C) 1985-2022 Intel Corporation. All rights reserved. + + """) + ifx = Ifx() + with mock.patch.object(ifx, "run", mock.Mock(return_value=full_output)): + assert ifx.get_version() == (2023, 0, 0) + + +def test_ifx_get_version_with_icc_string(): + '''Tests the ifx class with an icc version output.''' + full_output = dedent(""" + icc (ICC) 2021.10.0 20230609 + Copyright (C) 1985-2023 Intel Corporation. All rights reserved. + + """) + ifx = Ifx() + with mock.patch.object(ifx, "run", mock.Mock(return_value=full_output)): + with pytest.raises(RuntimeError) as err: + ifx.get_version() + assert "Unexpected version output format for compiler" in str(err.value) From d7b20083a19ceea025dd72fc43202dc82a971621 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Fri, 18 Oct 2024 16:40:06 +1300 Subject: [PATCH 03/12] Added support for nvidia compiler. --- source/fab/tools/__init__.py | 4 +- source/fab/tools/compiler.py | 92 ++++++++++++++--- tests/unit_tests/tools/test_compiler.py | 132 +++++++++++++++++++++--- 3 files changed, 202 insertions(+), 26 deletions(-) diff --git a/source/fab/tools/__init__.py b/source/fab/tools/__init__.py index 42357f2a..ec2a6e23 100644 --- a/source/fab/tools/__init__.py +++ b/source/fab/tools/__init__.py @@ -10,7 +10,7 @@ from fab.tools.ar import Ar from fab.tools.category import Category from fab.tools.compiler import (CCompiler, Compiler, FortranCompiler, Gcc, - Gfortran, Icc, Icx, Ifort, Ifx) + Gfortran, Icc, Icx, Ifort, Ifx, Nvc, Nvfortran) from fab.tools.compiler_wrapper import CompilerWrapper, Mpicc, Mpif90 from fab.tools.flags import Flags from fab.tools.linker import Linker @@ -45,6 +45,8 @@ "Linker", "Mpif90", "Mpicc", + "Nvc", + "Nvfortran", "Preprocessor", "Psyclone", "Rsync", diff --git a/source/fab/tools/compiler.py b/source/fab/tools/compiler.py index 587dfa1a..b1f2d74f 100644 --- a/source/fab/tools/compiler.py +++ b/source/fab/tools/compiler.py @@ -377,6 +377,8 @@ def __init__(self, name: str = "gfortran", # ============================================================================ +# intel-classic +# class Icc(CCompiler): '''Class for the Intel's icc compiler. @@ -391,6 +393,24 @@ def __init__(self, name: str = "icc", exec_name: str = "icc"): # ============================================================================ +class Ifort(FortranCompiler): + '''Class for Intel's ifort compiler. + + :param name: name of this compiler. + :param exec_name: name of the executable. + ''' + + def __init__(self, name: str = "ifort", exec_name: str = "ifort"): + super().__init__(name, exec_name, suite="intel-classic", + module_folder_flag="-module", + openmp_flag="-qopenmp", + syntax_only_flag="-syntax-only", + version_regex=r"ifort \(IFORT\) (\d[\d\.]+\d) ") + + +# ============================================================================ +# intel-llvm +# class Icx(CCompiler): '''Class for the Intel's new llvm based icx compiler. @@ -405,32 +425,80 @@ def __init__(self, name: str = "icx", exec_name: str = "icx"): # ============================================================================ -class Ifort(FortranCompiler): - '''Class for Intel's ifort compiler. +class Ifx(FortranCompiler): + '''Class for Intel's new ifx compiler. :param name: name of this compiler. :param exec_name: name of the executable. ''' - def __init__(self, name: str = "ifort", exec_name: str = "ifort"): - super().__init__(name, exec_name, suite="intel-classic", + def __init__(self, name: str = "ifx", exec_name: str = "ifx"): + super().__init__(name, exec_name, suite="intel-llvm", module_folder_flag="-module", openmp_flag="-qopenmp", syntax_only_flag="-syntax-only", - version_regex=r"ifort \(IFORT\) (\d[\d\.]+\d) ") + version_regex=r"ifx \(IFORT\) (\d[\d\.]+\d) ") # ============================================================================ -class Ifx(FortranCompiler): - '''Class for Intel's new ifx compiler. +# nvidia +# +class Nvc(CCompiler): + '''Class for Nvidia's nvc compiler. Nvc has a '-' in the + version number. In order to get this, we overwrite run_version_command + and replace any '-' with a '.' :param name: name of this compiler. :param exec_name: name of the executable. ''' - def __init__(self, name: str = "ifx", exec_name: str = "ifx"): - super().__init__(name, exec_name, suite="intel-llvm", + def __init__(self, name: str = "nvc", exec_name: str = "nvc"): + super().__init__(name, exec_name, suite="nvidia", + openmp_flag="-mp", + version_regex=r"nvc (\d[\d\.-]+\d)") + + def run_version_command( + self, version_command: Optional[str] = '--version') -> str: + '''Run the compiler's command to get its version. This implementation + runs the function in the base class, and changes any '-' into a + '.' to support nvidia version numbers which have dashes, e.g. 23.5-0. + + :param version_command: The compiler argument used to get version info. + + :returns: The output from the version command, with any '-' replaced + with '.' + ''' + version_string = super().run_version_command() + return version_string.replace("-", ".") + + +# ============================================================================ +class Nvfortran(FortranCompiler): + '''Class for Nvidia's nvfortran compiler. Nvfortran has a '-' in the + version number. In order to get this, we overwrite run_version_command + and replace any '-' with a '.' + + :param name: name of this compiler. + :param exec_name: name of the executable. + ''' + + def __init__(self, name: str = "nvfortran", exec_name: str = "nvfortran"): + super().__init__(name, exec_name, suite="nvidia", module_folder_flag="-module", - openmp_flag="-qopenmp", - syntax_only_flag="-syntax-only", - version_regex=r"ifx \(IFORT\) (\d[\d\.]+\d) ") + openmp_flag="-mp", + syntax_only_flag="-Msyntax-only", + version_regex=r"nvfortran (\d[\d\.-]+\d)") + + def run_version_command( + self, version_command: Optional[str] = '--version') -> str: + '''Run the compiler's command to get its version. This implementation + runs the function in the base class, and changes any '-' into a + '.' to support nvidia version numbers which have dashes, e.g. 23.5-0. + + :param version_command: The compiler argument used to get version info. + + :returns: The output from the version command, with any '-' replaced + with '.' + ''' + version_string = super().run_version_command() + return version_string.replace("-", ".") diff --git a/tests/unit_tests/tools/test_compiler.py b/tests/unit_tests/tools/test_compiler.py index 761825d6..4f0566bd 100644 --- a/tests/unit_tests/tools/test_compiler.py +++ b/tests/unit_tests/tools/test_compiler.py @@ -15,7 +15,7 @@ import pytest from fab.tools import (Category, CCompiler, Compiler, FortranCompiler, - Gcc, Gfortran, Icc, Icx, Ifort, Ifx) + Gcc, Gfortran, Icc, Icx, Ifort, Ifx, Nvc, Nvfortran) def test_compiler(): @@ -205,6 +205,9 @@ def test_compiler_with_add_args(): openmp=True, syntax_only=True) +# ============================================================================ +# Test version number handling +# ============================================================================ def test_get_version_string(): '''Tests the get_version_string() method. ''' @@ -365,6 +368,8 @@ def test_get_version_bad_result_is_not_cached(): assert c.run.called +# ============================================================================ +# gcc # ============================================================================ def test_gcc(): '''Tests the gcc class.''' @@ -400,6 +405,8 @@ def test_gcc_get_version_with_icc_string(): assert "Unexpected version output format for compiler" in str(err.value) +# ============================================================================ +# gfortran # ============================================================================ def test_gfortran(): '''Tests the gfortran class.''' @@ -484,7 +491,8 @@ def test_gfortran_get_version_12(): """) gfortran = Gfortran() - with mock.patch.object(gfortran, "run", mock.Mock(return_value=full_output)): + with mock.patch.object(gfortran, "run", + mock.Mock(return_value=full_output)): assert gfortran.get_version() == (12, 1, 0) @@ -496,12 +504,16 @@ def test_gfortran_get_version_with_ifort_string(): """) gfortran = Gfortran() - with mock.patch.object(gfortran, "run", mock.Mock(return_value=full_output)): + with mock.patch.object(gfortran, "run", + mock.Mock(return_value=full_output)): with pytest.raises(RuntimeError) as err: gfortran.get_version() - assert "Unexpected version output format for compiler" in str(err.value) + assert ("Unexpected version output format for compiler" + in str(err.value)) +# ============================================================================ +# icc # ============================================================================ def test_icc(): '''Tests the icc class.''' @@ -534,9 +546,12 @@ def test_icc_get_version_with_gcc_string(): with mock.patch.object(icc, "run", mock.Mock(return_value=full_output)): with pytest.raises(RuntimeError) as err: icc.get_version() - assert "Unexpected version output format for compiler" in str(err.value) + assert ("Unexpected version output format for compiler" + in str(err.value)) +# ============================================================================ +# ifort # ============================================================================ def test_ifort(): '''Tests the ifort class.''' @@ -606,7 +621,8 @@ def test_ifort_get_version_with_icc_string(): with mock.patch.object(ifort, "run", mock.Mock(return_value=full_output)): with pytest.raises(RuntimeError) as err: ifort.get_version() - assert "Unexpected version output format for compiler" in str(err.value) + assert ("Unexpected version output format for compiler" + in str(err.value)) @pytest.mark.parametrize("version", ["5.15f.2", @@ -625,9 +641,12 @@ def test_ifort_get_version_invalid_version(version): with mock.patch.object(icc, "run", mock.Mock(return_value=full_output)): with pytest.raises(RuntimeError) as err: icc.get_version() - assert "Unexpected version output format for compiler" in str(err.value) + assert ("Unexpected version output format for compiler" + in str(err.value)) +# ============================================================================ +# icx # ============================================================================ def test_icx(): '''Tests the icx class.''' @@ -645,7 +664,8 @@ def test_icx_get_version_2023(): Target: x86_64-unknown-linux-gnu Thread model: posix InstalledDir: /opt/intel/oneapi/compiler/2023.0.0/linux/bin-llvm -Configuration file: /opt/intel/oneapi/compiler/2023.0.0/linux/bin-llvm/../bin/icx.cfg +Configuration file: /opt/intel/oneapi/compiler/2023.0.0/linux/bin-llvm/""" + """../bin/icx.cfg """) icx = Icx() @@ -664,9 +684,12 @@ def test_icx_get_version_with_icc_string(): with mock.patch.object(icx, "run", mock.Mock(return_value=full_output)): with pytest.raises(RuntimeError) as err: icx.get_version() - assert "Unexpected version output format for compiler" in str(err.value) + assert ("Unexpected version output format for compiler" + in str(err.value)) +# ============================================================================ +# ifx # ============================================================================ def test_ifx(): '''Tests the ifx class.''' @@ -689,15 +712,98 @@ def test_ifx_get_version_2023(): assert ifx.get_version() == (2023, 0, 0) -def test_ifx_get_version_with_icc_string(): +def test_ifx_get_version_with_ifort_string(): '''Tests the ifx class with an icc version output.''' full_output = dedent(""" - icc (ICC) 2021.10.0 20230609 - Copyright (C) 1985-2023 Intel Corporation. All rights reserved. + ifort (IFORT) 19.0.0.117 20180804 + Copyright (C) 1985-2018 Intel Corporation. All rights reserved. """) ifx = Ifx() with mock.patch.object(ifx, "run", mock.Mock(return_value=full_output)): with pytest.raises(RuntimeError) as err: ifx.get_version() - assert "Unexpected version output format for compiler" in str(err.value) + assert ("Unexpected version output format for compiler" + in str(err.value)) + + +# ============================================================================ +# nvc +# ============================================================================ +def test_nvc(): + '''Tests the nvc class.''' + nvc = Nvc() + assert nvc.name == "nvc" + assert isinstance(nvc, CCompiler) + assert nvc.category == Category.C_COMPILER + assert not nvc.mpi + + +def test_nvc_get_version_2023(): + '''Test nvc .23.5 version detection.''' + full_output = dedent(""" + +nvc 23.5-0 64-bit target on x86-64 Linux -tp icelake-server +NVIDIA Compilers and Tools +Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + """) + nvc = Nvc() + with mock.patch.object(nvc, "run", mock.Mock(return_value=full_output)): + assert nvc.get_version() == (23, 5, 0) + + +def test_nvc_get_version_with_icc_string(): + '''Tests the nvc class with an icc version output.''' + full_output = dedent(""" + icc (ICC) 2021.10.0 20230609 + Copyright (C) 1985-2023 Intel Corporation. All rights reserved. + + """) + nvc = Nvc() + with mock.patch.object(nvc, "run", mock.Mock(return_value=full_output)): + with pytest.raises(RuntimeError) as err: + nvc.get_version() + assert ("Unexpected version output format for compiler" + in str(err.value)) + + +# ============================================================================ +# nvfortran +# ============================================================================ +def test_nvfortran(): + '''Tests the nvfortran class.''' + nvfortran = Nvfortran() + assert nvfortran.name == "nvfortran" + assert isinstance(nvfortran, FortranCompiler) + assert nvfortran.category == Category.FORTRAN_COMPILER + assert not nvfortran.mpi + + +def test_nvfortran_get_version_2023(): + '''Test nvfortran .23.5 version detection.''' + full_output = dedent(""" + +nvfortran 23.5-0 64-bit target on x86-64 Linux -tp icelake-server +NVIDIA Compilers and Tools +Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + """) + nvfortran = Nvfortran() + with mock.patch.object(nvfortran, "run", + mock.Mock(return_value=full_output)): + assert nvfortran.get_version() == (23, 5, 0) + + +def test_nvfortran_get_version_with_ifort_string(): + '''Tests the nvfortran class with an icc version output.''' + full_output = dedent(""" + ifort (IFORT) 19.0.0.117 20180804 + Copyright (C) 1985-2018 Intel Corporation. All rights reserved. + + """) + nvfortran = Nvfortran() + with mock.patch.object(nvfortran, "run", + mock.Mock(return_value=full_output)): + with pytest.raises(RuntimeError) as err: + nvfortran.get_version() + assert ("Unexpected version output format for compiler" + in str(err.value)) From 9005b3b6f1c970eaa0b565f8b30a9e7d79f04753 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Fri, 18 Oct 2024 18:18:03 +1300 Subject: [PATCH 04/12] Add preliminary support for Cray compiler. --- source/fab/tools/__init__.py | 7 +- source/fab/tools/compiler.py | 50 ++++++++- tests/unit_tests/tools/test_compiler.py | 129 ++++++++++++++++++++++-- 3 files changed, 174 insertions(+), 12 deletions(-) diff --git a/source/fab/tools/__init__.py b/source/fab/tools/__init__.py index ec2a6e23..4dc59d14 100644 --- a/source/fab/tools/__init__.py +++ b/source/fab/tools/__init__.py @@ -9,8 +9,9 @@ from fab.tools.ar import Ar from fab.tools.category import Category -from fab.tools.compiler import (CCompiler, Compiler, FortranCompiler, Gcc, - Gfortran, Icc, Icx, Ifort, Ifx, Nvc, Nvfortran) +from fab.tools.compiler import (CCompiler, Compiler, Craycc, Crayftn, + FortranCompiler, Gcc, Gfortran, Icc, + Icx, Ifort, Ifx, Nvc, Nvfortran) from fab.tools.compiler_wrapper import CompilerWrapper, Mpicc, Mpif90 from fab.tools.flags import Flags from fab.tools.linker import Linker @@ -31,6 +32,8 @@ "CompilerWrapper", "Cpp", "CppFortran", + "Craycc", + "Crayftn", "Fcm", "Flags", "FortranCompiler", diff --git a/source/fab/tools/compiler.py b/source/fab/tools/compiler.py index b1f2d74f..0f10c01a 100644 --- a/source/fab/tools/compiler.py +++ b/source/fab/tools/compiler.py @@ -158,11 +158,12 @@ def get_version(self) -> Tuple[int, ...]: # Multiline is required in case that the version number is the end # of the string, otherwise the $ would not match the end of line matches = re.search(self._version_regex, output, re.MULTILINE) + print("XXX", output, matches) if not matches: raise RuntimeError(f"Unexpected version output format for " f"compiler '{self.name}': {output}") version_string = matches.groups()[0] - + print("YYY", matches.groups(), version_string) # Expect the version to be dot-separated integers. try: version = tuple(int(x) for x in version_string.split('.')) @@ -337,6 +338,8 @@ def compile_file(self, input_file: Path, add_flags=params) +# ============================================================================ +# Gnu # ============================================================================ class Gcc(CCompiler): '''Class for GNU's gcc compiler. @@ -378,7 +381,7 @@ def __init__(self, name: str = "gfortran", # ============================================================================ # intel-classic -# +# ============================================================================ class Icc(CCompiler): '''Class for the Intel's icc compiler. @@ -410,7 +413,7 @@ def __init__(self, name: str = "ifort", exec_name: str = "ifort"): # ============================================================================ # intel-llvm -# +# ============================================================================ class Icx(CCompiler): '''Class for the Intel's new llvm based icx compiler. @@ -442,7 +445,7 @@ def __init__(self, name: str = "ifx", exec_name: str = "ifx"): # ============================================================================ # nvidia -# +# ============================================================================ class Nvc(CCompiler): '''Class for Nvidia's nvc compiler. Nvc has a '-' in the version number. In order to get this, we overwrite run_version_command @@ -502,3 +505,42 @@ def run_version_command( ''' version_string = super().run_version_command() return version_string.replace("-", ".") + + +# ============================================================================ +# Cray compiler +# ============================================================================ +class Craycc(CCompiler): + '''Class for the native Cray C compiler. Cray has two different compilers. + Older ones have as version number: + Cray C : Version 8.7.0 Tue Jul 23, 2024 07:39:46 + Newer compiler (several lines, the important one): + Cray clang version 15.0.1 (66f7391d6a03cf932f321b9f6b1d8612ef5f362c) + We use the beginning ("cray c") to identify the compiler, which works for + both cray c and cray clang. Then we ignore non-numbers, to reach the + version number which is then extracted. + + :param name: name of this compiler. + :param exec_name: name of the executable. + ''' + def __init__(self, name: str = "craycc", exec_name: str = "craycc"): + super().__init__(name, exec_name, suite="cray", mpi=True, + openmp_flag="-qopenmp", + version_regex=r"Cray [Cc][^\d]* (\d[\d\.]+\d) ") + + +# ============================================================================ +class Crayftn(FortranCompiler): + '''Class for the native Cray Fortran compiler. + + :param name: name of this compiler. + :param exec_name: name of the executable. + ''' + + def __init__(self, name: str = "crayftn", exec_name: str = "crayftn"): + super().__init__(name, exec_name, suite="cray", mpi=True, + module_folder_flag="-module", + openmp_flag="-qopenmp", + syntax_only_flag="-syntax-only", + version_regex=(r"Cray Fortran : Version " + r"(\d[\d\.]+\d) ")) diff --git a/tests/unit_tests/tools/test_compiler.py b/tests/unit_tests/tools/test_compiler.py index 4f0566bd..6f3138dc 100644 --- a/tests/unit_tests/tools/test_compiler.py +++ b/tests/unit_tests/tools/test_compiler.py @@ -14,8 +14,9 @@ import pytest -from fab.tools import (Category, CCompiler, Compiler, FortranCompiler, - Gcc, Gfortran, Icc, Icx, Ifort, Ifx, Nvc, Nvfortran) +from fab.tools import (Category, CCompiler, Compiler, Craycc, Crayftn, + FortranCompiler, Gcc, Gfortran, Icc, Icx, Ifort, Ifx, + Nvc, Nvfortran) def test_compiler(): @@ -739,8 +740,8 @@ def test_nvc(): assert not nvc.mpi -def test_nvc_get_version_2023(): - '''Test nvc .23.5 version detection.''' +def test_nvc_get_version_23_5_0(): + '''Test nvc 23.5.0 version detection.''' full_output = dedent(""" nvc 23.5-0 64-bit target on x86-64 Linux -tp icelake-server @@ -779,8 +780,8 @@ def test_nvfortran(): assert not nvfortran.mpi -def test_nvfortran_get_version_2023(): - '''Test nvfortran .23.5 version detection.''' +def test_nvfortran_get_version_23_5_0(): + '''Test nvfortran 23.5 version detection.''' full_output = dedent(""" nvfortran 23.5-0 64-bit target on x86-64 Linux -tp icelake-server @@ -807,3 +808,119 @@ def test_nvfortran_get_version_with_ifort_string(): nvfortran.get_version() assert ("Unexpected version output format for compiler" in str(err.value)) + + +# ============================================================================ +# Craycc +# ============================================================================ +def test_craycc(): + '''Tests the Craycc class.''' + craycc = Craycc() + assert craycc.name == "craycc" + assert isinstance(craycc, CCompiler) + assert craycc.category == Category.C_COMPILER + assert craycc.mpi + + +def test_craycc_get_version_8_7_0(): + '''Test craycc .23.5 version detection.''' + full_output = dedent(""" +Cray C : Version 8.7.0 Tue Jul 23, 2024 07:39:46 + + """) + craycc = Craycc() + with mock.patch.object(craycc, "run", mock.Mock(return_value=full_output)): + assert craycc.get_version() == (8, 7, 0) + + +def test_craycc_get_version_2023(): + '''Test craycc .23.5 version detection.''' + full_output = dedent(""" +Cray clang version 15.0.1 (66f7391d6a03cf932f321b9f6b1d8612ef5f362c) + +Target: x86_64-unknown-linux-gnu + +Thread model: posix + +InstalledDir: /opt/cray/pe/cce/15.0.1/cce-clang/x86_64/share/../bin + +Found candidate GCC installation: /opt/gcc/10.3.0/snos/lib/gcc/x86_64-""" + """suse-linux/10.3.0 + +Selected GCC installation: /opt/gcc/10.3.0/snos/lib/gcc/x86_64-suse-""" + """linux/10.3.0 + +Candidate multilib: .;@m64 + +Selected multilib: .;@m64 + +OFFICIAL + """) + craycc = Craycc() + with mock.patch.object(craycc, "run", mock.Mock(return_value=full_output)): + assert craycc.get_version() == (15, 0, 1) + + +def test_craycc_get_version_with_icc_string(): + '''Tests the Craycc class with an icc version output.''' + full_output = dedent(""" + icc (ICC) 2021.10.0 20230609 + Copyright (C) 1985-2023 Intel Corporation. All rights reserved. + + """) + craycc = Craycc() + with mock.patch.object(craycc, "run", mock.Mock(return_value=full_output)): + with pytest.raises(RuntimeError) as err: + craycc.get_version() + assert ("Unexpected version output format for compiler" + in str(err.value)) + + +# ============================================================================ +# Crayftn +# ============================================================================ +def test_crayftn(): + '''Tests the Crayftn class.''' + crayftn = Crayftn() + assert crayftn.name == "crayftn" + assert isinstance(crayftn, FortranCompiler) + assert crayftn.category == Category.FORTRAN_COMPILER + assert crayftn.mpi + + +def test_crayftn_get_version_8_7_0(): + '''Test crayftn .23.5 version detection.''' + full_output = dedent(""" +Cray Fortran : Version 8.7.0 Tue Jul 23, 2024 07:39:25 + """) + crayftn = Crayftn() + with mock.patch.object(crayftn, "run", + mock.Mock(return_value=full_output)): + assert crayftn.get_version() == (8, 7, 0) + + +def test_crayftn_get_version_15_0_1(): + '''Test Crayftn 15.0.1 version detection.''' + full_output = dedent(""" +Cray Fortran : Version 15.0.1 Tue Jul 23, 2024 07:39:25 + """) + crayftn = Crayftn() + with mock.patch.object(crayftn, "run", + mock.Mock(return_value=full_output)): + assert crayftn.get_version() == (15, 0, 1) + + +def test_crayftn_get_version_with_ifort_string(): + '''Tests the crayftn class with an icc version output.''' + full_output = dedent(""" + ifort (IFORT) 19.0.0.117 20180804 + Copyright (C) 1985-2018 Intel Corporation. All rights reserved. + + """) + crayftn = Crayftn() + with mock.patch.object(crayftn, "run", + mock.Mock(return_value=full_output)): + with pytest.raises(RuntimeError) as err: + crayftn.get_version() + assert ("Unexpected version output format for compiler" + in str(err.value)) From 8771e8068823ce79e7d630405bd3dba4dfda2d30 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Fri, 18 Oct 2024 19:02:58 +1300 Subject: [PATCH 05/12] Added Cray compiler wrapper ftn and cc. --- source/fab/tools/__init__.py | 5 +- source/fab/tools/compiler.py | 2 - source/fab/tools/compiler_wrapper.py | 24 +++++++++ source/fab/tools/tool_repository.py | 31 ++++++++---- .../unit_tests/tools/test_compiler_wrapper.py | 49 ++++++++++++++++++- 5 files changed, 97 insertions(+), 14 deletions(-) diff --git a/source/fab/tools/__init__.py b/source/fab/tools/__init__.py index 4dc59d14..4ee807da 100644 --- a/source/fab/tools/__init__.py +++ b/source/fab/tools/__init__.py @@ -12,7 +12,8 @@ from fab.tools.compiler import (CCompiler, Compiler, Craycc, Crayftn, FortranCompiler, Gcc, Gfortran, Icc, Icx, Ifort, Ifx, Nvc, Nvfortran) -from fab.tools.compiler_wrapper import CompilerWrapper, Mpicc, Mpif90 +from fab.tools.compiler_wrapper import (CompilerWrapper, CrayCc, CrayFtn, + Mpicc, Mpif90) from fab.tools.flags import Flags from fab.tools.linker import Linker from fab.tools.psyclone import Psyclone @@ -33,7 +34,9 @@ "Cpp", "CppFortran", "Craycc", + "CrayCc", "Crayftn", + "CrayFtn", "Fcm", "Flags", "FortranCompiler", diff --git a/source/fab/tools/compiler.py b/source/fab/tools/compiler.py index 0f10c01a..84cd3ea3 100644 --- a/source/fab/tools/compiler.py +++ b/source/fab/tools/compiler.py @@ -158,12 +158,10 @@ def get_version(self) -> Tuple[int, ...]: # Multiline is required in case that the version number is the end # of the string, otherwise the $ would not match the end of line matches = re.search(self._version_regex, output, re.MULTILINE) - print("XXX", output, matches) if not matches: raise RuntimeError(f"Unexpected version output format for " f"compiler '{self.name}': {output}") version_string = matches.groups()[0] - print("YYY", matches.groups(), version_string) # Expect the version to be dot-separated integers. try: version = tuple(int(x) for x in version_string.split('.')) diff --git a/source/fab/tools/compiler_wrapper.py b/source/fab/tools/compiler_wrapper.py index 4dc24199..1b9b35ac 100644 --- a/source/fab/tools/compiler_wrapper.py +++ b/source/fab/tools/compiler_wrapper.py @@ -193,3 +193,27 @@ class Mpicc(CompilerWrapper): def __init__(self, compiler: Compiler): super().__init__(name=f"mpicc-{compiler.name}", exec_name="mpicc", compiler=compiler, mpi=True) + + +# ============================================================================ +class CrayFtn(CompilerWrapper): + '''Class for the Cray Fortran compiler wrapper. + + :param compiler: the compiler that the ftn wrapper will use. + ''' + + def __init__(self, compiler: Compiler): + super().__init__(name=f"crayftn-{compiler.name}", + exec_name="ftn", compiler=compiler, mpi=True) + + +# ============================================================================ +class CrayCc(CompilerWrapper): + '''Class for the Cray C compiler wrapper + + :param compiler: the compiler that the mpicc wrapper will use. + ''' + + def __init__(self, compiler: Compiler): + super().__init__(name=f"craycc-{compiler.name}", + exec_name="cc", compiler=compiler, mpi=True) diff --git a/source/fab/tools/tool_repository.py b/source/fab/tools/tool_repository.py index 7d3aa754..e18de0c4 100644 --- a/source/fab/tools/tool_repository.py +++ b/source/fab/tools/tool_repository.py @@ -17,8 +17,12 @@ from fab.tools.tool import Tool from fab.tools.category import Category from fab.tools.compiler import Compiler +from fab.tools.compiler_wrapper import CrayCc, CrayFtn, Mpif90, Mpicc from fab.tools.linker import Linker from fab.tools.versioning import Fcm, Git, Subversion +from fab.tools import (Ar, Cpp, CppFortran, Craycc, Crayftn, + Gcc, Gfortran, Icc, Icx, Ifort, Ifx, + Nvc, Nvfortran, Psyclone, Rsync) class ToolRepository(dict): @@ -57,26 +61,35 @@ def __init__(self): # Add the FAB default tools: # TODO: sort the defaults so that they actually work (since not all - # tools FAB knows about are available). For now, disable Fpp: - # We get circular dependencies if imported at top of the file: - # pylint: disable=import-outside-toplevel - from fab.tools import (Ar, Cpp, CppFortran, Gcc, Gfortran, - Icc, Ifort, Psyclone, Rsync) - - for cls in [Gcc, Icc, Gfortran, Ifort, Cpp, CppFortran, - Fcm, Git, Subversion, Ar, Psyclone, Rsync]: + # tools FAB knows about are available). For now, disable Fpp (by not + # adding it). IF someone actually uses it it can added. + for cls in [Craycc, Crayftn, + Gcc, Gfortran, + Icc, Icx, Ifort, Ifx, + Nvc, Nvfortran, + Cpp, CppFortran, + Ar, Fcm, Git, Psyclone, Rsync, Subversion]: self.add_tool(cls()) - from fab.tools.compiler_wrapper import Mpif90, Mpicc + # Now create the potential mpif90 and Cray ftn wrapper all_fc = self[Category.FORTRAN_COMPILER][:] for fc in all_fc: mpif90 = Mpif90(fc) self.add_tool(mpif90) + # I assume cray has (besides cray) only support for gfortran/ifort + if fc.name in ["gfortran", "ifort"]: + crayftn = CrayFtn(fc) + self.add_tool(crayftn) + # Now create the potential mpicc and Cray cc wrapper all_cc = self[Category.C_COMPILER][:] for cc in all_cc: mpicc = Mpicc(cc) self.add_tool(mpicc) + # I assume cray has (besides cray) only support for gfortran/ifort + if cc.name in ["gcc", "icc"]: + craycc = CrayCc(cc) + self.add_tool(craycc) def add_tool(self, tool: Tool): '''Creates an instance of the specified class and adds it diff --git a/tests/unit_tests/tools/test_compiler_wrapper.py b/tests/unit_tests/tools/test_compiler_wrapper.py index 11fdde57..42ee31ab 100644 --- a/tests/unit_tests/tools/test_compiler_wrapper.py +++ b/tests/unit_tests/tools/test_compiler_wrapper.py @@ -12,8 +12,9 @@ import pytest -from fab.tools import (Category, CompilerWrapper, Gcc, Gfortran, Icc, Ifort, - Mpicc, Mpif90, ToolRepository) +from fab.tools import (Category, CompilerWrapper, CrayCc, CrayFtn, + Gcc, Gfortran, Icc, Ifort, Mpicc, Mpif90, + ToolRepository) def test_compiler_wrapper_compiler_getter(): @@ -346,3 +347,47 @@ def test_compiler_wrapper_mpi_ifort(): assert mpi_ifort.category == Category.FORTRAN_COMPILER assert mpi_ifort.mpi assert mpi_ifort.suite == "intel-classic" + + +def test_compiler_wrapper_cray_icc(): + '''Tests the Cray wrapper for icc.''' + craycc = CrayCc(Icc()) + assert craycc.name == "craycc-icc" + assert str(craycc) == "CrayCc(icc)" + assert isinstance(craycc, CompilerWrapper) + assert craycc.category == Category.C_COMPILER + assert craycc.mpi + assert craycc.suite == "intel-classic" + + +def test_compiler_wrapper_cray_ifort(): + '''Tests the Cray wrapper for ifort.''' + crayftn = CrayFtn(Ifort()) + assert crayftn.name == "crayftn-ifort" + assert str(crayftn) == "CrayFtn(ifort)" + assert isinstance(crayftn, CompilerWrapper) + assert crayftn.category == Category.FORTRAN_COMPILER + assert crayftn.mpi + assert crayftn.suite == "intel-classic" + + +def test_compiler_wrapper_cray_gcc(): + '''Tests the Cray wrapper for gcc.''' + craycc = CrayCc(Gcc()) + assert craycc.name == "craycc-gcc" + assert str(craycc) == "CrayCc(gcc)" + assert isinstance(craycc, CompilerWrapper) + assert craycc.category == Category.C_COMPILER + assert craycc.mpi + assert craycc.suite == "gnu" + + +def test_compiler_wrapper_cray_gfortran(): + '''Tests the Cray wrapper for gfortran.''' + crayftn = CrayFtn(Gfortran()) + assert crayftn.name == "crayftn-gfortran" + assert str(crayftn) == "CrayFtn(gfortran)" + assert isinstance(crayftn, CompilerWrapper) + assert crayftn.category == Category.FORTRAN_COMPILER + assert crayftn.mpi + assert crayftn.suite == "gnu" From 01880508947b7159f627c6c75fb11948f62218ee Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Tue, 22 Oct 2024 14:06:54 +1100 Subject: [PATCH 06/12] Follow a more consistent naming scheme for crays, even though the native compiler names are longer (crayftn-cray, craycc-cray). --- source/fab/tools/compiler.py | 21 +++++++++++++-------- tests/unit_tests/tools/test_compiler.py | 4 ++-- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/source/fab/tools/compiler.py b/source/fab/tools/compiler.py index 84cd3ea3..816f4ebc 100644 --- a/source/fab/tools/compiler.py +++ b/source/fab/tools/compiler.py @@ -509,8 +509,11 @@ def run_version_command( # Cray compiler # ============================================================================ class Craycc(CCompiler): - '''Class for the native Cray C compiler. Cray has two different compilers. - Older ones have as version number: + '''Class for the native Cray C compiler. Since cc is actually a compiler + wrapper, follow the naming scheme of a compiler wrapper and call it: + craycc-cray. + + Cray has two different compilers. Older ones have as version number: Cray C : Version 8.7.0 Tue Jul 23, 2024 07:39:46 Newer compiler (several lines, the important one): Cray clang version 15.0.1 (66f7391d6a03cf932f321b9f6b1d8612ef5f362c) @@ -521,24 +524,26 @@ class Craycc(CCompiler): :param name: name of this compiler. :param exec_name: name of the executable. ''' - def __init__(self, name: str = "craycc", exec_name: str = "craycc"): + def __init__(self, name: str = "craycc-cray", exec_name: str = "cc"): super().__init__(name, exec_name, suite="cray", mpi=True, - openmp_flag="-qopenmp", + openmp_flag="-homp", version_regex=r"Cray [Cc][^\d]* (\d[\d\.]+\d) ") # ============================================================================ class Crayftn(FortranCompiler): - '''Class for the native Cray Fortran compiler. + '''Class for the native Cray Fortran compiler. Since ftn is actually a + compiler wrapper, follow the naming scheme of Cray compiler wrapper + and call it crayftn-cray. :param name: name of this compiler. :param exec_name: name of the executable. ''' - def __init__(self, name: str = "crayftn", exec_name: str = "crayftn"): + def __init__(self, name: str = "crayftn-cray", exec_name: str = "ftn"): super().__init__(name, exec_name, suite="cray", mpi=True, - module_folder_flag="-module", - openmp_flag="-qopenmp", + module_folder_flag="-J", + openmp_flag="-homp", syntax_only_flag="-syntax-only", version_regex=(r"Cray Fortran : Version " r"(\d[\d\.]+\d) ")) diff --git a/tests/unit_tests/tools/test_compiler.py b/tests/unit_tests/tools/test_compiler.py index 6f3138dc..834fccb0 100644 --- a/tests/unit_tests/tools/test_compiler.py +++ b/tests/unit_tests/tools/test_compiler.py @@ -816,7 +816,7 @@ def test_nvfortran_get_version_with_ifort_string(): def test_craycc(): '''Tests the Craycc class.''' craycc = Craycc() - assert craycc.name == "craycc" + assert craycc.name == "craycc-cray" assert isinstance(craycc, CCompiler) assert craycc.category == Category.C_COMPILER assert craycc.mpi @@ -882,7 +882,7 @@ def test_craycc_get_version_with_icc_string(): def test_crayftn(): '''Tests the Crayftn class.''' crayftn = Crayftn() - assert crayftn.name == "crayftn" + assert crayftn.name == "crayftn-cray" assert isinstance(crayftn, FortranCompiler) assert crayftn.category == Category.FORTRAN_COMPILER assert crayftn.mpi From 3c569bd249fd23a356efb4e09999f65f48bd7b3a Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Tue, 22 Oct 2024 14:17:59 +1100 Subject: [PATCH 07/12] Changed names again. --- source/fab/tools/compiler.py | 4 ++-- tests/unit_tests/tools/test_compiler.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/fab/tools/compiler.py b/source/fab/tools/compiler.py index 816f4ebc..e83bcd42 100644 --- a/source/fab/tools/compiler.py +++ b/source/fab/tools/compiler.py @@ -524,7 +524,7 @@ class Craycc(CCompiler): :param name: name of this compiler. :param exec_name: name of the executable. ''' - def __init__(self, name: str = "craycc-cray", exec_name: str = "cc"): + def __init__(self, name: str = "craycc-cc", exec_name: str = "cc"): super().__init__(name, exec_name, suite="cray", mpi=True, openmp_flag="-homp", version_regex=r"Cray [Cc][^\d]* (\d[\d\.]+\d) ") @@ -540,7 +540,7 @@ class Crayftn(FortranCompiler): :param exec_name: name of the executable. ''' - def __init__(self, name: str = "crayftn-cray", exec_name: str = "ftn"): + def __init__(self, name: str = "crayftn-ftn", exec_name: str = "ftn"): super().__init__(name, exec_name, suite="cray", mpi=True, module_folder_flag="-J", openmp_flag="-homp", diff --git a/tests/unit_tests/tools/test_compiler.py b/tests/unit_tests/tools/test_compiler.py index 834fccb0..5a31bbbc 100644 --- a/tests/unit_tests/tools/test_compiler.py +++ b/tests/unit_tests/tools/test_compiler.py @@ -816,7 +816,7 @@ def test_nvfortran_get_version_with_ifort_string(): def test_craycc(): '''Tests the Craycc class.''' craycc = Craycc() - assert craycc.name == "craycc-cray" + assert craycc.name == "craycc-cc" assert isinstance(craycc, CCompiler) assert craycc.category == Category.C_COMPILER assert craycc.mpi @@ -882,7 +882,7 @@ def test_craycc_get_version_with_icc_string(): def test_crayftn(): '''Tests the Crayftn class.''' crayftn = Crayftn() - assert crayftn.name == "crayftn-cray" + assert crayftn.name == "crayftn-ftn" assert isinstance(crayftn, FortranCompiler) assert crayftn.category == Category.FORTRAN_COMPILER assert crayftn.mpi From 2c298b91c0f057c5c3350274bb39ebf8e2c4cfa8 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 23 Oct 2024 14:53:32 +1100 Subject: [PATCH 08/12] Added shell tool. --- source/fab/tools/__init__.py | 2 + source/fab/tools/category.py | 1 + source/fab/tools/compiler.py | 2 +- source/fab/tools/shell.py | 44 ++++++++++++++++++++ source/fab/tools/tool.py | 13 +++--- source/fab/tools/tool_repository.py | 9 +++- tests/unit_tests/tools/test_shell.py | 61 ++++++++++++++++++++++++++++ 7 files changed, 124 insertions(+), 8 deletions(-) create mode 100644 source/fab/tools/shell.py create mode 100644 tests/unit_tests/tools/test_shell.py diff --git a/source/fab/tools/__init__.py b/source/fab/tools/__init__.py index 4ee807da..3316f01c 100644 --- a/source/fab/tools/__init__.py +++ b/source/fab/tools/__init__.py @@ -19,6 +19,7 @@ from fab.tools.psyclone import Psyclone from fab.tools.rsync import Rsync from fab.tools.preprocessor import Cpp, CppFortran, Fpp, Preprocessor +from fab.tools.shell import Shell from fab.tools.tool import Tool, CompilerSuiteTool # Order here is important to avoid a circular import from fab.tools.tool_repository import ToolRepository @@ -56,6 +57,7 @@ "Preprocessor", "Psyclone", "Rsync", + "Shell", "Subversion", "Tool", "ToolBox", diff --git a/source/fab/tools/category.py b/source/fab/tools/category.py index 6eab9b9d..a64781f1 100644 --- a/source/fab/tools/category.py +++ b/source/fab/tools/category.py @@ -25,6 +25,7 @@ class Category(Enum): SUBVERSION = auto() AR = auto() RSYNC = auto() + SHELL = auto() MISC = auto() def __str__(self): diff --git a/source/fab/tools/compiler.py b/source/fab/tools/compiler.py index e83bcd42..06047fe3 100644 --- a/source/fab/tools/compiler.py +++ b/source/fab/tools/compiler.py @@ -55,7 +55,7 @@ def __init__(self, name: str, compile_flag: Optional[str] = None, output_flag: Optional[str] = None, openmp_flag: Optional[str] = None, - availability_option: Optional[str] = None): + availability_option: Optional[Union[str, List[str]]] = None): super().__init__(name, exec_name, suite, category=category, availability_option=availability_option) self._version: Union[Tuple[int, ...], None] = None diff --git a/source/fab/tools/shell.py b/source/fab/tools/shell.py new file mode 100644 index 00000000..44688528 --- /dev/null +++ b/source/fab/tools/shell.py @@ -0,0 +1,44 @@ +############################################################################## +# (c) Crown copyright Met Office. All rights reserved. +# For further details please refer to the file COPYRIGHT +# which you should have received as part of this distribution +############################################################################## + +"""This file contains a base class for shells. This can be used to execute +other scripts. +""" + +from pathlib import Path +from typing import List, Union + +from fab.tools.category import Category +from fab.tools.tool import Tool + + +class Shell(Tool): + '''A simple wrapper that runs a shell script. There seems to be no + consistent way to simply check if a shell is working - not only support + a version command (e.g. sh and dash don't). Instead, availability + is tested by running a simple 'echo' command. + + :name: the path to the script to run. + ''' + def __init__(self, name: str): + super().__init__(name=name, exec_name=name, + availability_option=["-c", "echo hello"], + category=Category.SHELL) + + def exec(self, command: Union[str, List[Union[Path, str]]]) -> str: + '''Executes the specified command. + + :param command: the command and potential parameters to execute. + + :returns: stdout of the result. + ''' + if isinstance(command, str): + params = ["-c", command] + else: + params = ["-c"] + params.extend(command) + return super().run(additional_parameters=params, + capture_output=True) diff --git a/source/fab/tools/tool.py b/source/fab/tools/tool.py index cb8a7a06..a870c657 100644 --- a/source/fab/tools/tool.py +++ b/source/fab/tools/tool.py @@ -16,7 +16,7 @@ import logging from pathlib import Path import subprocess -from typing import Dict, List, Optional, Union +from typing import Dict, List, Optional, Sequence, Union from fab.tools.category import Category from fab.tools.flags import Flags @@ -36,7 +36,7 @@ class Tool: def __init__(self, name: str, exec_name: Union[str, Path], category: Category = Category.MISC, - availability_option: Optional[str] = None): + availability_option: Optional[Union[str, List[str]]] = None): self._logger = logging.getLogger(__name__) self._name = name self._exec_name = str(exec_name) @@ -63,7 +63,8 @@ def check_available(self) -> bool: :returns: whether the tool is working (True) or not. ''' try: - self.run(self._availability_option) + op = self._availability_option + self.run(op) except (RuntimeError, FileNotFoundError): return False return True @@ -107,7 +108,7 @@ def name(self) -> str: return self._name @property - def availability_option(self) -> str: + def availability_option(self) -> Union[str, List[str]]: ''':returns: the option to use to check if the tool is available.''' return self._availability_option @@ -139,7 +140,7 @@ def __str__(self): def run(self, additional_parameters: Optional[ - Union[str, List[Union[Path, str]]]] = None, + Union[str, Sequence[Union[Path, str]]]] = None, env: Optional[Dict[str, str]] = None, cwd: Optional[Union[Path, str]] = None, capture_output=True) -> str: @@ -210,7 +211,7 @@ class CompilerSuiteTool(Tool): ''' def __init__(self, name: str, exec_name: Union[str, Path], suite: str, category: Category, - availability_option: Optional[str] = None): + availability_option: Optional[Union[str, List[str]]] = None): super().__init__(name, exec_name, category, availability_option=availability_option) self._suite = suite diff --git a/source/fab/tools/tool_repository.py b/source/fab/tools/tool_repository.py index e18de0c4..e596a530 100644 --- a/source/fab/tools/tool_repository.py +++ b/source/fab/tools/tool_repository.py @@ -22,7 +22,7 @@ from fab.tools.versioning import Fcm, Git, Subversion from fab.tools import (Ar, Cpp, CppFortran, Craycc, Crayftn, Gcc, Gfortran, Icc, Icx, Ifort, Ifx, - Nvc, Nvfortran, Psyclone, Rsync) + Nvc, Nvfortran, Psyclone, Rsync, Shell) class ToolRepository(dict): @@ -71,6 +71,13 @@ def __init__(self): Ar, Fcm, Git, Psyclone, Rsync, Subversion]: self.add_tool(cls()) + # Add the common shells. While Fab itself does not need this, + # it is a very convenient tool for user configuration (e.g. to + # query nc-config etc) + for shell_name in ["bash", "sh", "ksh", "dash"]: + self.add_tool(Shell(shell_name)) + self.get_tool(Category.SHELL, shell_name) + # Now create the potential mpif90 and Cray ftn wrapper all_fc = self[Category.FORTRAN_COMPILER][:] for fc in all_fc: diff --git a/tests/unit_tests/tools/test_shell.py b/tests/unit_tests/tools/test_shell.py new file mode 100644 index 00000000..38ba8c71 --- /dev/null +++ b/tests/unit_tests/tools/test_shell.py @@ -0,0 +1,61 @@ +############################################################################## +# (c) Crown copyright Met Office. All rights reserved. +# For further details please refer to the file COPYRIGHT +# which you should have received as part of this distribution +############################################################################## + +'''Tests the shell implementation. +''' + +from unittest import mock + +from fab.tools import Category, Shell + + +def test_shell_constructor(): + '''Test the Shell constructor.''' + bash = Shell("bash") + assert bash.category == Category.SHELL + assert bash.name == "bash" + assert bash.exec_name == "bash" + + +def test_shell_check_available(): + '''Tests the is_available functionality.''' + bash = Shell("bash") + mock_result = mock.Mock(returncode=0) + with mock.patch('fab.tools.tool.subprocess.run', + return_value=mock_result) as tool_run: + assert bash.check_available() + tool_run.assert_called_once_with( + ["bash", "-c", "echo hello"], capture_output=True, env=None, + cwd=None, check=False) + + # Test behaviour if a runtime error happens: + with mock.patch("fab.tools.tool.Tool.run", + side_effect=RuntimeError("")) as tool_run: + assert not bash.check_available() + + +def test_shell_exec_single_arg(): + '''Test running a shell script without additional parameters.''' + bash = Shell("ksh") + mock_result = mock.Mock(returncode=0) + with mock.patch('fab.tools.tool.subprocess.run', + return_value=mock_result) as tool_run: + bash.exec("echo") + tool_run.assert_called_with(['ksh', '-c', 'echo'], + capture_output=True, env=None, cwd=None, + check=False) + + +def test_shell_exec_multiple_args(): + '''Test running a shell script with parameters.''' + bash = Shell("ksh") + mock_result = mock.Mock(returncode=0) + with mock.patch('fab.tools.tool.subprocess.run', + return_value=mock_result) as tool_run: + bash.exec(["some", "shell", "function"]) + tool_run.assert_called_with(['ksh', '-c', 'some', 'shell', 'function'], + capture_output=True, env=None, cwd=None, + check=False) From 0107d25d558f63fca6013ac924d955e2d496805f Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 23 Oct 2024 15:01:26 +1100 Subject: [PATCH 09/12] Try to make mypy happy. --- source/fab/tools/shell.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/fab/tools/shell.py b/source/fab/tools/shell.py index 44688528..162649fb 100644 --- a/source/fab/tools/shell.py +++ b/source/fab/tools/shell.py @@ -35,6 +35,8 @@ def exec(self, command: Union[str, List[Union[Path, str]]]) -> str: :returns: stdout of the result. ''' + # Make mypy happy: + params: List[Union[str, Path]] if isinstance(command, str): params = ["-c", command] else: From e314b2b3cdaa8c39f4731f2b8b942fc860d79de1 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 23 Oct 2024 23:39:35 +1100 Subject: [PATCH 10/12] Removed debug code. --- source/fab/tools/tool_repository.py | 1 - 1 file changed, 1 deletion(-) diff --git a/source/fab/tools/tool_repository.py b/source/fab/tools/tool_repository.py index e596a530..2d080bbb 100644 --- a/source/fab/tools/tool_repository.py +++ b/source/fab/tools/tool_repository.py @@ -76,7 +76,6 @@ def __init__(self): # query nc-config etc) for shell_name in ["bash", "sh", "ksh", "dash"]: self.add_tool(Shell(shell_name)) - self.get_tool(Category.SHELL, shell_name) # Now create the potential mpif90 and Cray ftn wrapper all_fc = self[Category.FORTRAN_COMPILER][:] From f6e16e98cfda3812b0658c904fd0697088c2af2a Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 24 Oct 2024 10:07:04 +1100 Subject: [PATCH 11/12] ToolRepository now only returns default that are available. Updated tests to make tools as available. --- source/fab/tools/tool_repository.py | 24 +++++--- tests/unit_tests/steps/test_grab.py | 60 +++++++++++++++---- .../unit_tests/tools/test_tool_repository.py | 47 ++++++++++----- 3 files changed, 95 insertions(+), 36 deletions(-) diff --git a/source/fab/tools/tool_repository.py b/source/fab/tools/tool_repository.py index 2d080bbb..b6a0250a 100644 --- a/source/fab/tools/tool_repository.py +++ b/source/fab/tools/tool_repository.py @@ -62,7 +62,7 @@ def __init__(self): # Add the FAB default tools: # TODO: sort the defaults so that they actually work (since not all # tools FAB knows about are available). For now, disable Fpp (by not - # adding it). IF someone actually uses it it can added. + # adding it). If someone actually uses it it can added. for cls in [Craycc, Crayftn, Gcc, Gfortran, Icc, Icx, Ifort, Ifx, @@ -99,9 +99,10 @@ def __init__(self): def add_tool(self, tool: Tool): '''Creates an instance of the specified class and adds it - to the tool repository. + to the tool repository. If the tool is a compiler, it automatically + adds the compiler as a linker as well (named "linker-{tool.name}"). - :param cls: the tool to instantiate. + :param tool: the tool to add. ''' # We do not test if a tool is actually available. The ToolRepository @@ -158,10 +159,10 @@ def set_default_compiler_suite(self, suite: str): def get_default(self, category: Category, mpi: Optional[bool] = None): - '''Returns the default tool for a given category. For most tools - that will be the first entry in the list of tools. The exception - are compilers and linker: in this case it must be specified if - MPI support is required or not. And the default return will be + '''Returns the default tool for a given category that is available. + For most tools that will be the first entry in the list of tools. The + exception are compilers and linker: in this case it must be specified + if MPI support is required or not. And the default return will be the first tool that either supports MPI or not. :param category: the category for which to return the default tool. @@ -178,7 +179,12 @@ def get_default(self, category: Category, # If not a compiler or linker, return the first tool if not category.is_compiler and category != Category.LINKER: - return self[category][0] + for tool in self[category]: + if tool.is_available: + return tool + tool_names = ",".join(i.name for i in self[category]) + raise RuntimeError(f"Can't find available '{category}' tool. " + f"Tools are '{tool_names}'.") if not isinstance(mpi, bool): raise RuntimeError(f"Invalid or missing mpi specification " @@ -186,7 +192,7 @@ def get_default(self, category: Category, for tool in self[category]: # If the tool supports/does not support MPI, return the first one - if mpi == tool.mpi: + if tool.is_available and mpi == tool.mpi: return tool # Don't bother returning an MPI enabled tool if no-MPI is requested - diff --git a/tests/unit_tests/steps/test_grab.py b/tests/unit_tests/steps/test_grab.py index 348dc293..dc222a22 100644 --- a/tests/unit_tests/steps/test_grab.py +++ b/tests/unit_tests/steps/test_grab.py @@ -3,6 +3,10 @@ # For further details please refer to the file COPYRIGHT # which you should have received as part of this distribution ############################################################################## + +'''Test various grab implementation - folders and fcm. +''' + from pathlib import Path from types import SimpleNamespace from unittest import mock @@ -15,14 +19,21 @@ class TestGrabFolder: + '''Test grab folder functionality.''' def test_trailing_slash(self): - with pytest.warns(UserWarning, match="_metric_send_conn not set, cannot send metrics"): - self._common(grab_src='/grab/source/', expect_grab_src='/grab/source/') + '''Test folder grabbing with a trailing slash.''' + with pytest.warns(UserWarning, match="_metric_send_conn not set, " + "cannot send metrics"): + self._common(grab_src='/grab/source/', + expect_grab_src='/grab/source/') def test_no_trailing_slash(self): - with pytest.warns(UserWarning, match="_metric_send_conn not set, cannot send metrics"): - self._common(grab_src='/grab/source', expect_grab_src='/grab/source/') + '''Test folder grabbing without a trailing slash.''' + with pytest.warns(UserWarning, match="_metric_send_conn not set, " + "cannot send metrics"): + self._common(grab_src='/grab/source', + expect_grab_src='/grab/source/') def _common(self, grab_src, expect_grab_src): source_root = Path('/workspace/source') @@ -30,9 +41,15 @@ def _common(self, grab_src, expect_grab_src): mock_config = SimpleNamespace(source_root=source_root, tool_box=ToolBox()) + # Since is_available calls run, in order to test a single run call, + # we patch is_available to be always true. with mock.patch('pathlib.Path.mkdir'): with mock.patch('fab.tools.tool.Tool.run') as mock_run: - grab_folder(mock_config, src=grab_src, dst_label=dst) + with mock.patch( + 'fab.tools.tool.Tool.is_available', + new_callable=mock.PropertyMock) as is_available: + is_available.return_value = True + grab_folder(mock_config, src=grab_src, dst_label=dst) expect_dst = mock_config.source_root / dst mock_run.assert_called_once_with( @@ -41,8 +58,10 @@ def _common(self, grab_src, expect_grab_src): class TestGrabFcm: + '''Test FCM functionality.''' def test_no_revision(self): + '''Test FCM without specifying a revision.''' source_root = Path('/workspace/source') source_url = '/www.example.com/bar' dst_label = 'bar' @@ -50,15 +69,23 @@ def test_no_revision(self): mock_config = SimpleNamespace(source_root=source_root, tool_box=ToolBox()) with mock.patch('pathlib.Path.mkdir'): - with mock.patch('fab.tools.tool.Tool.run') as mock_run, \ - pytest.warns(UserWarning, match="_metric_send_conn not set, cannot send metrics"): - fcm_export(config=mock_config, src=source_url, dst_label=dst_label) + with mock.patch('fab.tools.tool.Tool.is_available', + new_callable=mock.PropertyMock) as is_available: + is_available.return_value = True + with mock.patch('fab.tools.tool.Tool.run') as mock_run, \ + pytest.warns(UserWarning, + match="_metric_send_conn not " + "set, cannot send metrics"): + fcm_export(config=mock_config, src=source_url, + dst_label=dst_label) mock_run.assert_called_once_with(['export', '--force', source_url, str(source_root / dst_label)], - env=None, cwd=None, capture_output=True) + env=None, cwd=None, + capture_output=True) def test_revision(self): + '''Test that the revision is passed on correctly in fcm export.''' source_root = Path('/workspace/source') source_url = '/www.example.com/bar' dst_label = 'bar' @@ -67,12 +94,19 @@ def test_revision(self): mock_config = SimpleNamespace(source_root=source_root, tool_box=ToolBox()) with mock.patch('pathlib.Path.mkdir'): - with mock.patch('fab.tools.tool.Tool.run') as mock_run, \ - pytest.warns(UserWarning, match="_metric_send_conn not set, cannot send metrics"): - fcm_export(mock_config, src=source_url, dst_label=dst_label, revision=revision) + with mock.patch('fab.tools.tool.Tool.is_available', + new_callable=mock.PropertyMock) as is_available: + is_available.return_value = True + with mock.patch('fab.tools.tool.Tool.run') as mock_run, \ + pytest.warns( + UserWarning, match="_metric_send_conn not set, " + "cannot send metrics"): + fcm_export(mock_config, src=source_url, + dst_label=dst_label, revision=revision) mock_run.assert_called_once_with( - ['export', '--force', '--revision', '42', f'{source_url}', str(source_root / dst_label)], + ['export', '--force', '--revision', '42', f'{source_url}', + str(source_root / dst_label)], env=None, cwd=None, capture_output=True) # todo: test missing repo diff --git a/tests/unit_tests/tools/test_tool_repository.py b/tests/unit_tests/tools/test_tool_repository.py index e16ad00d..e56da023 100644 --- a/tests/unit_tests/tools/test_tool_repository.py +++ b/tests/unit_tests/tools/test_tool_repository.py @@ -108,17 +108,36 @@ def test_tool_repository_default_compiler_suite(): '''Tests the setting of default suite for compiler and linker.''' tr = ToolRepository() tr.set_default_compiler_suite("gnu") - for cat in [Category.C_COMPILER, Category.FORTRAN_COMPILER, - Category.LINKER]: - def_tool = tr.get_default(cat, mpi=False) - assert def_tool.suite == "gnu" - - tr.set_default_compiler_suite("intel-classic") - for cat in [Category.C_COMPILER, Category.FORTRAN_COMPILER, - Category.LINKER]: - def_tool = tr.get_default(cat, mpi=False) - assert def_tool.suite == "intel-classic" - with pytest.raises(RuntimeError) as err: - tr.set_default_compiler_suite("does-not-exist") - assert ("Cannot find 'FORTRAN_COMPILER' in the suite 'does-not-exist'" - in str(err.value)) + + # Mark all compiler and linker as available. + with mock.patch('fab.tools.tool.Tool.is_available', + new_callable=mock.PropertyMock) as is_available: + is_available.return_value = True + for cat in [Category.C_COMPILER, Category.FORTRAN_COMPILER, + Category.LINKER]: + def_tool = tr.get_default(cat, mpi=False) + assert def_tool.suite == "gnu" + + tr.set_default_compiler_suite("intel-classic") + for cat in [Category.C_COMPILER, Category.FORTRAN_COMPILER, + Category.LINKER]: + def_tool = tr.get_default(cat, mpi=False) + assert def_tool.suite == "intel-classic" + with pytest.raises(RuntimeError) as err: + tr.set_default_compiler_suite("does-not-exist") + assert ("Cannot find 'FORTRAN_COMPILER' in the suite 'does-not-exist'" + in str(err.value)) + + +def test_tool_repository_no_tool_available(): + '''Tests error handling if no tool is available.''' + + tr = ToolRepository() + tr.set_default_compiler_suite("gnu") + with mock.patch('fab.tools.tool.Tool.is_available', + new_callable=mock.PropertyMock) as is_available: + is_available.return_value = False + with pytest.raises(RuntimeError) as err: + def_tool = tr.get_default(Category.SHELL) + assert ("Can't find available 'SHELL' tool. Tools are 'bash,sh,ksh," + "dash'" in str(err.value)) From 54be09baa7c0c509239933136eff9881a10da0ae Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 24 Oct 2024 10:10:22 +1100 Subject: [PATCH 12/12] Fixed flake8. --- tests/unit_tests/tools/test_tool_repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit_tests/tools/test_tool_repository.py b/tests/unit_tests/tools/test_tool_repository.py index e56da023..4b90e569 100644 --- a/tests/unit_tests/tools/test_tool_repository.py +++ b/tests/unit_tests/tools/test_tool_repository.py @@ -138,6 +138,6 @@ def test_tool_repository_no_tool_available(): new_callable=mock.PropertyMock) as is_available: is_available.return_value = False with pytest.raises(RuntimeError) as err: - def_tool = tr.get_default(Category.SHELL) + tr.get_default(Category.SHELL) assert ("Can't find available 'SHELL' tool. Tools are 'bash,sh,ksh," "dash'" in str(err.value))