add module-search-path-headers configuration option to control how modules set search paths to header files

easybuild/framework/easyblock.py

@@ -74,6 +74,8 @@
 from easybuild.tools.build_log import print_error, print_msg, print_warning
 from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES
 from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES
+from easybuild.tools.config import DEFAULT_MOD_SEARCH_PATH_HEADERS, MOD_SEARCH_PATH_HEADERS
+from easybuild.tools.config import SEARCH_PATH_BIN_DIRS, SEARCH_PATH_HEADER_DIRS, SEARCH_PATH_LIB_DIRS
 from easybuild.tools.config import EASYBUILD_SOURCES_URL # noqa
 from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath
 from easybuild.tools.config import install_path, log_path, package_path, source_paths
@@ -107,10 +109,13 @@
 from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg
 from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION
 
-DEFAULT_BIN_LIB_SUBDIRS = ('bin', 'lib', 'lib64')
+DEFAULT_BIN_LIB_SUBDIRS = SEARCH_PATH_BIN_DIRS + SEARCH_PATH_LIB_DIRS
 
 MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, POSTITER_STEP, SANITYCHECK_STEP]
 
+# search paths that require some file in their top directory
+NON_RECURSIVE_SEARCH_PATHS = ["PATH", "LD_LIBRARY_PATH"]
+
 # string part of URL for Python packages on PyPI that indicates needs to be rewritten (see derive_alt_pypi_url)
 PYPI_PKG_URL_PATTERN = 'pypi.python.org/packages/source/'
 
@@ -1555,109 +1560,149 @@ def make_module_group_check(self):
 
  return txt
 
- def make_module_req(self):
+ def make_module_req(self, fake=False):
  """
- Generate the environment-variables to run the module.
+ Generate the environment-variables required to run the module.
+ Fake modules can set search paths to empty directories.
  """
- requirements = self.make_module_req_guess()
-
- lines = ['\n']
- if os.path.isdir(self.installdir):
- old_dir = change_dir(self.installdir)
- else:
- old_dir = None
+ mod_lines = ['\n']
 
  if self.dry_run:
  self.dry_run_msg("List of paths that would be searched and added to module file:\n")
  note = "note: glob patterns are not expanded and existence checks "
  note += "for paths are skipped for the statements below due to dry run"
- lines.append(self.module_generator.comment(note))
-
- # For these environment variables, the corresponding directory must include at least one file.
- # The values determine if detection is done recursively, i.e. if it accepts directories where files
- # are only in subdirectories.
- keys_requiring_files = {
- 'PATH': False,
- 'LD_LIBRARY_PATH': False,
- 'LIBRARY_PATH': True,
- 'CPATH': True,
- 'CMAKE_PREFIX_PATH': True,
- 'CMAKE_LIBRARY_PATH': True,
- }
+ mod_lines.append(self.module_generator.comment(note))
 
- for key, reqs in sorted(requirements.items()):
- if isinstance(reqs, str):
- self.log.warning("Hoisting string value %s into a list before iterating over it", reqs)
- reqs = [reqs]
+ env_var_requirements = self.make_module_req_guess()
+ for env_var, search_paths in sorted(env_var_requirements.items()):
+ if isinstance(search_paths, str):
+ self.log.warning("Hoisting string value %s into a list before iterating over it", search_paths)
+ search_paths = [search_paths]
+
+ mod_env_paths = []
+ recursive = env_var not in NON_RECURSIVE_SEARCH_PATHS
  if self.dry_run:
- self.dry_run_msg(" $%s: %s" % (key, ', '.join(reqs)))
- # Don't expand globs or do any filtering below for dry run
- paths = reqs
+ self.dry_run_msg(f" ${env_var}:{', '.join(search_paths)}")
+ # Don't expand globs or do any filtering for dry run
+ mod_env_paths = search_paths
  else:
- # Expand globs but only if the string is non-empty
- # empty string is a valid value here (i.e. to prepend the installation prefix, cfr $CUDA_HOME)
- paths = sum((glob.glob(path) if path else [path] for path in reqs), []) # sum flattens to list
-
- # If lib64 is just a symlink to lib we fixup the paths to avoid duplicates
- lib64_is_symlink = (all(os.path.isdir(path) for path in ['lib', 'lib64']) and
- os.path.samefile('lib', 'lib64'))
- if lib64_is_symlink:
- fixed_paths = []
- for path in paths:
- if (path + os.path.sep).startswith('lib64' + os.path.sep):
- # We only need CMAKE_LIBRARY_PATH if there is a separate lib64 path, so skip symlink
- if key == 'CMAKE_LIBRARY_PATH':
- continue
- path = path.replace('lib64', 'lib', 1)
- fixed_paths.append(path)
- if fixed_paths != paths:
- self.log.info("Fixed symlink lib64 in paths for %s: %s -> %s", key, paths, fixed_paths)
- paths = fixed_paths
- # remove duplicate paths preserving order
- paths = nub(paths)
- if key in keys_requiring_files:
- # only retain paths that contain at least one file
- recursive = keys_requiring_files[key]
- retained_paths = []
- for pth in paths:
- fullpath = os.path.join(self.installdir, pth)
- if os.path.isdir(fullpath) and dir_contains_files(fullpath, recursive=recursive):
- retained_paths.append(pth)
- if retained_paths != paths:
- self.log.info("Only retaining paths for %s that contain at least one file: %s -> %s",
- key, paths, retained_paths)
- paths = retained_paths
-
- if paths:
- lines.append(self.module_generator.prepend_paths(key, paths))
+ for sp in search_paths:
+ mod_env_paths.extend(self._expand_module_search_path(sp, recursive, fake=fake))
+
+ if mod_env_paths:
+ mod_env_paths = nub(mod_env_paths) # remove duplicates
+ mod_lines.append(self.module_generator.prepend_paths(env_var, mod_env_paths))
+
  if self.dry_run:
  self.dry_run_msg('')
 
- if old_dir is not None:
- change_dir(old_dir)
+ return "".join(mod_lines)
 
- return ''.join(lines)
+ def _expand_module_search_path(self, search_path, recursive, fake=False):
+ """
+ Expand given path glob and return list of suitable paths to be used as search paths:
+ - Files must exist and directories be non-empty
+ - Fake modules can set search paths to empty directories
+ - Search paths to 'lib64' symlinked to 'lib' are discarded
+ """
+ # Expand globs but only if the string is non-empty
+ # empty string is a valid value here (i.e. to prepend the installation prefix root directory)
+ abs_search_path = os.path.join(self.installdir, search_path)
+ exp_search_paths = [abs_search_path] if search_path == "" else glob.glob(abs_search_path)
+
+ retained_search_paths = []
+ for abs_path in exp_search_paths:
+ # return relative paths
+ tentative_path = os.path.relpath(abs_path, start=self.installdir)
+ tentative_path = "" if tentative_path == "." else tentative_path # use empty string instead of dot
+
+ # avoid duplicate entries if lib64 is just a symlink to lib
+ if (tentative_path + os.path.sep).startswith("lib64" + os.path.sep):
+ abs_lib_path = os.path.join(self.installdir, "lib")
+ abs_lib64_path = os.path.join(self.installdir, "lib64")
+ if os.path.islink(abs_lib64_path) and os.path.samefile(abs_lib_path, abs_lib64_path):
+ self.log.debug("Discarded search path to symlink lib64: %s", tentative_path)
+ break
+
+ # only retain paths to directories that contain at least one file
+ if os.path.isdir(abs_path) and not dir_contains_files(abs_path, recursive=recursive) and not fake:
+ self.log.debug("Discarded search path to empty directory: %s", tentative_path)
+ break
+
+ retained_search_paths.append(tentative_path)
+
+ return retained_search_paths
 
  def make_module_req_guess(self):
  """
- A dictionary of possible directories to look for.
+ A dictionary of common search path variables to be loaded by environment modules
+ Each key contains the list of known directories related to the search path
  """
- lib_paths = ['lib', 'lib32', 'lib64']
- return {
- 'PATH': ['bin', 'sbin'],
- 'LD_LIBRARY_PATH': lib_paths,
- 'LIBRARY_PATH': lib_paths,
- 'CPATH': ['include'],
+ module_req_guess = {
+ 'PATH': SEARCH_PATH_BIN_DIRS + ['sbin'],
+ 'LD_LIBRARY_PATH': SEARCH_PATH_LIB_DIRS,
+ 'LIBRARY_PATH': SEARCH_PATH_LIB_DIRS,
  'MANPATH': ['man', os.path.join('share', 'man')],
- 'PKG_CONFIG_PATH': [os.path.join(x, 'pkgconfig') for x in lib_paths + ['share']],
+ 'PKG_CONFIG_PATH': [os.path.join(x, 'pkgconfig') for x in SEARCH_PATH_LIB_DIRS + ['share']],
  'ACLOCAL_PATH': [os.path.join('share', 'aclocal')],
  'CLASSPATH': ['*.jar'],
  'XDG_DATA_DIRS': ['share'],
- 'GI_TYPELIB_PATH': [os.path.join(x, 'girepository-*') for x in lib_paths],
+ 'GI_TYPELIB_PATH': [os.path.join(x, 'girepository-*') for x in SEARCH_PATH_LIB_DIRS],
  'CMAKE_PREFIX_PATH': [''],
- 'CMAKE_LIBRARY_PATH': ['lib64'], # lib and lib32 are searched through the above
+ 'CMAKE_LIBRARY_PATH': ['lib64'], # only needed for installations whith standalone lib64
  }
 
+ module_req_guess.update(self._make_search_path_guess("module_search_path_headers"))
+
+ return module_req_guess
+
+ @property
+ def module_header_dirs(self):
+ """List of relative search paths to look for header files"""
+ return SEARCH_PATH_HEADER_DIRS
+
+ def _make_search_path_guess(self, search_path_opt):
+ """
+ Return dictionary with guesses for given search path option
+ """
+ # known module_search_path_xxx options
+ opt_settings = {
+ "module_search_path_headers": (
+ MOD_SEARCH_PATH_HEADERS, DEFAULT_MOD_SEARCH_PATH_HEADERS, self.module_header_dirs
+ ),
+ }
+
+ if search_path_opt not in opt_settings:
+ raise EasyBuildError(
+ "Unknown option given to make search path guess: %s. Choose one of: %s",
+ search_path_opt, ", ".join(opt_settings.keys())
+ )
+
+ search_path_modes, mode_default, search_path_dirs = opt_settings[search_path_opt]
+
+ # easyconfig parameters have precedence over command line options
+ cfg_param = self.cfg[search_path_opt]
+ build_opt = build_option(search_path_opt)
+ mode = mode_default
+ if cfg_param is not False:
+ mode = cfg_param
+ elif build_opt is not False:
+ mode = build_opt
+
+ if mode not in search_path_modes:
+ raise EasyBuildError(
+ "Unknown value selected for option %s: %s. Choose one of: %s",
+ search_path_opt, mode, ", ".join(search_path_modes)
+ )
+
+ guess = {}
+ for env_var in search_path_modes[mode]:
+ dbg_msg = "Adding search paths to environment variable '%s': %s"
+ self.log.debug(dbg_msg, env_var, ", ".join(search_path_dirs))
+ guess.update({env_var: search_path_dirs})
+
+ return guess
+
  def load_module(self, mod_paths=None, purge=True, extra_modules=None, verbose=True):
  """
  Load module for this software package/version, after purging all currently loaded modules.
@@ -3795,7 +3840,7 @@ def make_module_step(self, fake=False):
  txt += self.make_module_deppaths()
  txt += self.make_module_dep()
  txt += self.make_module_extend_modpath()
- txt += self.make_module_req()
+ txt += self.make_module_req(fake=fake)
  txt += self.make_module_extra()
  txt += self.make_module_footer()
 

easybuild/framework/easyconfig/default.py

@@ -208,10 +208,11 @@
  'moduleloadnoconflict': [False, "Don't check for conflicts, unload other versions instead ", MODULES],
  'module_depends_on': [False, 'Use depends_on (Lmod 7.6.1+) for dependencies in generated module '
  '(implies recursive unloading of modules).', MODULES],
+ 'module_search_path_headers': [False, "Environment variable set by modules on load with search paths "
+ "to header files", MODULES],
  'recursive_module_unload': [None, "Recursive unload of all dependencies when unloading module "
- "(True/False to hard enable/disable; None implies honoring "
- "the --recursive-module-unload EasyBuild configuration setting",
- MODULES],
+ "(True/False to hard enable/disable; None implies honoring the "
+ "--recursive-module-unload EasyBuild configuration setting", MODULES],
 
  # MODULES documentation easyconfig parameters
  # (docurls is part of MANDATORY)

easybuild/tools/config.py

@@ -168,13 +168,27 @@
 LOCAL_VAR_NAMING_CHECK_WARN = WARN
 LOCAL_VAR_NAMING_CHECKS = [LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG, LOCAL_VAR_NAMING_CHECK_WARN]
 
-
 OUTPUT_STYLE_AUTO = 'auto'
 OUTPUT_STYLE_BASIC = 'basic'
 OUTPUT_STYLE_NO_COLOR = 'no_color'
 OUTPUT_STYLE_RICH = 'rich'
 OUTPUT_STYLES = (OUTPUT_STYLE_AUTO, OUTPUT_STYLE_BASIC, OUTPUT_STYLE_NO_COLOR, OUTPUT_STYLE_RICH)
 
+SEARCH_PATH_BIN_DIRS = ["bin"]
+SEARCH_PATH_HEADER_DIRS = ["include"]
+SEARCH_PATH_LIB_DIRS = ["lib", "lib64"]
+
+# modes to handle header search paths in environment of modules
+MOD_SEARCH_PATH_HEADERS_NONE = "none"
+MOD_SEARCH_PATH_HEADERS_CPATH = "CPATH"
+MOD_SEARCH_PATH_HEADERS_INCLUDE = "INCLUDE_PATHS"
+MOD_SEARCH_PATH_HEADERS = {
+ MOD_SEARCH_PATH_HEADERS_NONE: [],
+ MOD_SEARCH_PATH_HEADERS_CPATH: ["CPATH"],
+ MOD_SEARCH_PATH_HEADERS_INCLUDE: ["C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH"],
+}
+DEFAULT_MOD_SEARCH_PATH_HEADERS = MOD_SEARCH_PATH_HEADERS_CPATH
+
 
 class Singleton(ABCMeta):
  """Serves as metaclass for classes that should implement the Singleton pattern.
@@ -297,6 +311,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
  'logtostdout',
  'minimal_toolchains',
  'module_only',
+ 'module_search_path_headers',
  'package',
  'parallel_extensions_install',
  'read_only_installdir',

easybuild/tools/options.py

@@ -69,6 +69,7 @@
 from easybuild.tools.config import DEFAULT_JOB_EB_CMD, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS
 from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL
 from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL
+from easybuild.tools.config import DEFAULT_MOD_SEARCH_PATH_HEADERS, MOD_SEARCH_PATH_HEADERS
 from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_EXTRA_SOURCE_URLS
 from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT
 from easybuild.tools.config import DEFAULT_PR_TARGET_ACCOUNT, DEFAULT_FILTER_RPATH_SANITY_LIBS
@@ -597,6 +598,9 @@ def config_options(self):
  'module-extensions': ("Include 'extensions' statement in generated module file (Lua syntax only)",
  None, 'store_true', True),
  'module-naming-scheme': ("Module naming scheme to use", None, 'store', DEFAULT_MNS),
+ 'module-search-path-headers': ("Environment variable set by modules on load with search paths "
+ "to header files", 'choice', 'store', DEFAULT_MOD_SEARCH_PATH_HEADERS,
+ [*MOD_SEARCH_PATH_HEADERS]),
  'module-syntax': ("Syntax to be used for module files", 'choice', 'store', DEFAULT_MODULE_SYNTAX,
  sorted(avail_module_generators().keys())),
  'moduleclasses': (("Extend supported module classes "