From 92bb54f0d3242b23b2f956e740a75be2f9f79afb Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 5 Apr 2024 16:25:25 +0200 Subject: [PATCH 01/26] Ignore Python from virtualenvs in GROMACS configure --- easybuild/easyblocks/g/gromacs.py | 36 +++++++++++++++++++------------ 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/easybuild/easyblocks/g/gromacs.py b/easybuild/easyblocks/g/gromacs.py index 5a010d4f26..96f719606c 100644 --- a/easybuild/easyblocks/g/gromacs.py +++ b/easybuild/easyblocks/g/gromacs.py @@ -167,7 +167,9 @@ def prepare_step(self, *args, **kwargs): def configure_step(self): """Custom configuration procedure for GROMACS: set configure options for configure or cmake.""" - if LooseVersion(self.version) >= LooseVersion('4.6'): + gromacs_version = LooseVersion(self.version) + + if gromacs_version >= '4.6': cuda = get_software_root('CUDA') if cuda: # CUDA with double precision is currently not supported in GROMACS yet @@ -188,10 +190,11 @@ def configure_step(self): self.log.info("skipping configure step") return - if LooseVersion(self.version) >= LooseVersion('2021'): - self.cfg.update('configopts', "-DGMX_GPU=CUDA -DCUDA_TOOLKIT_ROOT_DIR=%s" % cuda) + if gromacs_version >= '2021': + self.cfg.update('configopts', "-DGMX_GPU=CUDA") else: - self.cfg.update('configopts', "-DGMX_GPU=ON -DCUDA_TOOLKIT_ROOT_DIR=%s" % cuda) + self.cfg.update('configopts', "-DGMX_GPU=ON") + self.cfg.update('configopts', "-DCUDA_TOOLKIT_ROOT_DIR=%s" % cuda) # Set CUDA capabilities based on template value. if '-DGMX_CUDA_TARGET_SM' not in self.cfg['configopts']: @@ -236,14 +239,14 @@ def configure_step(self): # Ensure that the GROMACS log files report how the code was patched # during the build, so that any problems are easier to diagnose. # The GMX_VERSION_STRING_OF_FORK feature is available since 2020. - if (LooseVersion(self.version) >= LooseVersion('2020') and + if (gromacs_version >= '2020' and '-DGMX_VERSION_STRING_OF_FORK=' not in self.cfg['configopts']): gromacs_version_string_suffix = 'EasyBuild-%s' % EASYBUILD_VERSION if plumed_root: gromacs_version_string_suffix += '-PLUMED-%s' % get_software_version('PLUMED') self.cfg.update('configopts', '-DGMX_VERSION_STRING_OF_FORK=%s' % gromacs_version_string_suffix) - if LooseVersion(self.version) < LooseVersion('4.6'): + if gromacs_version < '4.6': self.log.info("Using configure script for configuring GROMACS build.") if self.cfg['build_shared_libs']: @@ -259,7 +262,7 @@ def configure_step(self): self.cfg.update('configopts', "--without-x") # OpenMP is not supported for versions older than 4.5. - if LooseVersion(self.version) >= LooseVersion('4.5'): + if gromacs_version >= '4.5': # enable OpenMP support if desired if self.toolchain.options.get('openmp', None): self.cfg.update('configopts', "--enable-threads") @@ -310,22 +313,27 @@ def configure_step(self): mpiexec_path, self.cfg.get('mpiexec_numproc_flag'), mpi_numprocs) - if LooseVersion(self.version) >= LooseVersion('2019'): + if gromacs_version >= '2019': # Building the gmxapi interface requires shared libraries, # this is handled in the class initialisation so --module-only works self.cfg.update('configopts', "-DGMXAPI=ON") - if LooseVersion(self.version) >= LooseVersion('2020'): + if gromacs_version >= '2020': # build Python bindings if Python is loaded as a dependency python_root = get_software_root('Python') if python_root: + self.cfg.update('configopts', "-DGMX_PYTHON_PACKAGE=ON") bin_python = os.path.join(python_root, 'bin', 'python') + # For find_package(PythonInterp) self.cfg.update('configopts', "-DPYTHON_EXECUTABLE=%s" % bin_python) - self.cfg.update('configopts', "-DGMX_PYTHON_PACKAGE=ON") + if gromacs_version >= '2021': + # For find_package(Python3) - Ignore virtual envs + self.cfg.update('configopts', "-DPython3_FIND_VIRTUALENV=STANDARD") + # Now patch GROMACS for PLUMED before cmake if plumed_root: - if LooseVersion(self.version) >= LooseVersion('5.1'): + if gromacs_version >= '5.1': # Use shared or static patch depending on # setting of self.cfg['build_shared_libs'] # and adapt cmake flags accordingly as per instructions @@ -355,7 +363,7 @@ def configure_step(self): if self.toolchain.toolchain_family() != toolchain.CRAYPE: gmx_simd = self.get_gromacs_arch() if gmx_simd: - if LooseVersion(self.version) < LooseVersion('5.0'): + if gromacs_version < '5.0': self.cfg.update('configopts', "-DGMX_CPU_ACCELERATION=%s" % gmx_simd) else: self.cfg.update('configopts', "-DGMX_SIMD=%s" % gmx_simd) @@ -404,7 +412,7 @@ def configure_step(self): env.setvar('LDFLAGS', "%s -lgfortran -lm" % os.environ.get('LDFLAGS', '')) # no more GSL support in GROMACS 5.x, see http://redmine.gromacs.org/issues/1472 - if LooseVersion(self.version) < LooseVersion('5.0'): + if gromacs_version < '5.0': # enable GSL when it's provided if get_software_root('GSL'): self.cfg.update('configopts', "-DGMX_GSL=ON") @@ -424,7 +432,7 @@ def configure_step(self): out = super(EB_GROMACS, self).configure_step() # for recent GROMACS versions, make very sure that a decent BLAS, LAPACK and FFT is found and used - if LooseVersion(self.version) >= LooseVersion('4.6.5'): + if gromacs_version >= '4.6.5': patterns = [ r"Using external FFT library - \S*", r"Looking for dgemm_ - found", From e6828f9d7ada029d1ac3e2605d1ba3102c2f0c27 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 5 Apr 2024 16:34:37 +0200 Subject: [PATCH 02/26] Remove blank line --- easybuild/easyblocks/g/gromacs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/easyblocks/g/gromacs.py b/easybuild/easyblocks/g/gromacs.py index 96f719606c..6aa96f92ac 100644 --- a/easybuild/easyblocks/g/gromacs.py +++ b/easybuild/easyblocks/g/gromacs.py @@ -330,7 +330,6 @@ def configure_step(self): # For find_package(Python3) - Ignore virtual envs self.cfg.update('configopts', "-DPython3_FIND_VIRTUALENV=STANDARD") - # Now patch GROMACS for PLUMED before cmake if plumed_root: if gromacs_version >= '5.1': From 7fd6e9d7bc2e43951531f2a5153b5c51418cb66f Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 4 Sep 2024 11:48:55 +0200 Subject: [PATCH 03/26] allow specifying a max version for the system python in pythonpackage based easyconfigs --- easybuild/easyblocks/generic/pythonpackage.py | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index dc9881e88c..cc52cba304 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -83,7 +83,7 @@ def det_python_version(python_cmd): return out.strip() -def pick_python_cmd(req_maj_ver=None, req_min_ver=None): +def pick_python_cmd(req_maj_ver=None, req_min_ver=None, max_maj_ver=None, max_min_ver=None): """ Pick 'python' command to use, based on specified version requirements. If the major version is specified, it must be an exact match (==). @@ -130,6 +130,20 @@ def check_python_cmd(python_cmd): log.debug("Minimal requirement for minor Python version not satisfied: %s vs %s", pyver, req_majmin_ver) return False + if max_maj_ver is not None: + if max_min_ver is None: + max_majmin_ver = '%s.0' % max_maj_ver + else: + max_majmin_ver = '%s.%s' % (max_maj_ver, max_min_ver) + + pyver = det_python_version(python_cmd) + + if LooseVersion(pyver) > LooseVersion(max_majmin_ver) + log.debug("Python version (%s) on the system is newer than the maximum python version " + "specified in the easyconfig %s", + pyver, max_majmin_ver) + return False + # all check passed log.debug("All check passed for Python command '%s'!", python_cmd) return True @@ -348,6 +362,8 @@ def extra_options(extra_vars=None): "Enabled by default if the EB option --debug is used.", CUSTOM], 'req_py_majver': [None, "Required major Python version (only relevant when using system Python)", CUSTOM], 'req_py_minver': [None, "Required minor Python version (only relevant when using system Python)", CUSTOM], + 'max_py_majver': [None, "Maximum major Python version (only relevant when using system Python)", CUSTOM], + 'max_py_minver': [None, "Maximum minor Python version (only relevant when using system Python)", CUSTOM], 'sanity_pip_check': [False, "Run 'python -m pip check' to ensure all required Python packages are " "installed and check for any package with an invalid (0.0.0) version.", CUSTOM], 'runtest': [True, "Run unit tests.", CUSTOM], # overrides default @@ -509,16 +525,18 @@ def prepare_python(self): req_py_minver = sys.version_info[1] # if using system Python, go hunting for a 'python' command that satisfies the requirements - python = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver) + python = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver, + max_py_majver=max_py_majver, max_py_minver) if python: self.python_cmd = python self.log.info("Python command being used: %s", self.python_cmd) elif self.require_python: - if req_py_majver is not None or req_py_minver is not None: + if req_py_majver is not None or req_py_minver is not None or max_py_majver is not None or max_py_minver is not None: raise EasyBuildError( "Failed to pick python command that satisfies requirements in the EasyConfigs " - "(req_py_majver = %s, req_py_minver = %s)", req_py_majver, req_py_minver + "(req_py_majver = %s, req_py_minver = %s, max_py_majver = %s, max_py_minver = %s)" + , req_py_majver, req_py_minver, max_py_majver, max_py_minver ) else: raise EasyBuildError("Failed to pick Python command to use") From fdf535b18a74fd32184fa41bf192a757996ed1a0 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 4 Sep 2024 13:12:25 +0200 Subject: [PATCH 04/26] Fix implementation of max_py_majver and max_py_minver in pythonpackage and also implement in pythonbundle --- easybuild/easyblocks/generic/pythonbundle.py | 12 +++++++++--- easybuild/easyblocks/generic/pythonpackage.py | 8 ++++++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonbundle.py b/easybuild/easyblocks/generic/pythonbundle.py index 6a3c533062..49e667f955 100644 --- a/easybuild/easyblocks/generic/pythonbundle.py +++ b/easybuild/easyblocks/generic/pythonbundle.py @@ -107,15 +107,21 @@ def prepare_step(self, *args, **kwargs): if req_py_minver is None: req_py_minver = sys.version_info[1] - python_cmd = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver) + # Get the max_py_majver and max_py_minver from the config + max_py_majver = self.cfg['max_py_majver'] + max_py_minver = self.cfg['max_py_minver'] + + python_cmd = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver, max_py_ver=max_py_majver + max_py_majver=max_py_majver, max_py_minver=max_py_minver) if python_cmd: self.log.info("Python command being used: %s", python_cmd) else: - if req_py_majver is not None or req_py_minver is not None: + if req_py_majver is not None or req_py_minver is not None or max_py_majver is not None or max_py_minver is not None: raise EasyBuildError( "Failed to pick python command that satisfies requirements in the EasyConfigs " - "(req_py_majver = %s, req_py_minver = %s)", req_py_majver, req_py_minver + "(req_py_majver = %s, req_py_minver = %s, max_py_majver = %s, max_py_minver = %s)" + , req_py_majver, req_py_minver, max_py_majver, max_py_minver ) else: raise EasyBuildError("Failed to pick Python command to use") diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index cc52cba304..c199498a1e 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -83,7 +83,7 @@ def det_python_version(python_cmd): return out.strip() -def pick_python_cmd(req_maj_ver=None, req_min_ver=None, max_maj_ver=None, max_min_ver=None): +def pick_python_cmd(req_maj_ver=None, req_min_ver=None, max_py_majver=None, max_py_minver=None): """ Pick 'python' command to use, based on specified version requirements. If the major version is specified, it must be an exact match (==). @@ -524,9 +524,13 @@ def prepare_python(self): if req_py_minver is None: req_py_minver = sys.version_info[1] + # Get the max_py_majver and max_py_minver from the config + max_py_majver = self.cfg['max_py_majver'] + max_py_minver = self.cfg['max_py_minver'] + # if using system Python, go hunting for a 'python' command that satisfies the requirements python = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver, - max_py_majver=max_py_majver, max_py_minver) + max_py_majver=max_py_majver, max_py_minver=max_py_minver) if python: self.python_cmd = python From 944872cfda1375461dbd1206ec23dbed85e2f8f8 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 4 Sep 2024 13:25:53 +0200 Subject: [PATCH 05/26] Fix hound issues --- easybuild/easyblocks/generic/pythonbundle.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonbundle.py b/easybuild/easyblocks/generic/pythonbundle.py index 49e667f955..098c205e2b 100644 --- a/easybuild/easyblocks/generic/pythonbundle.py +++ b/easybuild/easyblocks/generic/pythonbundle.py @@ -111,17 +111,18 @@ def prepare_step(self, *args, **kwargs): max_py_majver = self.cfg['max_py_majver'] max_py_minver = self.cfg['max_py_minver'] - python_cmd = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver, max_py_ver=max_py_majver + python_cmd = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver, max_py_majver=max_py_majver, max_py_minver=max_py_minver) if python_cmd: self.log.info("Python command being used: %s", python_cmd) else: - if req_py_majver is not None or req_py_minver is not None or max_py_majver is not None or max_py_minver is not None: + if req_py_majver is not None or req_py_minver is not None or + max_py_majver is not None or max_py_minver is not None: raise EasyBuildError( "Failed to pick python command that satisfies requirements in the EasyConfigs " - "(req_py_majver = %s, req_py_minver = %s, max_py_majver = %s, max_py_minver = %s)" - , req_py_majver, req_py_minver, max_py_majver, max_py_minver + "(req_py_majver = %s, req_py_minver = %s, max_py_majver = %s, max_py_minver = %s)", + req_py_majver, req_py_minver, max_py_majver, max_py_minver ) else: raise EasyBuildError("Failed to pick Python command to use") From 1d24517b314f01f7a3d59dce237f17a02961c456 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 4 Sep 2024 13:26:23 +0200 Subject: [PATCH 06/26] Formatting --- easybuild/easyblocks/generic/pythonpackage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index c199498a1e..2424378d64 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -539,8 +539,8 @@ def prepare_python(self): if req_py_majver is not None or req_py_minver is not None or max_py_majver is not None or max_py_minver is not None: raise EasyBuildError( "Failed to pick python command that satisfies requirements in the EasyConfigs " - "(req_py_majver = %s, req_py_minver = %s, max_py_majver = %s, max_py_minver = %s)" - , req_py_majver, req_py_minver, max_py_majver, max_py_minver + "(req_py_majver = %s, req_py_minver = %s, max_py_majver = %s, max_py_minver = %s)", + req_py_majver, req_py_minver, max_py_majver, max_py_minver ) else: raise EasyBuildError("Failed to pick Python command to use") From 2194e6483c10e31f0428a34e8c9d7680e15abd80 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 4 Sep 2024 13:27:53 +0200 Subject: [PATCH 07/26] Fix more formatting issues --- easybuild/easyblocks/generic/pythonbundle.py | 4 ++-- easybuild/easyblocks/generic/pythonpackage.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonbundle.py b/easybuild/easyblocks/generic/pythonbundle.py index 098c205e2b..5cb97aa4a8 100644 --- a/easybuild/easyblocks/generic/pythonbundle.py +++ b/easybuild/easyblocks/generic/pythonbundle.py @@ -117,8 +117,8 @@ def prepare_step(self, *args, **kwargs): if python_cmd: self.log.info("Python command being used: %s", python_cmd) else: - if req_py_majver is not None or req_py_minver is not None or - max_py_majver is not None or max_py_minver is not None: + if (req_py_majver is not None or req_py_minver is not None or + max_py_majver is not None or max_py_minver is not None): raise EasyBuildError( "Failed to pick python command that satisfies requirements in the EasyConfigs " "(req_py_majver = %s, req_py_minver = %s, max_py_majver = %s, max_py_minver = %s)", diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 2424378d64..ad43bffaf2 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -536,7 +536,8 @@ def prepare_python(self): self.python_cmd = python self.log.info("Python command being used: %s", self.python_cmd) elif self.require_python: - if req_py_majver is not None or req_py_minver is not None or max_py_majver is not None or max_py_minver is not None: + if (req_py_majver is not None or req_py_minver is not None + or max_py_majver is not None or max_py_minver is not None): raise EasyBuildError( "Failed to pick python command that satisfies requirements in the EasyConfigs " "(req_py_majver = %s, req_py_minver = %s, max_py_majver = %s, max_py_minver = %s)", From 7256a1c94bf047f8dee5cc0fee47c7dc5ac28e3d Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 4 Sep 2024 13:29:05 +0200 Subject: [PATCH 08/26] Fix more formatting issues --- easybuild/easyblocks/generic/pythonbundle.py | 2 +- easybuild/easyblocks/generic/pythonpackage.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonbundle.py b/easybuild/easyblocks/generic/pythonbundle.py index 5cb97aa4a8..7efe626c9f 100644 --- a/easybuild/easyblocks/generic/pythonbundle.py +++ b/easybuild/easyblocks/generic/pythonbundle.py @@ -118,7 +118,7 @@ def prepare_step(self, *args, **kwargs): self.log.info("Python command being used: %s", python_cmd) else: if (req_py_majver is not None or req_py_minver is not None or - max_py_majver is not None or max_py_minver is not None): + max_py_majver is not None or max_py_minver is not None): raise EasyBuildError( "Failed to pick python command that satisfies requirements in the EasyConfigs " "(req_py_majver = %s, req_py_minver = %s, max_py_majver = %s, max_py_minver = %s)", diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index ad43bffaf2..71ce365918 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -537,7 +537,7 @@ def prepare_python(self): self.log.info("Python command being used: %s", self.python_cmd) elif self.require_python: if (req_py_majver is not None or req_py_minver is not None - or max_py_majver is not None or max_py_minver is not None): + or max_py_majver is not None or max_py_minver is not None): raise EasyBuildError( "Failed to pick python command that satisfies requirements in the EasyConfigs " "(req_py_majver = %s, req_py_minver = %s, max_py_majver = %s, max_py_minver = %s)", From 8f2244f6fe247b0ab88b989957dc9b71381fc425 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 4 Sep 2024 13:30:28 +0200 Subject: [PATCH 09/26] Fix indent --- easybuild/easyblocks/generic/pythonpackage.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 71ce365918..a98dbf8825 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -130,19 +130,19 @@ def check_python_cmd(python_cmd): log.debug("Minimal requirement for minor Python version not satisfied: %s vs %s", pyver, req_majmin_ver) return False - if max_maj_ver is not None: - if max_min_ver is None: - max_majmin_ver = '%s.0' % max_maj_ver - else: - max_majmin_ver = '%s.%s' % (max_maj_ver, max_min_ver) - - pyver = det_python_version(python_cmd) - - if LooseVersion(pyver) > LooseVersion(max_majmin_ver) - log.debug("Python version (%s) on the system is newer than the maximum python version " - "specified in the easyconfig %s", - pyver, max_majmin_ver) - return False + if max_maj_ver is not None: + if max_min_ver is None: + max_majmin_ver = '%s.0' % max_maj_ver + else: + max_majmin_ver = '%s.%s' % (max_maj_ver, max_min_ver) + + pyver = det_python_version(python_cmd) + + if LooseVersion(pyver) > LooseVersion(max_majmin_ver) + log.debug("Python version (%s) on the system is newer than the maximum python version " + "specified in the easyconfig %s", + pyver, max_majmin_ver) + return False # all check passed log.debug("All check passed for Python command '%s'!", python_cmd) From beb99f5eb06706d7cdfd0da298fa21d0c44eb508 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 4 Sep 2024 13:31:38 +0200 Subject: [PATCH 10/26] Fix indent --- easybuild/easyblocks/generic/pythonpackage.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index a98dbf8825..f4420ab945 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -133,16 +133,16 @@ def check_python_cmd(python_cmd): if max_maj_ver is not None: if max_min_ver is None: max_majmin_ver = '%s.0' % max_maj_ver - else: + else: max_majmin_ver = '%s.%s' % (max_maj_ver, max_min_ver) - pyver = det_python_version(python_cmd) + pyver = det_python_version(python_cmd) - if LooseVersion(pyver) > LooseVersion(max_majmin_ver) - log.debug("Python version (%s) on the system is newer than the maximum python version " - "specified in the easyconfig %s", + if LooseVersion(pyver) > LooseVersion(max_majmin_ver) + log.debug("Python version (%s) on the system is newer than the maximum python version " + "specified in the easyconfig %s", pyver, max_majmin_ver) - return False + return False # all check passed log.debug("All check passed for Python command '%s'!", python_cmd) From 40020d25ccce129cd410a6971b29c921d23e00b3 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 4 Sep 2024 13:32:38 +0200 Subject: [PATCH 11/26] Add missing colon --- easybuild/easyblocks/generic/pythonpackage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index f4420ab945..1f8bf125a6 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -138,10 +138,10 @@ def check_python_cmd(python_cmd): pyver = det_python_version(python_cmd) - if LooseVersion(pyver) > LooseVersion(max_majmin_ver) + if LooseVersion(pyver) > LooseVersion(max_majmin_ver): log.debug("Python version (%s) on the system is newer than the maximum python version " "specified in the easyconfig %s", - pyver, max_majmin_ver) + pyver, max_majmin_ver) return False # all check passed From debc72ebc656230f830861921f64492cda5ee210 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 4 Sep 2024 13:34:16 +0200 Subject: [PATCH 12/26] Renamed parameters to match argument names --- easybuild/easyblocks/generic/pythonpackage.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 1f8bf125a6..f8cbc92b75 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -130,11 +130,11 @@ def check_python_cmd(python_cmd): log.debug("Minimal requirement for minor Python version not satisfied: %s vs %s", pyver, req_majmin_ver) return False - if max_maj_ver is not None: - if max_min_ver is None: - max_majmin_ver = '%s.0' % max_maj_ver + if max_py_majver is not None: + if max_py_minver is None: + max_majmin_ver = '%s.0' % max_py_majver else: - max_majmin_ver = '%s.%s' % (max_maj_ver, max_min_ver) + max_majmin_ver = '%s.%s' % (max_py_majver, max_py_minver) pyver = det_python_version(python_cmd) From 1a6802bbca5d0661908bd2b3948fa0e3a7c8e643 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 5 Sep 2024 16:31:45 +0200 Subject: [PATCH 13/26] Fix https://github.com/easybuilders/easybuild-easyblocks/pull/3430/files#r1745641431 --- easybuild/easyblocks/generic/pythonbundle.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonbundle.py b/easybuild/easyblocks/generic/pythonbundle.py index 7efe626c9f..2cd7d645e8 100644 --- a/easybuild/easyblocks/generic/pythonbundle.py +++ b/easybuild/easyblocks/generic/pythonbundle.py @@ -91,6 +91,10 @@ def prepare_step(self, *args, **kwargs): # when system Python is used, the first 'python' command in $PATH will not be $EBROOTPYTHON/bin/python, # since $EBROOTPYTHON is set to just 'Python' in that case # (see handling of allow_system_deps in EasyBlock.prepare_step) + req_py_majver = None + req_py_minver = None + max_py_majver = None + max_py_minver = None if which('python') == os.path.join(python_root, 'bin', 'python'): # if we're using a proper Python dependency, let det_pylibdir use 'python' like it does by default python_cmd = None @@ -127,9 +131,8 @@ def prepare_step(self, *args, **kwargs): else: raise EasyBuildError("Failed to pick Python command to use") - if python_cmd: - self.all_pylibdirs = get_pylibdirs(python_cmd=python_cmd) - self.pylibdir = self.all_pylibdirs[0] + self.all_pylibdirs = get_pylibdirs(python_cmd=python_cmd) + self.pylibdir = self.all_pylibdirs[0] # if 'python' is not used, we need to take that into account in the extensions filter # (which is also used during the sanity check) From 191a691cb330eee089b9278e1635681b16e5d84e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 5 Sep 2024 16:41:33 +0200 Subject: [PATCH 14/26] Initialize variables, so that they are present when we check for them --- easybuild/easyblocks/generic/pythonpackage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index f8cbc92b75..7935ca03c3 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -514,6 +514,8 @@ def prepare_python(self): python = os.path.join(python_root, 'bin', 'python') self.log.debug("Retaining 'python' command for Python dependency: %s", python) + req_py_majver = None + req_py_minver = None if python is None: # if no Python version requirements are specified, # use major/minor version of Python being used in this EasyBuild session From 4525a7655d05034faee44c82c2740108ab52d000 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 5 Sep 2024 17:04:51 +0200 Subject: [PATCH 15/26] Move the complicated error reporting within the 'python is None' block, there the req_py_majver and friends are guaranteed to be defined --- easybuild/easyblocks/generic/pythonpackage.py | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 7935ca03c3..07c4f6b75c 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -514,8 +514,6 @@ def prepare_python(self): python = os.path.join(python_root, 'bin', 'python') self.log.debug("Retaining 'python' command for Python dependency: %s", python) - req_py_majver = None - req_py_minver = None if python is None: # if no Python version requirements are specified, # use major/minor version of Python being used in this EasyBuild session @@ -534,21 +532,25 @@ def prepare_python(self): python = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver, max_py_majver=max_py_majver, max_py_minver=max_py_minver) - if python: - self.python_cmd = python - self.log.info("Python command being used: %s", self.python_cmd) - elif self.require_python: - if (req_py_majver is not None or req_py_minver is not None - or max_py_majver is not None or max_py_minver is not None): - raise EasyBuildError( - "Failed to pick python command that satisfies requirements in the EasyConfigs " - "(req_py_majver = %s, req_py_minver = %s, max_py_majver = %s, max_py_minver = %s)", - req_py_majver, req_py_minver, max_py_majver, max_py_minver - ) + # Check if we have python by now. If not, and if self.require_python, raise a sensible error + if python: + self.python_cmd = python + self.log.info("Python command being used: %s", self.python_cmd) + elif self.require_python: + if (req_py_majver is not None or req_py_minver is not None + or max_py_majver is not None or max_py_minver is not None): + raise EasyBuildError( + "Failed to pick python command that satisfies requirements in the EasyConfigs " + "(req_py_majver = %s, req_py_minver = %s, max_py_majver = %s, max_py_minver = %s)", + req_py_majver, req_py_minver, max_py_majver, max_py_minver + ) + else: + raise EasyBuildError("Failed to pick Python command to use") else: - raise EasyBuildError("Failed to pick Python command to use") + self.log.warning("No Python command found!") else: - self.log.warning("No Python command found!") + self.python_cmd = python + self.log.info("Python command being used: %s", self.python_cmd) if self.python_cmd: # set Python lib directories From 9999bf18a3d9d17db87b43e12016bae4128984f4 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 5 Sep 2024 17:06:28 +0200 Subject: [PATCH 16/26] Pull error inside block that is executed when using system python --- easybuild/easyblocks/generic/pythonbundle.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonbundle.py b/easybuild/easyblocks/generic/pythonbundle.py index 2cd7d645e8..b3ca25da24 100644 --- a/easybuild/easyblocks/generic/pythonbundle.py +++ b/easybuild/easyblocks/generic/pythonbundle.py @@ -118,18 +118,15 @@ def prepare_step(self, *args, **kwargs): python_cmd = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver, max_py_majver=max_py_majver, max_py_minver=max_py_minver) - if python_cmd: - self.log.info("Python command being used: %s", python_cmd) - else: - if (req_py_majver is not None or req_py_minver is not None or - max_py_majver is not None or max_py_minver is not None): + # If pick_python_cmd didn't find a (system) python command, we should raise an error + if python_cmd: + self.log.info("Python command being used: %s", python_cmd) + else: raise EasyBuildError( "Failed to pick python command that satisfies requirements in the EasyConfigs " "(req_py_majver = %s, req_py_minver = %s, max_py_majver = %s, max_py_minver = %s)", req_py_majver, req_py_minver, max_py_majver, max_py_minver ) - else: - raise EasyBuildError("Failed to pick Python command to use") self.all_pylibdirs = get_pylibdirs(python_cmd=python_cmd) self.pylibdir = self.all_pylibdirs[0] From d552094936b35df9c64c908896f089668cbfe811 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Sep 2024 10:37:06 +0200 Subject: [PATCH 17/26] No need to initialize the req/max python version variables --- easybuild/easyblocks/generic/pythonbundle.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonbundle.py b/easybuild/easyblocks/generic/pythonbundle.py index 9bd0c741d9..ad7e0af9d0 100644 --- a/easybuild/easyblocks/generic/pythonbundle.py +++ b/easybuild/easyblocks/generic/pythonbundle.py @@ -91,10 +91,6 @@ def prepare_step(self, *args, **kwargs): # when system Python is used, the first 'python' command in $PATH will not be $EBROOTPYTHON/bin/python, # since $EBROOTPYTHON is set to just 'Python' in that case # (see handling of allow_system_deps in EasyBlock.prepare_step) - req_py_majver = None - req_py_minver = None - max_py_majver = None - max_py_minver = None if which('python') == os.path.join(python_root, 'bin', 'python'): # if we're using a proper Python dependency, let det_pylibdir use 'python' like it does by default python_cmd = None From 1a23bf34d524c36f5ad840681a6993df44aa2795 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Sep 2024 10:46:52 +0200 Subject: [PATCH 18/26] Rephrase debug message --- easybuild/easyblocks/generic/pythonpackage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 1838562cea..eae50cae7c 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -139,8 +139,8 @@ def check_python_cmd(python_cmd): pyver = det_python_version(python_cmd) if LooseVersion(pyver) > LooseVersion(max_majmin_ver): - log.debug("Python version (%s) on the system is newer than the maximum python version " - "specified in the easyconfig %s", + log.debug("Python version (%s) on the system is newer than the maximum supported " + "Python version specified in the easyconfig (%s)", pyver, max_majmin_ver) return False From 1e6f7732c88d503acd75f4ee4bd07487b26b2d3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Sep 2024 13:23:44 +0200 Subject: [PATCH 19/26] add two tests for pick_python_cmd that use max_py_majver and max_py_minver --- test/easyblocks/module.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/easyblocks/module.py b/test/easyblocks/module.py index 6e37946c3a..122e1285ce 100644 --- a/test/easyblocks/module.py +++ b/test/easyblocks/module.py @@ -231,6 +231,8 @@ def test_pythonpackage_pick_python_cmd(self): self.assertTrue(pick_python_cmd(2) is not None) self.assertTrue(pick_python_cmd(2, 6) is not None) self.assertTrue(pick_python_cmd(123, 456) is None) + self.assertTrue(pick_python_cmd(2, 6, 123, 456) is not None) + self.assertTrue(pick_python_cmd(2, 6, 1, 1) is None) def template_module_only_test(self, easyblock, name, version='1.3.2', extra_txt='', tmpdir=None): From 0c5cc9d366d035a223dca2df1db849503451f1ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Sep 2024 13:31:01 +0200 Subject: [PATCH 20/26] deliberately try to break a test --- test/easyblocks/module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/easyblocks/module.py b/test/easyblocks/module.py index 122e1285ce..e79dd3dda5 100644 --- a/test/easyblocks/module.py +++ b/test/easyblocks/module.py @@ -232,7 +232,7 @@ def test_pythonpackage_pick_python_cmd(self): self.assertTrue(pick_python_cmd(2, 6) is not None) self.assertTrue(pick_python_cmd(123, 456) is None) self.assertTrue(pick_python_cmd(2, 6, 123, 456) is not None) - self.assertTrue(pick_python_cmd(2, 6, 1, 1) is None) + self.assertTrue(pick_python_cmd(2, 6, 1, 1) is not None) def template_module_only_test(self, easyblock, name, version='1.3.2', extra_txt='', tmpdir=None): From 53778ed084d8f22bacb2e995c5b14964555a08b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Sep 2024 13:33:09 +0200 Subject: [PATCH 21/26] this indeed broke the test, so revert it --- test/easyblocks/module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/easyblocks/module.py b/test/easyblocks/module.py index e79dd3dda5..122e1285ce 100644 --- a/test/easyblocks/module.py +++ b/test/easyblocks/module.py @@ -232,7 +232,7 @@ def test_pythonpackage_pick_python_cmd(self): self.assertTrue(pick_python_cmd(2, 6) is not None) self.assertTrue(pick_python_cmd(123, 456) is None) self.assertTrue(pick_python_cmd(2, 6, 123, 456) is not None) - self.assertTrue(pick_python_cmd(2, 6, 1, 1) is not None) + self.assertTrue(pick_python_cmd(2, 6, 1, 1) is None) def template_module_only_test(self, easyblock, name, version='1.3.2', extra_txt='', tmpdir=None): From 74704714519ccfb58c4dacdd5c978a86c702397b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 14 Sep 2024 23:15:43 +0200 Subject: [PATCH 22/26] bump version to 4.9.4dev --- easybuild/easyblocks/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/__init__.py b/easybuild/easyblocks/__init__.py index 938112f55a..d72e0449f0 100644 --- a/easybuild/easyblocks/__init__.py +++ b/easybuild/easyblocks/__init__.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.9.3') +VERSION = LooseVersion('4.9.4.dev0') UNKNOWN = 'UNKNOWN' From 7074ad263a9281fe6f8727d5a3f17106aaa8ddf5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 15 Sep 2024 16:49:35 +0200 Subject: [PATCH 23/26] copy EasyConfig instance in constructor of Bundle and Cargo easyblocks before changes to it --- easybuild/easyblocks/generic/bundle.py | 4 ++++ easybuild/easyblocks/generic/cargo.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index 23da2338c2..b8abecf8fc 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -85,6 +85,10 @@ def __init__(self, *args, **kwargs): if self.cfg['patches']: raise EasyBuildError("List of patches for bundle itself must be empty, found %s", self.cfg['patches']) + # copy EasyConfig instance before we make changes to it + # (like adding component sources to top-level sources easyconfig parameter) + self.cfg = self.cfg.copy() + # disable templating to avoid premature resolving of template values self.cfg.enable_templating = False diff --git a/easybuild/easyblocks/generic/cargo.py b/easybuild/easyblocks/generic/cargo.py index c49e950969..d03407862d 100644 --- a/easybuild/easyblocks/generic/cargo.py +++ b/easybuild/easyblocks/generic/cargo.py @@ -151,6 +151,9 @@ def __init__(self, *args, **kwargs): 'filename': self.crate_src_filename(crate, version), }) + # copy EasyConfig instance before we make changes to it + self.cfg = self.cfg.copy() + self.cfg.update('sources', sources) @property From 6ca8a9e346cd5b80624fb14eda5377247eee5459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= Date: Mon, 16 Sep 2024 16:00:51 +0200 Subject: [PATCH 24/26] GCC: Fix NVPTX mapping error when with no CC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When no compute capability is set but NVPTX is enabled, trying to figure out the NVPTX architecture fails with the error: ``` File "easybuild/easyblocks/g/gcc.py", line 431, in map_nvptx_capability return sorted_gcc_cc[0] IndexError: list index out of range ``` The error occurs because of an insufficient check for an unset CUDA compute capability. This commit changes the checked conditions, so that empty lists are also correctly handled. Signed-off-by: Jan André Reuter --- easybuild/easyblocks/g/gcc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/g/gcc.py b/easybuild/easyblocks/g/gcc.py index 1ec0f0e37b..76562c4001 100644 --- a/easybuild/easyblocks/g/gcc.py +++ b/easybuild/easyblocks/g/gcc.py @@ -383,7 +383,7 @@ def map_nvptx_capability(self): architecture_mappings_replacement = "misa=," # Determine which compute capabilities are configured. If there are none, return immediately. - if cuda_cc_list is None: + if not cuda_cc_list: return None cuda_sm_list = [f"sm_{cc.replace('.', '')}" for cc in cuda_cc_list] From 012324e47a5a5f9cfbd659149912e8fbefe88e42 Mon Sep 17 00:00:00 2001 From: lara Date: Thu, 19 Sep 2024 16:24:59 +0200 Subject: [PATCH 25/26] add licence to NCCL installation --- easybuild/easyblocks/n/nccl.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easybuild/easyblocks/n/nccl.py b/easybuild/easyblocks/n/nccl.py index ea9acc25f8..15ce298ce0 100644 --- a/easybuild/easyblocks/n/nccl.py +++ b/easybuild/easyblocks/n/nccl.py @@ -26,10 +26,14 @@ EasyBuild support for building NCCL, implemented as an easyblock @author: Simon Branford (University of Birmingham) +@author: Lara Peeters (Gent University) """ +import os + from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.config import build_option from easybuild.tools.systemtools import get_shared_lib_ext +from easybuild.tools.filetools import copy_file class EB_NCCL(ConfigureMake): @@ -67,6 +71,8 @@ def install_step(self): """Install NCCL""" self.cfg.update('installopts', "PREFIX=%s" % self.installdir) + copy_file(os.path.join(self.cfg['start_dir'], 'LICENSE.txt'), os.path.join(self.installdir, 'LICENSE.txt')) + super(EB_NCCL, self).install_step() def sanity_check_step(self): From 2e19a285a7339f381203644cbae83a31faaff318 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 22 Sep 2024 09:14:50 +0200 Subject: [PATCH 26/26] prepare release notes for EasyBuild v4.9.4 + bump version to 4.9.4 --- RELEASE_NOTES | 15 +++++++++++++++ easybuild/easyblocks/__init__.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index ec28f4fb93..c847c65b3a 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -6,10 +6,25 @@ These release notes can also be consulted at http://easybuild.readthedocs.org/en The latest version of easybuild-easyblocks provides 259 software-specific easyblocks and 43 generic easyblocks. +v4.9.4 (22 september 2024) +-------------------------- + +update/bugfix release + +- various enhancements, including: + - allow Python bundles and packages to specify a maximum Python version for the system toolchain (#3431) + - copy EasyConfig instance in constructor of Bundle and Cargo easyblocks before making changes to it (#3448) + - fix crash in GCC easyblock when cuda-compute-capabilities EasyBuild configuration option is not set (#3449) +- various bug fixes, including: + - ignore Python from virtualenvs in GROMACS configure via -DPython3_FIND_VIRTUALENV=STANDARD (#3283) + - enhance custom easyblock for NCCL: add licence to NCCL installation (#3451) + + v4.9.3 (14 September 2024) -------------------------- update/bugfix release + - minor updates, including: - update custom easyblock for Tensorflow for versions 2.14 + 2.15 (#3303) - add support for versions >= 2024a to MCR easyblock (#3369) diff --git a/easybuild/easyblocks/__init__.py b/easybuild/easyblocks/__init__.py index d72e0449f0..7b5379a951 100644 --- a/easybuild/easyblocks/__init__.py +++ b/easybuild/easyblocks/__init__.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.9.4.dev0') +VERSION = LooseVersion('4.9.4') UNKNOWN = 'UNKNOWN'