\\
**[[cluster:0|Back]]**
===== EasyBuild =====
EasyBuild is a software build and installation framework that allows you to manage (scientific) software on High Performance Computing (HPC) systems in an efficient way.
EasyBuild 4.4.2 supports 2469 different software packages (incl. toolchains, bundles):
* https://docs.easybuild.io/en/latest/version-specific/Supported_software.html
* https://docs.easybuild.io/en/latest/Installation.html#requirements
* https://docs.easybuild.io/en/latest/version-specific/help.html
**NOTE** this eats up lots of disk space with multiple compiler and tool chains as you will see, hence installed in /sanscratch/CENTOS7
# install using pip from python =>3.5
export PATH=/share/apps/CENTOS7/python/3.8.3/bin:$PATH
export LD_LIBRARY_PATH=/share/apps/CENTOS7/python/3.8.3/lib:$LD_LIBRARY_PATH
export PYTHONPATH=/share/apps/CENTOS7/python/3.8.3/lib/python3.8/site-packages:$PYTHONPATH
# easybuild install into python
[hmeij@greentail52 ~]$ which pip3
/share/apps/CENTOS7/python/3.8.3/bin/pip3
[hmeij@greentail52 ~]$ pip3 install easybuild
Collecting easybuild
Downloading easybuild-4.4.2.tar.gz (10 kB)
Collecting easybuild-framework==4.4.2
Downloading easybuild-framework-4.4.2.tar.gz (2.0 MB)
Collecting easybuild-easyblocks==4.4.2
Downloading easybuild-easyblocks-4.4.2.tar.gz (501 kB)
Collecting easybuild-easyconfigs==4.4.2
Downloading easybuild-easyconfigs-4.4.2.tar.gz (6.8 MB)
Building wheels for collected packages:...
Successfully installed easybuild-4.4.2 easybuild-easyblocks-4.4.2
easybuild-easyconfigs-4.4.2 easybuild-framework-4.4.2
[hmeij@greentail52 ~]$ eb --version
This is EasyBuild 4.4.2 (framework: 4.4.2, easyblocks: 4.4.2) on host greentail52.
# yum install Lmod
# then create local eb module usinf easybuild
eb --install-latest-eb-release --prefix $HOME/easybuild
== COMPLETED: Installation ended successfully (took 2 mins 2 secs)
== Results of the build can be found in the log file(s)
/zfshomes/hmeij/easybuild/software/EasyBuild/4.4.2/\
easybuild/easybuild-EasyBuild-4.4.2-20211026.101256.log
[hmeij@greentail52]$ module use /zfshomes/hmeij/easybuild/modules/all
[hmeij@greentail52]$ module avail
--- /zfshomes/hmeij/easybuild/modules/all ---
EasyBuild/4.4.2
--- /etc/modulefiles ---
mpi/openmpi-x86_64
--- /usr/share/lmod/lmod/modulefiles/Core ---
lmod settarg
# load module
[hmeij@greentail52 easybuild]$ module load EasyBuild/4.4.2
[hmeij@greentail52 easybuild]$ module list
Currently Loaded Modules:
1) EasyBuild/4.4.2
[hmeij@greentail52]$ which eb
~/easybuild/software/EasyBuild/4.4.2/bin/eb
** Can you copy this to say:** module use /opt/ohpc/pub/easybuild/modules/all
===== Usage =====
* set PYTHONPATH, see above
* module use /zfshomes/hmeij/easybuild/modules/all
* module load EasyBuild/4.4.2
* using --prefix or by defining $EASYBUILD_PREFIX, in order to redefine the
* build/install/source path prefix to be used;
* default value is: $HOME/.local/easybuild.
* To enable dependency resolution, use the --robot command line option
* eb --list-toolchains # eb resides in ~hmeij/easybuild, load module first
* eb --search mesquite
* eb --show-default-configfiles
* eb SOMETHING.eb --dry-run --robot --missing-modules # get a road map!
# An example configuration file
[basic]
# always enable logging to stdout
logtostdout = true
[config]
# use Lmod as modules tool
modules-tool: Lmod
# use different default installation path
prefix=/home/you/work/easybuild/
# on greentail52 we are using environment modules written in lua
[root@greentail52 ~]# rpm -qa |grep -i Lmod
Lmod-8.2.7-1.el7.x86_64
[root@greentail52 ~]# less /share/apps/CENTOS7/easybuild/4.4.2/modules/all/flex/2.6.4.lua
# sidebar
# on compute nodes enable Lmod and environment-modules if not present (likely)
# yumdownloader --desdir=somewhere package-name
# centos 7 nodes only
cd /sanscratch/tmp/rpms/ \
rpm -ivh environment-modules-3.2.10-10.el7.x86_64.rpm \
rpm -ivh lua-filesystem-1.6.2-2.el7.x86_64.rpm \
lua-json-1.3.2-2.el7.noarch.rpm \
lua-posix-32-2.el7.x86_64.rpm \
lua-term-0.03-3.el7.x86_64.rpm \
lua-bitop-1.0.2-3.el7.x86_64.rpm \
lua-lpeg-0.12-1.el7.x86_64.rpm \
Lmod-8.2.7-1.el7.x86_64.rpm \
# log out and log back in
# end sidebar
# example easybuild installation of software
eb Bowtie2-2.4.2-GCC-9.3.0.eb --dry-run --robot --prefix=/sanscratch/CENTOS7/easybuild/4.4.2
# do not forget prefix else it goes to $HOME/.local/easybuild
# once done
module use /sanscratch/CENTOS7/easybuild/4.4.2/modules/all
# use 'module avail' to discover modules. then load
[hmeij@greentail52 ~]$ module load Bowtie2/2.4.2-GCC-9.3.0
[hmeij@greentail52 ~]$ module list
Currently Loaded Modules:
1) EasyBuild/4.4.2
2) GCCcore/9.3.0
3) zlib/1.2.11-GCCcore-9.3.0
4) binutils/2.34-GCCcore-9.3.0
5) GCC/9.3.0
6) tbb/2020.1-GCCcore-9.3.0
7) Bowtie2/2.4.2-GCC-9.3.0
[hmeij@greentail52 ~]$ which bowtie2
/sanscratch/CENTOS7/easybuild/4.4.2/software/Bowtie2/2.4.2-GCC-9.3.0/bin/bowtie2
Another compilation/installation (one can observe how complicated all this is going to get). This uses GCC 10.2.0 vs GCC 9.3.0 above.
# this required libibverbs and libibverbs-devel so not sure it will run on compute nodes
eb astropy-4.2.1-intelcuda-2020b.eb --robot --prefix=/sanscratch/CENTOS7/easybuild/4.4.2
# then ran into icc license problems, 2020b, check out license failed
# trying i/icc, fails finding tarball file for 2016 parallel studio
# hint add eula flag when trying i/intel toolchain
eb intel-2021a.eb --robot \
--prefix=/sanscratch/CENTOS7/easybuild/4.4.2 \
--accept-eula-for=Intel-oneAPI,Intel-C,Intel-C++,Intel-icc,Intel-Fortran,Intel-ifort,Intel-MPI,Intel-MKL,Intel-CUDA
# built intel-compilers/2021.2.0, hangs on Perl-5.32.1-GCCcore-10.3.0-minimal.eb build? try separate
# or try: Intel C, C++ & Fortran compilers (classic and oneAPI)
intel-compilers-2021.3.0.eb
# installed, no errors
# or try: Intel C, C++ & Fortran compilers, Intel MPI & Intel MKL, with CUDA toolkit
intelcuda-2020b.eb
# try 2021.3.0 dependency
# switched away from intel nightmare, there should be a OneAPI toolchain...
# created a astropy-fosscuda file ...
easyblock = 'Toolchain'
name = 'fosscuda'
version = '2020b'
homepage = '(none)'
description = """GCC based compiler toolchain __with CUDA support__, and including
# to find config, change to that location - also run from ~hmeij login shell
# aand you must set python 3.8.3 as indicated above, else run.py from
# native OS python is found yielding 'make -j 16' error
cd easybuild/software/EasyBuild/4.4.2/easybuild/easyconfigs/a/astropy/ \
eb astropy-4.2.1-fosscuda-2020b.eb \
--dry-run --robot --prefix=/sanscratch/CENTOS7/easybuild/4.4.2
# fosscuda attempt worked, see below for continuation
===== Astropy =====
Continuing with installation of ''greentail52'''s local disk array with astropy. Note: had to manually install numpy in python 3.8.6 (easybuild misconfiguration, or rather, 2 of 10k tests failed so no go, really?)
module use /sanscratch/CENTOS7/easybuild/4.4.2/modules/all
module avail
module load astropy/4.2.1-fosscuda-2020b
module list
# now we see why we need a software install manager
Currently Loaded Modules:
1) GCCcore/10.2.0 13) libevent/2.1.12-GCCcore-10.2.0 25) bzip2/1.0.8-GCCcore-10.2.0
2) zlib/1.2.11-GCCcore-10.2.0 14) Check/0.15.2-GCCcore-10.2.0 26) ncurses/6.2-GCCcore-10.2.0
3) binutils/2.35-GCCcore-10.2.0 15) GDRCopy/2.1-GCCcore-10.2.0-CUDA-11.1.1 27) libreadline/8.0-GCCcore-10.2.0
4) GCC/10.2.0 16) UCX/1.9.0-GCCcore-10.2.0-CUDA-11.1.1 28) Tcl/8.6.10-GCCcore-10.2.0
5) CUDAcore/11.1.1 17) libfabric/1.11.0-GCCcore-10.2.0 29) SQLite/3.33.0-GCCcore-10.2.0
6) CUDA/11.1.1-GCC-10.2.0 18) PMIx/3.1.5-GCCcore-10.2.0 30) GMP/6.2.0-GCCcore-10.2.0
7) gcccuda/2020b 19) OpenMPI/4.0.5-gcccuda-2020b 31) libffi/3.3-GCCcore-10.2.0
8) numactl/2.0.13-GCCcore-10.2.0 20) OpenBLAS/0.3.12-GCC-10.2.0 32) Python/3.8.6-GCCcore-10.2.0
9) XZ/5.2.5-GCCcore-10.2.0 21) gompic/2020b 33) pybind11/2.6.0-GCCcore-10.2.0
10) libxml2/2.9.10-GCCcore-10.2.0 22) FFTW/3.3.8-gompic-2020b 34) SciPy-bundle/2020.11-fosscuda-2020b
11) libpciaccess/0.16-GCCcore-10.2.0 23) ScaLAPACK/2.1.0-gompic-2020b 35) astropy/4.2.1-fosscuda-2020b
12) hwloc/2.2.0-GCCcore-10.2.0 24) fosscuda/2020b
# in our environment we must location of libs, this seems odd
export LD_LIBRARY_PATH=/sanscratch/CENTOS7/easybuild/4.4.2/software/CUDAcore/11.1.1/stubs/lib64:$LD_LIBRARY_PATH
[hmeij@greentail52 ~]$ which python nvcc mpirun
/sanscratch/CENTOS7/easybuild/4.4.2/software/Python/3.8.6-GCCcore-10.2.0/bin/python
/sanscratch/CENTOS7/easybuild/4.4.2/software/CUDAcore/11.1.1/bin/nvcc
/sanscratch/CENTOS7/easybuild/4.4.2/software/OpenMPI/4.0.5-gcccuda-2020b/bin/mpirun
# imports on greentail52
[hmeij@greentail52 ~]$ python
Python 3.8.6 (default, Oct 31 2021, 14:41:27)
[GCC 10.2.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import numpy as np
>>> import scipy as sp
>>> import astropy as ap
>>> dir()
['__annotations__', '__builtins__', '__doc__', '__loader__', '__name__', '__package__', '__spec__', 'ap', 'np', 'sp']
# testing imports on n84 for example
[hmeij@n84 ~]$ module use /sanscratch/CENTOS7/easybuild/4.4.2/modules/all
[hmeij@n84 ~]$ module load astropy/4.2.1-fosscuda-2020b
[hmeij@n84 ~]$ module list
...
Helios GPU tutorial\\
https://heliosexo.readthedocs.io/en/latest/sections/tutorial.html?highlight=GPU#tutorial
# pycuda not working, why would fosscuda?
ImportError: /sanscratch/CENTOS7/easybuild/4.4.2/software/Python/3.8.6-GCCcore-10.2.0/lib/python3.8/site-packages/pycuda/_driver.cpython-38-x86_64-linux-gnu.so:
undefined symbol: cuDevicePrimaryCtxRelease_v2
https://stackoverflow.com/questions/66035567/missing-symbol-cudeviceprimaryctxrelease-vs-cudeviceprimaryctxrelease-v2
===== Emcee =====
module list
Currently Loaded Modules:
1) GCCcore/8.2.0 14) ScaLAPACK/2.0.2-gompi-2019a-OpenBLAS-0.3.5
2) zlib/1.2.11-GCCcore-8.2.0 15) foss/2019a
3) binutils/2.31.1-GCCcore-8.2.0 16) bzip2/1.0.6-GCCcore-8.2.0
4) GCC/8.2.0-2.31.1 17) ncurses/6.1-GCCcore-8.2.0
5) numactl/2.0.12-GCCcore-8.2.0 18) libreadline/8.0-GCCcore-8.2.0
6) XZ/5.2.4-GCCcore-8.2.0 19) Tcl/8.6.9-GCCcore-8.2.0
7) libxml2/2.9.8-GCCcore-8.2.0 20) SQLite/3.27.2-GCCcore-8.2.0
8) libpciaccess/0.14-GCCcore-8.2.0 21) GMP/6.1.2-GCCcore-8.2.0
9) hwloc/1.11.11-GCCcore-8.2.0 22) libffi/3.2.1-GCCcore-8.2.0
10) OpenMPI/3.1.3-GCC-8.2.0-2.31.1 23) Python/3.7.2-GCCcore-8.2.0 <---
11) OpenBLAS/0.3.5-GCC-8.2.0-2.31.1 24) SciPy-bundle/2019.03-foss-2019a
12) gompi/2019a 25) emcee/2.2.1-foss-2019a
13) FFTW/3.3.8-gompi-2019a
I noticed emcee loaded two python versions, the default being 3.7.2 (requires centos7). But you can unload this python module and load module Python/2.7.15-GCCcore-8.2.0 and emcee will import. That means with 2.7.15 emcee could run on centos6 nodes.
Flexible but weird.
=====PyCUDA =====
[hmeij@greentail52 ~]$ module load PyCUDA/2020.1-fosscuda-2020b
[hmeij@greentail52 ~]$ module list
Currently Loaded Modules:
1) GCCcore/10.2.0 20) OpenBLAS/0.3.12-GCC-10.2.0
2) zlib/1.2.11-GCCcore-10.2.0 21) gompic/2020b
3) binutils/2.35-GCCcore-10.2.0 22) FFTW/3.3.8-gompic-2020b
4) GCC/10.2.0 23) ScaLAPACK/2.1.0-gompic-2020b
5) CUDAcore/11.1.1 24) fosscuda/2020b
6) CUDA/11.1.1-GCC-10.2.0 25) bzip2/1.0.8-GCCcore-10.2.0
7) gcccuda/2020b 26) ncurses/6.2-GCCcore-10.2.0
8) numactl/2.0.13-GCCcore-10.2.0 27) libreadline/8.0-GCCcore-10.2.0
9) XZ/5.2.5-GCCcore-10.2.0 28) Tcl/8.6.10-GCCcore-10.2.0
10) libxml2/2.9.10-GCCcore-10.2.0 29) SQLite/3.33.0-GCCcore-10.2.0
11) libpciaccess/0.16-GCCcore-10.2.0 30) GMP/6.2.0-GCCcore-10.2.0
12) hwloc/2.2.0-GCCcore-10.2.0 31) libffi/3.3-GCCcore-10.2.0
13) libevent/2.1.12-GCCcore-10.2.0 32) Python/3.8.6-GCCcore-10.2.0
14) Check/0.15.2-GCCcore-10.2.0 33) pybind11/2.6.0-GCCcore-10.2.0
15) GDRCopy/2.1-GCCcore-10.2.0-CUDA-11.1.1 34) SciPy-bundle/2020.11-fosscuda-2020b
16) UCX/1.9.0-GCCcore-10.2.0-CUDA-11.1.1 35) Boost/1.74.0-GCC-10.2.0
17) libfabric/1.11.0-GCCcore-10.2.0 36) Boost.Python/1.74.0-GCC-10.2.0
18) PMIx/3.1.5-GCCcore-10.2.0 37) Mako/1.1.3-GCCcore-10.2.0
19) OpenMPI/4.0.5-gcccuda-2020b 38) PyCUDA/2020.1-fosscuda-2020b
# same error as pycuda inside of astrpy module
ImportError: /sanscratch/CENTOS7/easybuild/4.4.2/software/PyCUDA/2020.1-fosscuda-2020b/lib/python3.8/site-packages/pycuda/_driver.cpython-38-x86_64-linux-gnu.so: undefined symbol: cuIpcOpenMemHandle_v2
===== Another way =====
from openhpc-users list, find the thread "How far to let Spack/EasyBuild build existing binary libraries?"
mkdir /opt/ohpc/pub/easybuild/\\
mkdir /opt/ohpc/pub/easybuild/build\\
mkdir /opt/ohpc/pub/easybuild/ebfiles_repo\\
mkdir /opt/ohpc/pub/easybuild/modules\\
mkdir /opt/ohpc/pub/easybuild/software\\
# pick an installation prefix to install EasyBuild to (change this to your liking)
EASYBUILD_PREFIX=/opt/ohpc/pub/easybuild
# download script
curl -O https://raw.githubusercontent.com/easybuilders/\\
easybuild-framework/develop/easybuild/scripts/bootstrap_eb.py
# bootstrap EasyBuild
python bootstrap_eb.py $EASYBUILD_PREFIX
# update .bash_profile to load the EasyBuild module
nano .bash_profile
module use /opt/ohpc/pub/easybuild/modules/all
ml av
ml EasyBuild
Prerequisites\\
sudo yum install rdma-core-devel (necessary to resolve OS dependancy)\\
sudo yum install java-devel\\
Install Programs
eb --show-config
eb -S bowtie
Set the APP variable then run the install..
APP=Bowtie-1.2.2-foss-2018b.eb
eb $APP --robot \\
--installpath='/opt/ohpc/pub/easybuild' \\
--buildpath='/opt/ohpc/pub/easybuild/build' \\
--sourcepath='/opt/ohpc/pub/easybuild/sources' \\
--repositorypath='/opt/ohpc/pub/easybuild/ebfiles_repo'
...
===== OHPC =====
Load module, search for application, do a dry-run, don't forget the prefix!
Then remove the dry-run flag.
module load EasyBuild/4.5.0
which eb
eb --search PyCUDA
find /opt/ohpc/pub/libs/easybuild/4.5.0/software/EasyBuild/4.5.0/easybuild/easyconfigs \
-name PyCUDA*
# dry-run
eb \
/opt/ohpc/pub/libs/easybuild/4.5.0/software/EasyBuild/4.5.0/\
easybuild/easyconfigs/p/PyCUDA/PyCUDA-2020.1-fosscuda-2020b.eb \
--dry-run --robot --prefix=/sanscratch/CENTOS8/easybuild/ohpc
[hmeij@cottontail2 ~]$ module list
Currently Loaded Modules:
1) autotools 3) gnu9/9.4.0 5) ucx/1.11.2 7) openmpi4/4.1.1
2) prun/2.2 4) hwloc/2.5.0 6) libfabric/1.13.0 8) ohpc
[hmeij@cottontail2 ~]$ module load PyCUDA/2020.1-fosscuda-2020b
The following have been reloaded with a version change:
1) hwloc/2.5.0 => hwloc/2.2.0-GCCcore-10.2.0 2) libfabric/1.13.0 => libfabric/1.11.0-GCCcore-10.2.0
[hmeij@cottontail2 ~]$ module list
Currently Loaded Modules:
1) autotools 23) libfabric/1.11.0-GCCcore-10.2.0
2) prun/2.2 24) PMIx/3.1.5-GCCcore-10.2.0
3) gnu9/9.4.0 <--- still there 25) OpenMPI/4.0.5-gcccuda-2020b
4) ucx/1.11.2 26) OpenBLAS/0.3.12-GCC-10.2.0
5) openmpi4/4.1.1 27) gompic/2020b
6) ohpc 28) FFTW/3.3.8-gompic-2020b
7) GCCcore/10.2.0 <--- active one 29) ScaLAPACK/2.1.0-gompic-2020b
8) zlib/1.2.11-GCCcore-10.2.0 30) fosscuda/2020b
9) binutils/2.35-GCCcore-10.2.0 31) bzip2/1.0.8-GCCcore-10.2.0
10) GCC/10.2.0 32) ncurses/6.2-GCCcore-10.2.0
11) CUDAcore/11.1.1 33) libreadline/8.0-GCCcore-10.2.0
12) CUDA/11.1.1-GCC-10.2.0 34) Tcl/8.6.10-GCCcore-10.2.0
13) gcccuda/2020b 35) SQLite/3.33.0-GCCcore-10.2.0
14) numactl/2.0.13-GCCcore-10.2.0 36) GMP/6.2.0-GCCcore-10.2.0
15) XZ/5.2.5-GCCcore-10.2.0 37) libffi/3.3-GCCcore-10.2.0
16) libxml2/2.9.10-GCCcore-10.2.0 38) Python/3.8.6-GCCcore-10.2.0
17) libpciaccess/0.16-GCCcore-10.2.0 39) pybind11/2.6.0-GCCcore-10.2.0
18) hwloc/2.2.0-GCCcore-10.2.0 40) SciPy-bundle/2020.11-fosscuda-2020b
19) libevent/2.1.12-GCCcore-10.2.0 41) Boost/1.74.0-GCC-10.2.0
20) Check/0.15.2-GCCcore-10.2.0 42) Boost.Python/1.74.0-GCC-10.2.0
21) GDRCopy/2.1-GCCcore-10.2.0-CUDA-11.1.1 43) Mako/1.1.3-GCCcore-10.2.0
22) UCX/1.9.0-GCCcore-10.2.0-CUDA-11.1.1 44) PyCUDA/2020.1-fosscuda-2020b
[hmeij@cottontail2 ~]$ which gcc mpicc nvcc python
/sanscratch/CENTOS8/easybuild/ohpc/software/GCCcore/10.2.0/bin/gcc # default now
/sanscratch/CENTOS8/easybuild/ohpc/software/OpenMPI/4.0.5-gcccuda-2020b/bin/mpicc
/sanscratch/CENTOS8/easybuild/ohpc/software/CUDAcore/11.1.1/bin/nvcc
/sanscratch/CENTOS8/easybuild/ohpc/software/Python/3.8.6-GCCcore-10.2.0/bin/python
# look at the OHPC software page for a smaple pycuda gpu "hello" program
\\
**[[cluster:0|Back]]**