DokuWiki

User Tools

Site Tools

Trace: • 98 • 77 • 209 • 216 • 200
cluster:209

Differences

This shows you the differences between two versions of the page.

Link to this comparison view

Both sides previous revisionPrevious revision
Next revision		Previous revision
cluster:209 [2021/11/02 13:32] – hmeij07cluster:209 [2022/04/10 19:49] (current) – [OHPC] hmeij07
Line 10: Line 10:
   * https://docs.easybuild.io/en/latest/version-specific/Supported_software.html   * https://docs.easybuild.io/en/latest/version-specific/Supported_software.html
   * https://docs.easybuild.io/en/latest/Installation.html#requirements   * https://docs.easybuild.io/en/latest/Installation.html#requirements
 +  * https://docs.easybuild.io/en/latest/version-specific/help.html
 +
 +**NOTE** this eats up lots of disk space with multiple compiler and tool chains as you will see, hence installed in /sanscratch/CENTOS7
  
 <code> <code>
Line 109: Line 112:
 # yumdownloader --desdir=somewhere  package-name # yumdownloader --desdir=somewhere  package-name
 # centos 7 nodes only # centos 7 nodes only
-cd /sanscratch/tmp/rpms/+cd /sanscratch/tmp/rpms/ \ 
 +rpm -ivh environment-modules-3.2.10-10.el7.x86_64.rpm \
 rpm -ivh lua-filesystem-1.6.2-2.el7.x86_64.rpm \ rpm -ivh lua-filesystem-1.6.2-2.el7.x86_64.rpm \
 lua-json-1.3.2-2.el7.noarch.rpm \ lua-json-1.3.2-2.el7.noarch.rpm \
Line 116: Line 120:
 lua-bitop-1.0.2-3.el7.x86_64.rpm \ lua-bitop-1.0.2-3.el7.x86_64.rpm \
 lua-lpeg-0.12-1.el7.x86_64.rpm  \ lua-lpeg-0.12-1.el7.x86_64.rpm  \
-Lmod-8.2.7-1.el7.x86_64.rpm+Lmod-8.2.7-1.el7.x86_64.rpm \
 # log out and log back in  # log out and log back in 
 # end sidebar # end sidebar
Line 125: Line 129:
  
 # example easybuild installation of software # example easybuild installation of software
-eb Bowtie2-2.4.2-GCC-9.3.0.eb --dry-run --robot --prefix=/share/apps/CENTOS7/easybuild/4.4.2+eb Bowtie2-2.4.2-GCC-9.3.0.eb --dry-run --robot --prefix=/sanscratch/CENTOS7/easybuild/4.4.2
 # do not forget prefix else it goes to $HOME/.local/easybuild # do not forget prefix else it goes to $HOME/.local/easybuild
  
 # once done # once done
-module use /share/apps/CENTOS7/easybuild/4.4.2/modules/all+module use /sanscratch/CENTOS7/easybuild/4.4.2/modules/all
  
 # use 'module avail' to discover modules. then load # use 'module avail' to discover modules. then load
Line 145: Line 149:
  
 [hmeij@greentail52 ~]$ which bowtie2 [hmeij@greentail52 ~]$ which bowtie2
-/share/apps/CENTOS7/easybuild/4.4.2/software/Bowtie2/2.4.2-GCC-9.3.0/bin/bowtie2+/sanscratch/CENTOS7/easybuild/4.4.2/software/Bowtie2/2.4.2-GCC-9.3.0/bin/bowtie2
  
 </code> </code>
Line 155: Line 159:
 # this required libibverbs and libibverbs-devel so not sure it will run on compute nodes # this required libibverbs and libibverbs-devel so not sure it will run on compute nodes
  
-eb astropy-4.2.1-intelcuda-2020b.eb  --robot --prefix=/share/apps/CENTOS7/easybuild/4.4.2+eb astropy-4.2.1-intelcuda-2020b.eb  --robot --prefix=/sanscratch/CENTOS7/easybuild/4.4.2
  
 # then ran into icc license problems, 2020b, check out license failed # then ran into icc license problems, 2020b, check out license failed
Line 161: Line 165:
 # hint add eula flag when trying i/intel toolchain # hint add eula flag when trying i/intel toolchain
 eb intel-2021a.eb   --robot  \ eb intel-2021a.eb   --robot  \
---prefix=/share/apps/CENTOS7/easybuild/4.4.2 \+--prefix=/sanscratch/CENTOS7/easybuild/4.4.2 \
 --accept-eula-for=Intel-oneAPI,Intel-C,Intel-C++,Intel-icc,Intel-Fortran,Intel-ifort,Intel-MPI,Intel-MKL,Intel-CUDA --accept-eula-for=Intel-oneAPI,Intel-C,Intel-C++,Intel-icc,Intel-Fortran,Intel-ifort,Intel-MPI,Intel-MKL,Intel-CUDA
 # built intel-compilers/2021.2.0, hangs on Perl-5.32.1-GCCcore-10.3.0-minimal.eb build? try separate # built intel-compilers/2021.2.0, hangs on Perl-5.32.1-GCCcore-10.3.0-minimal.eb build? try separate
Line 184: Line 188:
 description = """GCC based compiler toolchain __with CUDA support__, and including description = """GCC based compiler toolchain __with CUDA support__, and including
  
-# to find config change to that location +# to find config, change to that location - also run from ~hmeij login shell 
-cd easybuild/software/EasyBuild/4.4.2/easybuild/easyconfigs/a/astropy/ +# aand you must set python 3.8.3 as indicated above, else run.py from 
-astropy]$ eb astropy-4.2.1-fosscuda-2020b.eb  --dry-run --robot --prefix=/sanscratch/CENTOS7/easybuild/4.4.2 +# native OS python is found yielding 'make -j 16' error 
 + 
 +cd easybuild/software/EasyBuild/4.4.2/easybuild/easyconfigs/a/astropy/ \ 
 +eb astropy-4.2.1-fosscuda-2020b.eb  \ 
 +--dry-run --robot --prefix=/sanscratch/CENTOS7/easybuild/4.4.2 
  
 # fosscuda attempt worked, see below for continuation # fosscuda attempt worked, see below for continuation
Line 219: Line 227:
  12) hwloc/2.2.0-GCCcore-10.2.0        24) fosscuda/2020b  12) hwloc/2.2.0-GCCcore-10.2.0        24) fosscuda/2020b
  
-# in our environment+# in our environment we must location of libs, this seems odd 
 +export LD_LIBRARY_PATH=/sanscratch/CENTOS7/easybuild/4.4.2/software/CUDAcore/11.1.1/stubs/lib64:$LD_LIBRARY_PATH
  
 [hmeij@greentail52 ~]$ which python nvcc mpirun [hmeij@greentail52 ~]$ which python nvcc mpirun
Line 243: Line 252:
 [hmeij@n84 ~]$ module list [hmeij@n84 ~]$ module list
 ... ...
 +
 +</code>
 +
 +Helios GPU tutorial\\
 +https://heliosexo.readthedocs.io/en/latest/sections/tutorial.html?highlight=GPU#tutorial
 +<code>
 +# pycuda not working, why would fosscuda?
 +ImportError: /sanscratch/CENTOS7/easybuild/4.4.2/software/Python/3.8.6-GCCcore-10.2.0/lib/python3.8/site-packages/pycuda/_driver.cpython-38-x86_64-linux-gnu.so: 
 +undefined symbol: cuDevicePrimaryCtxRelease_v2
 +</code>
 +https://stackoverflow.com/questions/66035567/missing-symbol-cudeviceprimaryctxrelease-vs-cudeviceprimaryctxrelease-v2
 +
 +===== Emcee =====
 +
 +<code>
 +
 +module list
 +
 +Currently Loaded Modules:
 +  1) GCCcore/8.2.0                    14) ScaLAPACK/2.0.2-gompi-2019a-OpenBLAS-0.3.5
 +  2) zlib/1.2.11-GCCcore-8.2.0        15) foss/2019a
 +  3) binutils/2.31.1-GCCcore-8.2.0    16) bzip2/1.0.6-GCCcore-8.2.0
 +  4) GCC/8.2.0-2.31.1                 17) ncurses/6.1-GCCcore-8.2.0
 +  5) numactl/2.0.12-GCCcore-8.2.0     18) libreadline/8.0-GCCcore-8.2.0
 +  6) XZ/5.2.4-GCCcore-8.2.0           19) Tcl/8.6.9-GCCcore-8.2.0
 +  7) libxml2/2.9.8-GCCcore-8.2.0      20) SQLite/3.27.2-GCCcore-8.2.0
 +  8) libpciaccess/0.14-GCCcore-8.2.0  21) GMP/6.1.2-GCCcore-8.2.0
 +  9) hwloc/1.11.11-GCCcore-8.2.0      22) libffi/3.2.1-GCCcore-8.2.0
 + 10) OpenMPI/3.1.3-GCC-8.2.0-2.31.1   23) Python/3.7.2-GCCcore-8.2.0          <---
 + 11) OpenBLAS/0.3.5-GCC-8.2.0-2.31.1  24) SciPy-bundle/2019.03-foss-2019a
 + 12) gompi/2019a                      25) emcee/2.2.1-foss-2019a
 + 13) FFTW/3.3.8-gompi-2019a
 +
 +</code>
 +
 +I noticed emcee loaded two python versions, the default being 3.7.2 (requires centos7). But you can unload this python module and load module Python/2.7.15-GCCcore-8.2.0 and emcee will import. That means with 2.7.15 emcee could run on centos6 nodes.
 +
 +Flexible but weird.
 +
 +=====PyCUDA =====
 +
 +<code>
 +
 +[hmeij@greentail52 ~]$ module load PyCUDA/2020.1-fosscuda-2020b
 +[hmeij@greentail52 ~]$ module list
 +
 +Currently Loaded Modules:
 +  1) GCCcore/10.2.0                          20) OpenBLAS/0.3.12-GCC-10.2.0
 +  2) zlib/1.2.11-GCCcore-10.2.0              21) gompic/2020b
 +  3) binutils/2.35-GCCcore-10.2.0            22) FFTW/3.3.8-gompic-2020b
 +  4) GCC/10.2.0                              23) ScaLAPACK/2.1.0-gompic-2020b
 +  5) CUDAcore/11.1.1                         24) fosscuda/2020b
 +  6) CUDA/11.1.1-GCC-10.2.0                  25) bzip2/1.0.8-GCCcore-10.2.0
 +  7) gcccuda/2020b                           26) ncurses/6.2-GCCcore-10.2.0
 +  8) numactl/2.0.13-GCCcore-10.2.0           27) libreadline/8.0-GCCcore-10.2.0
 +  9) XZ/5.2.5-GCCcore-10.2.0                 28) Tcl/8.6.10-GCCcore-10.2.0
 + 10) libxml2/2.9.10-GCCcore-10.2.0           29) SQLite/3.33.0-GCCcore-10.2.0
 + 11) libpciaccess/0.16-GCCcore-10.2.0        30) GMP/6.2.0-GCCcore-10.2.0
 + 12) hwloc/2.2.0-GCCcore-10.2.0              31) libffi/3.3-GCCcore-10.2.0
 + 13) libevent/2.1.12-GCCcore-10.2.0          32) Python/3.8.6-GCCcore-10.2.0
 + 14) Check/0.15.2-GCCcore-10.2.0             33) pybind11/2.6.0-GCCcore-10.2.0
 + 15) GDRCopy/2.1-GCCcore-10.2.0-CUDA-11.1.1  34) SciPy-bundle/2020.11-fosscuda-2020b
 + 16) UCX/1.9.0-GCCcore-10.2.0-CUDA-11.1.1    35) Boost/1.74.0-GCC-10.2.0
 + 17) libfabric/1.11.0-GCCcore-10.2.0         36) Boost.Python/1.74.0-GCC-10.2.0
 + 18) PMIx/3.1.5-GCCcore-10.2.0               37) Mako/1.1.3-GCCcore-10.2.0
 + 19) OpenMPI/4.0.5-gcccuda-2020b             38) PyCUDA/2020.1-fosscuda-2020b
 +
 +# same error as pycuda inside of astrpy module
 +ImportError: /sanscratch/CENTOS7/easybuild/4.4.2/software/PyCUDA/2020.1-fosscuda-2020b/lib/python3.8/site-packages/pycuda/_driver.cpython-38-x86_64-linux-gnu.so: undefined symbol: cuIpcOpenMemHandle_v2
  
 </code> </code>
Line 301: Line 379:
  
  ...  ...
 +
 +===== OHPC =====
 +
 +Load module, search for application, do a dry-run, don't forget the prefix!
 +
 +Then remove the dry-run flag.
 +
 +<code>
 +
 + module load EasyBuild/4.5.0
 + which eb
 +
 + eb --search PyCUDA
 + find /opt/ohpc/pub/libs/easybuild/4.5.0/software/EasyBuild/4.5.0/easybuild/easyconfigs \
 + -name PyCUDA*
 +
 +# dry-run
 + eb \
 +/opt/ohpc/pub/libs/easybuild/4.5.0/software/EasyBuild/4.5.0/\
 +easybuild/easyconfigs/p/PyCUDA/PyCUDA-2020.1-fosscuda-2020b.eb \
 + --dry-run --robot --prefix=/sanscratch/CENTOS8/easybuild/ohpc
 +
 +[hmeij@cottontail2 ~]$ module list
 +
 +Currently Loaded Modules:
 +  1) autotools   3) gnu9/9.4.0    5) ucx/1.11.2         7) openmpi4/4.1.1
 +  2) prun/2.2    4) hwloc/2.5.0   6) libfabric/1.13.0   8) ohpc
 +
 +
 +
 +[hmeij@cottontail2 ~]$ module load PyCUDA/2020.1-fosscuda-2020b
 +
 +The following have been reloaded with a version change:
 +  1) hwloc/2.5.0 => hwloc/2.2.0-GCCcore-10.2.0     2) libfabric/1.13.0 => libfabric/1.11.0-GCCcore-10.2.0
 +
 +[hmeij@cottontail2 ~]$ module list
 +
 +Currently Loaded Modules:
 +  1) autotools                               23) libfabric/1.11.0-GCCcore-10.2.0
 +  2) prun/2.2                                24) PMIx/3.1.5-GCCcore-10.2.0
 +  3) gnu9/9.4.0          <--- still there    25) OpenMPI/4.0.5-gcccuda-2020b
 +  4) ucx/1.11.2                              26) OpenBLAS/0.3.12-GCC-10.2.0
 +  5) openmpi4/4.1.1                          27) gompic/2020b
 +  6) ohpc                                    28) FFTW/3.3.8-gompic-2020b
 +  7) GCCcore/10.2.0      <--- active one     29) ScaLAPACK/2.1.0-gompic-2020b
 +  8) zlib/1.2.11-GCCcore-10.2.0              30) fosscuda/2020b
 +  9) binutils/2.35-GCCcore-10.2.0            31) bzip2/1.0.8-GCCcore-10.2.0
 + 10) GCC/10.2.0                              32) ncurses/6.2-GCCcore-10.2.0
 + 11) CUDAcore/11.1.1                         33) libreadline/8.0-GCCcore-10.2.0
 + 12) CUDA/11.1.1-GCC-10.2.0                  34) Tcl/8.6.10-GCCcore-10.2.0
 + 13) gcccuda/2020b                           35) SQLite/3.33.0-GCCcore-10.2.0
 + 14) numactl/2.0.13-GCCcore-10.2.0           36) GMP/6.2.0-GCCcore-10.2.0
 + 15) XZ/5.2.5-GCCcore-10.2.0                 37) libffi/3.3-GCCcore-10.2.0
 + 16) libxml2/2.9.10-GCCcore-10.2.0           38) Python/3.8.6-GCCcore-10.2.0
 + 17) libpciaccess/0.16-GCCcore-10.2.0        39) pybind11/2.6.0-GCCcore-10.2.0
 + 18) hwloc/2.2.0-GCCcore-10.2.0              40) SciPy-bundle/2020.11-fosscuda-2020b
 + 19) libevent/2.1.12-GCCcore-10.2.0          41) Boost/1.74.0-GCC-10.2.0
 + 20) Check/0.15.2-GCCcore-10.2.0             42) Boost.Python/1.74.0-GCC-10.2.0
 + 21) GDRCopy/2.1-GCCcore-10.2.0-CUDA-11.1.1  43) Mako/1.1.3-GCCcore-10.2.0
 + 22) UCX/1.9.0-GCCcore-10.2.0-CUDA-11.1.1    44) PyCUDA/2020.1-fosscuda-2020b
 +
 +
 +
 +[hmeij@cottontail2 ~]$ which gcc mpicc nvcc python
 +/sanscratch/CENTOS8/easybuild/ohpc/software/GCCcore/10.2.0/bin/gcc  # default now
 +/sanscratch/CENTOS8/easybuild/ohpc/software/OpenMPI/4.0.5-gcccuda-2020b/bin/mpicc
 +/sanscratch/CENTOS8/easybuild/ohpc/software/CUDAcore/11.1.1/bin/nvcc
 +/sanscratch/CENTOS8/easybuild/ohpc/software/Python/3.8.6-GCCcore-10.2.0/bin/python
 +
 +# look at the OHPC software page for a smaple pycuda gpu "hello" program
 +
 +</code>
  
  
cluster/209.1635859921.txt.gz · Last modified: by hmeij07