Differences

This shows you the differences between two versions of the page.

--- cluster:212 [2022/03/02 13:58] – created hmeij07
+++ cluster:212 [2022/03/02 18:40] (current) – [HPC SDK] hmeij07
@@ Line 2: / Line 2: @@
 **[[cluster:0|Back]]**
-===== HPC SK =====
+===== HPC SDK =====
   * https://docs.nvidia.com/hpc-sdk/index.html
@@ Line 10: / Line 10: @@
 The NVIDIA Software Development Kit (SDK) Manager is an all-in-one tool that bundles developer software and provides an end-to-end development environment setup solution for NVIDIA SDK. Think "OneAPI" ala Intel. All Nvidia's compilers, math libraries, and communications tools in one easy to set up environment.
+I performed a "single system install" in a shared location (see above) using gcc 9.2.0. This will only be good for the ''exx96'' nodes. Bundled with the newest plus three previous CUDA versions (11.6, 11.2, 11.0, 10.2). cuda-10.2 runs the gpus in that queue. So first we set up
+<code>
+# gcc
+export PATH=/share/apps/CENTOS7/gcc/9.2.0/bin:$PATH
+export LD_LIBRARY_PATH=/share/apps/CENTOS7/gcc/9.2.0/lib64:\
+/share/apps/CENTOS7/gcc/9.2.0/libexec:\
+/share/apps/CENTOS7/gcc/9.2.0/lib:\
+$LD_LIBRARY_PATH
+# cuda
+# comes from SDK
+</code>
+Next we load the 22.2 module and view what all is available. Consult the documentation.
+<code>
+module use /share/apps/CENTOS7/nvidia/hpc_sdk/22.2/modulefiles/nvhpc/
+module load 22.2
+module list
+Currently Loaded Modules:
+) 22.2
+module show 22.2
+-------------------------------------------------------------------------------------------------------------------------
+   /share/apps/CENTOS7/nvidia/hpc_sdk/22.2/modulefiles/nvhpc/22.2:
+-------------------------------------------------------------------------------------------------------------------------
+conflict("nvhpc")
+conflict("nvhpc-nompi")
+conflict("nvhpc-byo-compiler")
+setenv("NVHPC","/share/apps/CENTOS7/nvidia/hpc_sdk/22.2")
+setenv("NVHPC_ROOT","/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2")
+setenv("CC","/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/compilers/bin/nvc")
+setenv("CXX","/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/compilers/bin/nvc++")
+setenv("FC","/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/compilers/bin/nvfortran")
+setenv("F90","/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/compilers/bin/nvfortran")
+setenv("F77","/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/compilers/bin/nvfortran")
+setenv("CPP","cpp")
+prepend_path("PATH","/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/cuda/bin")
+prepend_path("PATH","/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/compilers/bin")
+prepend_path("PATH","/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/comm_libs/mpi/bin")
+prepend_path("PATH","/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/compilers/extras/qd/bin")
+prepend_path("LD_LIBRARY_PATH","/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/cuda/lib64")
+prepend_path("LD_LIBRARY_PATH","/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/cuda/extras/CUPTI/lib64")
+prepend_path("LD_LIBRARY_PATH","/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/compilers/extras/qd/lib")
+prepend_path("LD_LIBRARY_PATH","/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/compilers/lib")
+prepend_path("LD_LIBRARY_PATH","/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/math_libs/lib64")
+prepend_path("LD_LIBRARY_PATH","/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/comm_libs/mpi/lib")
+prepend_path("LD_LIBRARY_PATH","/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/comm_libs/nccl/lib")
+prepend_path("LD_LIBRARY_PATH","/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/comm_libs/nvshmem/lib")
+prepend_path("CPATH","/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/math_libs/include")
+prepend_path("CPATH","/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/comm_libs/mpi/include")
+prepend_path("CPATH","/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/comm_libs/nccl/include")
+prepend_path("CPATH","/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/comm_libs/nvshmem/include")
+prepend_path("CPATH","/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/compilers/extras/qd/include/qd")
+prepend_path("MANPATH","/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/compilers/man")
+setenv("OPAL_PREFIX","/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/comm_libs/mpi")
+</code>
+And a whole bunch of other stuff, consult documentation.
+<code>
+[hmeij@n79 ~]$ which nvcc nvc++
+/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/compilers/bin/nvcc
+/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/compilers/bin/nvc
+[hmeij@n79 ~]$ which mpirun mpicc
+/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/comm_libs/mpi/bin/mpirun
+/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/comm_libs/mpi/bin/mpicc
+# no cublas for 10.2
+[hmeij@n79 ~]$ find /share/apps/CENTOS7/nvidia/hpc_sdk/22.2/ -type f -name libcublas.so
+/share/apps/CENTOS7/.../math_libs/11.6/targets/x86_64-linux/lib/stubs/libcublas.so
+/share/apps/CENTOS7/.../math_libs/11.2/targets/x86_64-linux/lib/stubs/libcublas.so
+/share/apps/CENTOS7/.../math_libs/11.0/targets/x86_64-linux/lib/stubs/libcublas.so
+</code>
+And with the module loaded in your script, it will work on queue ''exx96''
+<code>
+# gcc plus SDK
+export PATH=/share/apps/CENTOS7/gcc/9.2.0/bin:$PATH
+export LD_LIBRARY_PATH=/share/apps/CENTOS7/gcc/9.2.0/lib64:\
+/share/apps/CENTOS7/gcc/9.2.0/libexec:\
+/share/apps/CENTOS7/gcc/9.2.0/lib:\
+$LD_LIBRARY_PATH
+module use /share/apps/CENTOS7/nvidia/hpc_sdk/modulefiles/nvhpc/22.2
+module load 22.2
+# which and wrapper stdout
+/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/compilers/bin/nvcc
+/share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/comm_libs/mpi/bin/mpirun
+/share/apps/CENTOS7/python/3.8.3/bin/python
+GPU allocation n79:1
+executing: /share/apps/CENTOS7/nvidia/hpc_sdk/22.2/Linux_x86_64/22.2/comm_libs/mpi/bin/mpirun  \
+-x LD_LIBRARY_PATH -machinefile /zfshomes/hmeij/.lsbatch/mpi_machines.844864 \
+-np 1 pmemd.cuda -O -o mdout.844864 -inf mdinfo.1K10 -x mdcrd.1K10 -r restrt.1K10 -ref inpcrd
+</code>
 \\
 **[[cluster:0|Back]]**