It is important to define as many resources your job needs to run. This allows Slurm to run multiple jobs per node if resources are available (given our configuration).
Monitoring the allocated resources to your job will provide feedback if your #SBATCH specifications actually do what you think they should do.
You can control the output of squeue for a more refined look than the default layout. Here are some examples.
# compact1 export SQUEUE_FORMAT2=jobid:8,partition:10,name:.12,username:.14,statecompact:.4,timeused:.14,numnodes:.6,numcpus:.6,minmemory:.14,reasonList:.17 # ouput JOBID PARTITION NAME USER ST TIME NODES CPUS MIN_MEMORY NODELIST(REASON) 1006369 exx96 5JUP_N2_GGG_ ehwane PD 0:00 1 1 0 (Priority) 1006368 exx96 5JUP_N2_GGG_ ehwane PD 0:00 1 1 0 (Priority) 1006367 exx96 5JUP_N2_GGG_ ehwane PD 0:00 1 1 0 (Priority) 1006366 exx96 5JUP_N2_GGG_ ehwane PD 0:00 1 1 0 (Resources) 1006362 exx96 cp2k-796-d2 earbauge R 1:30:23 1 9 30000M n80 1006348 exx96 5JUP_N2_GGG_ ehwane R 5:43:10 1 1 0 n87 1006331 exx96 5JUP_N2_GGG_ ehwane R 1-02:31:28 1 1 0 n86 1006330 exx96 5JUP_N2_GGG_ ehwane R 1-02:31:33 1 1 0 n85 1006329 exx96 5JUP_N2_GGG_ ehwane R 1-02:31:47 1 1 0 n84 1006328 exx96 5JUP_N2_GGG_ ehwane R 1-02:31:54 1 1 0 n83 1006323 exx96 5JUP_N2_GGG_ ehwane R 1-03:04:49 1 1 0 n82 1006321 exx96 cp2k-796-d2 earbauge R 1-03:49:45 1 9 30000M n81 1006228 exx96 cp2k-796-d2 earbauge R 2-00:03:55 1 9 30000M n90 1006227 exx96 cp2k-796-d2 earbauge R 2-00:04:55 1 9 30000M n79 1006128 exx96 blyp-otf-str earbauge R 3-04:23:15 1 9 30000M n79 1005937 exx96 blyp-otf-str earbauge R 4-20:19:46 1 9 30000M n81 1005936 exx96 blyp-otf-str earbauge R 6-03:01:51 1 9 30000M n80 1005320 exx96 blyp-otf-str earbauge R 12-19:51:32 1 9 30000M n88 1005189 exx96 Hmass_MutS ejayarae R 13-04:23:40 1 1 0 n89 1004988 exx96 7_aMD_MutS ejayarae R 14-22:33:10 1 24 0 n89 1004757 exx96 set1_9_3_ext ejayarae R 19-21:03:57 1 24 0 n89 1004755 exx96 7_duplicate2 ejayarae R 19-21:11:16 1 24 0 n89 1005933 mw128 trianthracen emiseljie R 6-03:50:08 1 8 0 n65 1005930 mw128 trianthracen emiseljie R 6-03:50:17 1 8 0 n62 1006376 test trail4_GDP1_ ejayarae R 0:03 1 1 0 n100 1006375 test trail4_GDP1_ ejayarae R 0:09 1 1 0 n100 1006374 test trail4_GDP1_ ejayarae R 0:16 1 1 0 n100 1006373 test trail4_GDP1_ ejayarae R 20:27 1 1 0 n100e # compact2 export SQUEUE_FORMAT2=jobid:8,numnodes:.6,numcpus:.6,sockets:.18,cores:.18,threads:.18,minmemory:.14,gres:.30,reasonList:.17 # output JOBID NODES CPUS SOCKETS_PER_NODE CORES_PER_SOCKET THREADS_PER_CORE MIN_MEMORY TRES_PER_NODE NODELIST(REASON) 1006369 1 1 1 1 1 0 N/A (Priority) 1006368 1 1 1 1 1 0 N/A (Priority) 1006367 1 1 1 1 1 0 N/A (Priority) 1006366 1 1 1 1 1 0 N/A (Resources) 1006362 1 9 * * * 30000M N/A n80 1006348 1 1 1 1 1 0 N/A n87 1006331 1 1 1 1 1 0 N/A n86 1006330 1 1 1 1 1 0 N/A n85 1006329 1 1 1 1 1 0 N/A n84 1006328 1 1 1 1 1 0 N/A n83 1006323 1 1 1 1 1 0 N/A n82 1006321 1 9 * * * 30000M N/A n81 1006228 1 9 * * * 30000M N/A n90 1006227 1 9 * * * 30000M N/A n79 1006128 1 9 * * * 30000M N/A n79 1005937 1 9 * * * 30000M N/A n81 1005936 1 9 * * * 30000M N/A n80 1005320 1 9 * * * 30000M N/A n88 1005189 1 1 1 1 1 0 gres:gpu:geforce_rtx_2080_s:1 n89 1004988 1 24 1 1 1 0 gres:gpu:geforce_rtx_2080_s:1 n89 1004757 1 24 1 1 1 0 gres:gpu:geforce_rtx_2080_s:1 n89 1004755 1 24 1 1 1 0 gres:gpu:geforce_rtx_2080_s:1 n89 1005933 1 8 * * * 0 N/A n65 1005930 1 8 * * * 0 N/A n62 1006380 1 1 1 1 1 0 gres:gpu:quadro_rtx_5000:1 n101 1006379 1 1 1 1 1 0 gres:gpu:quadro_rtx_5000:1 n101 1006378 1 1 1 1 1 0 gres:gpu:quadro_rtx_5000:1 n101 1006377 1 1 1 1 1 0 gres:gpu:quadro_rtx_5000:1 n101 1006376 1 1 1 1 1 0 gres:gpu:quadro_rtx_5000:1 n100 1006375 1 1 1 1 1 0 gres:gpu:quadro_rtx_5000:1 n100 1006374 1 1 1 1 1 0 gres:gpu:quadro_rtx_5000:1 n100 1006373 1 1 1 1 1 0 gres:gpu:quadro_rtx_5000:1 n100 # 1+2 #export SQUEUE_FORMAT2=jobid:8,partition:10,name:.12,username:.14,statecompact:.4,timeused:.14,numnodes:.6,numcpus:.6,sockets:.18,cores:.18,threads:.18,minmemory:.14,gres:.30,reasonList:.17 # output JOBID PARTITION NAME USER ST TIME NODES CPUS SOCKETS_PER_NODE CORES_PER_SOCKET THREADS_PER_CORE MIN_MEMORY TRES_PER_NODE NODELIST(REASON) 1006369 exx96 5JUP_N2_GGG_ ehwane PD 0:00 1 1 1 1 1 0 N/A (Priority) 1006368 exx96 5JUP_N2_GGG_ ehwane PD 0:00 1 1 1 1 1 0 N/A (Priority) 1006367 exx96 5JUP_N2_GGG_ ehwane PD 0:00 1 1 1 1 1 0 N/A (Priority) 1006366 exx96 5JUP_N2_GGG_ ehwane PD 0:00 1 1 1 1 1 0 N/A (Resources) 1006362 exx96 cp2k-796-d2 earbauge R 1:26:20 1 9 * * * 30000M N/A n80 1006348 exx96 5JUP_N2_GGG_ ehwane R 5:39:07 1 1 1 1 1 0 N/A n87 1006331 exx96 5JUP_N2_GGG_ ehwane R 1-02:27:25 1 1 1 1 1 0 N/A n86 1006330 exx96 5JUP_N2_GGG_ ehwane R 1-02:27:30 1 1 1 1 1 0 N/A n85 1006329 exx96 5JUP_N2_GGG_ ehwane R 1-02:27:44 1 1 1 1 1 0 N/A n84 1006328 exx96 5JUP_N2_GGG_ ehwane R 1-02:27:51 1 1 1 1 1 0 N/A n83 1006323 exx96 5JUP_N2_GGG_ ehwane R 1-03:00:46 1 1 1 1 1 0 N/A n82 1006321 exx96 cp2k-796-d2 earbauge R 1-03:45:42 1 9 * * * 30000M N/A n81 1006228 exx96 cp2k-796-d2 earbauge R 1-23:59:52 1 9 * * * 30000M N/A n90 1006227 exx96 cp2k-796-d2 earbauge R 2-00:00:52 1 9 * * * 30000M N/A n79 1006128 exx96 blyp-otf-str earbauge R 3-04:19:12 1 9 * * * 30000M N/A n79 1005937 exx96 blyp-otf-str earbauge R 4-20:15:43 1 9 * * * 30000M N/A n81 1005936 exx96 blyp-otf-str earbauge R 6-02:57:48 1 9 * * * 30000M N/A n80 1005320 exx96 blyp-otf-str earbauge R 12-19:47:29 1 9 * * * 30000M N/A n88 1005189 exx96 Hmass_MutS ejayarae R 13-04:19:37 1 1 1 1 1 0 gres:gpu:geforce_rtx_2080_s:1 n89 1004988 exx96 7_aMD_MutS ejayarae R 14-22:29:07 1 24 1 1 1 0 gres:gpu:geforce_rtx_2080_s:1 n89 1004757 exx96 set1_9_3_ext ejayarae R 19-20:59:54 1 24 1 1 1 0 gres:gpu:geforce_rtx_2080_s:1 n89 1004755 exx96 7_duplicate2 ejayarae R 19-21:07:13 1 24 1 1 1 0 gres:gpu:geforce_rtx_2080_s:1 n89 1005933 mw128 trianthracen emiseljie R 6-03:46:05 1 8 * * * 0 N/A n65 1005930 mw128 trianthracen emiseljie R 6-03:46:14 1 8 * * * 0 N/A n62 1006373 test trail4_GDP1_ ejayarae R 16:24 1 1 1 1 1 0 gres:gpu:quadro_rtx_5000:1 n100