Differences
This shows you the differences between two versions of the page.
Both sides previous revision Previous revision Next revision | Previous revision | ||
pandoc:introduction-to-vsc:05_submitting_batch_jobs:slurm [2018/01/31 13:17] – Pandoc Auto-commit pandoc | pandoc:introduction-to-vsc:05_submitting_batch_jobs:slurm [2020/10/20 09:13] (current) – Pandoc Auto-commit pandoc | ||
---|---|---|---|
Line 26: | Line 26: | ||
< | < | ||
- | sbatch job.sh | + | $ sbatch job.sh |
- | </ | + | |
- | < | + | |
Submitted batch job 5250981 | Submitted batch job 5250981 | ||
</ | </ | ||
Line 39: | Line 37: | ||
< | < | ||
JOBID PARTITION | JOBID PARTITION | ||
- | 5250981 | + | 5250981 |
</ | </ | ||
Output files: | Output files: | ||
Line 91: | Line 89: | ||
- | {{: | + | {{.: |
==== SLURM: Accounts and Users ==== | ==== SLURM: Accounts and Users ==== | ||
- | {{: | + | {{.: |
==== SLURM: Partition and Quality of Service ==== | ==== SLURM: Partition and Quality of Service ==== | ||
- | {{: | + | {{.: |
==== VSC-3 Hardware Types ==== | ==== VSC-3 Hardware Types ==== | ||
- | ^partition^ | + | ^partition |
- | |mem_0064 | | + | |mem_0064* |
- | |mem_0128 | 128 GB| | + | |mem_0128 |
- | |mem_0256 | 256 GB| | + | |mem_0256 |
+ | |vsc3plus_0064| | ||
+ | |vsc3plus_0256| | ||
+ | |binf | ||
- | | + | |
- | | + | * default partition, QDR: Intel Truescale Infinipath (40Gbit/s), FDR: Mellanox ConnectX-3 (56Gbit/ |
- | * 2 x Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz (Ivy-Bridge) | + | |
+ | effective: 10/2018 | ||
+ | |||
+ | | ||
+ | * specify partition in job script: | ||
+ | |||
+ | < | ||
+ | #SBATCH -p < | ||
+ | </ | ||
+ | ==== Standard QOS ==== | ||
+ | |||
+ | ^partition | ||
+ | |mem_0064* | ||
+ | |mem_0128 | ||
+ | |mem_0256 | ||
+ | |vsc3plus_0064|vsc3plus_0064| | ||
+ | |vsc3plus_0256|vsc3plus_0256| | ||
+ | |binf | ||
+ | |||
+ | |||
+ | | ||
+ | |||
+ | < | ||
+ | #SBATCH --qos < | ||
+ | </ | ||
+ | |||
+ | ---- | ||
+ | |||
+ | ==== VSC-4 Hardware Types ==== | ||
+ | |||
+ | ^partition^ | ||
+ | |mem_0096*| | ||
+ | |mem_0384 | 384 |2x Intel Platinum 8174 @ 3.10GHz| | ||
+ | |mem_0768 | 768 |2x Intel Platinum 8174 @ 3.10GHz| | ||
+ | |||
+ | |||
+ | * default partition, EDR: Intel Omni-Path (100Gbit/s) | ||
+ | |||
+ | effective: 10/2020 | ||
+ | |||
+ | ==== Standard QOS ==== | ||
+ | |||
+ | ^partition^QOS | ||
+ | |mem_0096*|mem_0096| | ||
+ | |mem_0384 |mem_0384| | ||
+ | |mem_0768 |mem_0768| | ||
+ | |||
+ | |||
+ | |||
+ | ---- | ||
+ | |||
+ | ==== VSC Hardware Types ==== | ||
* Display information about partitions and their nodes: | * Display information about partitions and their nodes: | ||
Line 119: | Line 171: | ||
sinfo -o %P | sinfo -o %P | ||
scontrol show partition mem_0064 | scontrol show partition mem_0064 | ||
- | scontrol show node n01-001 | + | scontrol show node n301-001 |
</ | </ | ||
- | |||
==== QOS-Account/ | ==== QOS-Account/ | ||
- | {{: | + | {{.: |
1.+2.: | 1.+2.: | ||
Line 135: | Line 186: | ||
< | < | ||
- | default_account: | + | default_account: |
- | account: | + | account: |
- | + | ||
- | default_qos: | + | |
- | qos: devel_0128 | + | |
- | goodluck | + | |
- | | + | |
- | | + | |
- | | + | |
- | | + | |
- | | + | |
- | | + | |
+ | default_qos: | ||
+ | qos: devel_0128 | ||
+ | goodluck | ||
+ | gpu_gtx1080amd | ||
+ | gpu_gtx1080multi | ||
+ | | ||
+ | gpu_k20m | ||
+ | | ||
+ | | ||
+ | | ||
+ | | ||
+ | | ||
+ | | ||
+ | | ||
+ | | ||
</ | </ | ||
Line 161: | Line 216: | ||
</ | </ | ||
< | < | ||
- | qos_name total free | + | |
- | ========================================================== | + | ========================================================================= |
- | normal_0064 | + | |
- | normal_0256 | + | |
- | normal_0128 | + | |
- | | + | devel_0128 |
- | gpu_compute | + | |
- | | + | knl 4 1 |
- | goodluck | + | |
- | | + | |
+ | | ||
+ | gpu_k20m | ||
+ | | ||
+ | | ||
+ | vsc3plus_0256 | ||
+ | gpu_gtx1080amd | ||
</ | </ | ||
naming convention: | naming convention: | ||
Line 176: | Line 237: | ||
^QOS | ^QOS | ||
|*_0064|mem_0064 | | |*_0064|mem_0064 | | ||
+ | |||
Line 232: | Line 294: | ||
* must be a shell script (first line!) | * must be a shell script (first line!) | ||
* ‘# | * ‘# | ||
- | * environment variables are set by SLURM for use within the script (e.g. '' | + | * environment variables are set by SLURM for use within the script (e.g. '' |
Line 267: | Line 329: | ||
==== Bad job practices ==== | ==== Bad job practices ==== | ||
- | * looped | + | * job submissions in a loop (takes a long time): |
< | < | ||
Line 276: | Line 338: | ||
</ | </ | ||
- | * loop in job (sequential mpirun commands): | + | * loop inside |
< | < | ||
Line 286: | Line 348: | ||
- | ==== Array job ==== | + | ==== Array jobs ==== |
- | * run similar, | + | * submit/run a series of **independent** jobs via a single SLURM script |
- | * each task will be treated as a seperate job | + | * each job in the array gets a unique identifier (SLURM_ARRAY_TASK_ID) based on which various workloads can be organized |
- | * example ([[examples/ | + | * example ([[examples/ |
< | < | ||
Line 296: | Line 358: | ||
#SBATCH -J array | #SBATCH -J array | ||
#SBATCH -N 1 | #SBATCH -N 1 | ||
- | #SBATCH --array=1-30:7 | + | #SBATCH --array=1-10 |
- | ./sleep.sh $SLURM_ARRAY_TASK_ID | + | echo "Hi, this is array job number" |
+ | sleep $SLURM_ARRAY_TASK_ID | ||
</ | </ | ||
- | * computed tasks: 1, 8, 15, 22, 29 | + | * independent jobs: 1, 2, 3 … 10 |
< | < | ||
- | 5605039_[15-29] mem_0064 | + | VSC-4 > squeue |
- | 5605039_1 | + | JOBID PARTITION |
- | 5605039_8 | + | |
+ | 406846_4 | ||
+ | | ||
+ | 406846_6 | ||
</ | </ | ||
- | |||
- | |||
- | |||
- | useful variables within job: | ||
< | < | ||
- | SLURM_ARRAY_JOB_ID | + | VSC-4 > ls slurm-* |
- | SLURM_ARRAY_TASK_ID | + | slurm-406846_10.out |
- | SLURM_ARRAY_TASK_STEP | + | slurm-406846_1.out |
- | SLURM_ARRAY_TASK_MAX | + | slurm-406846_2.out |
- | SLURM_ARRAY_TASK_MIN | + | |
</ | </ | ||
- | |||
- | limit number of simultanously running jobs to 2: | ||
< | < | ||
- | # | + | VSC-4 > cat slurm-406846_8.out |
+ | Hi, this is array job number | ||
</ | </ | ||
- | ==== Single core ==== | ||
- | * use a complete compute node for several tasks at once | + | * fine-tuning via builtin variables (SLURM_ARRAY_TASK_MIN, |
- | * example: [[examples/ | + | * example |
< | < | ||
- | ... | + | #SBATCH --array=1-20: |
+ | </ | ||
- | max_num_tasks=16 | + | * example of limiting number of simultaneously running jobs to 2 (perhaps for licences) |
- | ... | + | < |
+ | #SBATCH --array=1-20: | ||
+ | </ | ||
+ | |||
+ | |||
+ | ==== Single core jobs ==== | ||
+ | |||
+ | * use an entire compute node for several independent jobs | ||
+ | * example: [[examples/ | ||
- | for i in `seq $task_start $task_increment $task_end` | + | < |
+ | for ((i=1; i<=48; i++)) | ||
do | do | ||
- | ./ | + | |
- | check_running_tasks #sleeps as long as max_num_tasks are running | + | |
done | done | ||
wait | wait | ||
</ | </ | ||
+ | * ‘& | ||
+ | * ‘wait’: waits for all processes in the background, otherwise script would terminate | ||
- | * ‘& | ||
- | * ‘wait’: waits for all processes in the background, otherwise script will finish | ||
+ | ==== Combination of array & single core job ==== | ||
- | + | * example: | |
- | ==== Array job + single core ==== | + | |
- | + | ||
- | [[examples/job_array_some_tasks.sh|job_array_some_tasks.sh]]: | + | |
< | < | ||
... | ... | ||
- | #SBATCH --array=1-100:32 | + | #SBATCH --array=1-144:48 |
- | ... | + | j=$SLURM_ARRAY_TASK_ID |
+ | ((j+=47)) | ||
- | task_start=$SLURM_ARRAY_TASK_ID | + | for ((i=$SLURM_ARRAY_TASK_ID; i<=$j; i++)) |
- | task_end=$(( $SLURM_ARRAY_TASK_ID | + | |
- | if [ $task_end -gt $SLURM_ARRAY_TASK_MAX ]; then | + | |
- | task_end=$SLURM_ARRAY_TASK_MAX | + | |
- | fi | + | |
- | task_increment=1 | + | |
- | + | ||
- | ... | + | |
- | + | ||
- | for i in `seq $task_start $task_increment $task_end` | + | |
do | do | ||
- | ./ | + | |
- | check_running_tasks | + | |
done | done | ||
wait | wait | ||
+ | |||
</ | </ | ||
==== Exercises ==== | ==== Exercises ==== | ||
* files are located in folder '' | * files are located in folder '' | ||
- | * download or copy [[examples/sleep.sh|sleep.sh]] and find out what it is doing | + | * look into [[examples/job_array.sh|job_array.sh]] and modify |
- | * run [[examples/job_array.sh|job_array.sh]] with tasks 4-20 and stepwidth 3 | + | * look into [[examples/single_node_multiple_jobs.sh|single_node_multiple_jobs.sh]] and also change it to go in steps of 5 |
- | * start a jobs for [[examples/job_singlenode_manytasks.sh|job_singlenode_manytasks.sh]] with max_num_tasks=16 | + | * run [[examples/combined_array_multiple_jobs.sh|combined_array_multiple_jobs.sh]] and check whether |
- | * run [[examples/ | + | |
==== Job/process setup ==== | ==== Job/process setup ==== | ||
Line 391: | Line 448: | ||
* normal jobs: | * normal jobs: | ||
- | ^# | + | ^# |
- | |-N | + | |-N |
- | |--ntasks-per-core | + | |--ntasks-per-core|SLURM_NTASKS_PER_CORE| |
- | |--ntasks-per-node | + | |--ntasks-per-node|SLURM_NTASKS_PER_NODE| |
- | |--ntasks-per-socket|SLURM_NTASKS_PER_SOCKET| | + | |--ntasks, -n |
- | |--ntasks, -n | + | |
* emails: | * emails: | ||
Line 408: | Line 464: | ||
< | < | ||
- | #SBATCH -C --constraint | ||
- | #SBATCH --gres= | ||
- | |||
#SBATCH -t, --time=< | #SBATCH -t, --time=< | ||
#SBATCH --time-min=< | #SBATCH --time-min=< | ||
</ | </ | ||
- | Valid time formats: | + | time format: |
- | * MM | ||
- | * [HH:]MM:SS | ||
* DD-HH[: | * DD-HH[: | ||
- | * backfilling: | + | * backfilling: |
- | | + | * get the remaining running time for your job: |
- | | + | |
+ | < | ||
+ | squeue -h -j $SLURM_JOBID -o %L | ||
+ | </ | ||
==== Licenses ==== | ==== Licenses ==== | ||
- | {{: | + | {{.: |
< | < | ||
- | slic | + | VSC-3 > |
</ | </ | ||
- | Within the job script add the flags as shown with ‘slic’, e.g. for using both Matlab and Mathematica: | + | Within the SLURN submit |
< | < | ||
#SBATCH -L matlab@vsc, | #SBATCH -L matlab@vsc, | ||
</ | </ | ||
- | Intel licenses are needed only for compiling code, not for running | + | Intel licenses are needed only when compiling code, not for running |
- | ==== Reservations | + | ==== Reservation |
- | * core-h accounting is done for the full reservation | + | * core-h accounting is done for the entire period of reservation |
- | * contact | + | * contact |
* reservations are named after the project id | * reservations are named after the project id | ||
Line 453: | Line 506: | ||
< | < | ||
- | scontrol show reservations | + | VSC-3 > |
</ | </ | ||
- | * use it: | + | * usage: |
< | < | ||
Line 471: | Line 524: | ||
echo " | echo " | ||
</ | </ | ||
- | ==== MPI + NTASKS_PER_NODE | + | ==== MPI + pinning ==== |
* understand what your code is doing and place the processes correctly | * understand what your code is doing and place the processes correctly | ||
Line 477: | Line 530: | ||
* details for pinning: https:// | * details for pinning: https:// | ||
- | Example: Two nodes with two mpi processes each: | + | Example: Two nodes with two MPI processes each: |
=== srun === | === srun === | ||
Line 485: | Line 538: | ||
#SBATCH --tasks-per-node=2 | #SBATCH --tasks-per-node=2 | ||
- | srun --cpu_bind=map_cpu: | + | srun --cpu_bind=map_cpu: |
</ | </ | ||
Line 495: | Line 548: | ||
#SBATCH --tasks-per-node=2 | #SBATCH --tasks-per-node=2 | ||
- | export I_MPI_PIN_PROCESSOR_LIST=0, | + | export I_MPI_PIN_PROCESSOR_LIST=0, |
mpirun ./ | mpirun ./ | ||
</ | </ | ||
Line 517: | Line 570: | ||
---- | ---- | ||
+ | |||