This version (2022/06/20 09:01) was approved by msiegel.

Sample parallel job script for slurm (VSC-3):

#!/bin/bash
#SBATCH -J abaqus
#SBATCH -N 2
#SBATCH --ntasks 32
#SBATCH --cpus-per-task 1

scontrol show hostname $SLURM_NODELIST | paste -d -s > hostlist

cpu=`expr $SLURM_NTASKS / $SLURM_JOB_NUM_NODES`
echo $cpu

mp_host_list="("
for i in $(cat hostlist)
do
mp_host_list="${mp_host_list}('$i',$cpu),"
done
mp_host_list=`echo ${mp_host_list} | sed -e "s/,$/,)/"`
echo "mp_host_list=${mp_host_list}" >> ~/abaqus_v6.env

abq6141 env=mp_host_list job=bb_aba1 cpus=$cpus parallel=domain domains=32 mp_mode=mpi input=input_file

abaqus_v6.env:

#
#        System-Wide Abaqus Environment File
#       -------------------------------------
standard_parallel = ALL
mp_mode = MPI
scratch  = "/fhgfs"
mp_file_system = (DETECT,DETECT)
mp_num_parallel_ftps = (4, 4)
mp_environment_export = ('MPI_PROPAGATE_TSTP',
                         'ABA_CM_BUFFERING',
                         'ABA_CM_BUFFERING_LIMIT',
                         'ABA_ITERATIVE_SOLVER_VERBOSE',
                         'ABA_DMPSOLVER_BWDPARALLELOFF',
                         'ABA_ELP_SURFACE_SPLIT',
                         'ABA_ELP_SUSPEND',
                         'ABA_HOME',
                         'ABA_MEMORY_MODE',
                         'ABA_MPI_MESSAGE_TRACKING',
                         'ABA_MPI_VERBOSE_LEVEL',
                         'ABA_PATH',
                         'ABAQUS_CSE_RELTIMETOLERANCE',
                         'ABA_RESOURCE_MONITOR',
                         'ABA_RESOURCE_USEMALLINFO',
                         'ABAQUS_LANG',
                         'ABAQUS_CSE_CURRCONFIGMAPPING',
                         'ABAQUS_MPF_DIAGNOSTIC_LEVEL',
                         'ABAQUSLM_LICENSE_FILE',
                         'ABQ_CRTMALLOC',
                         'ABQ_DATACHECK',
                         'ABQ_RECOVER',
                         'ABQ_RESTART',
                         'ABQ_SPLITFILE',
                         'ABQ_XPL_WINDOWDUMP',
                         'ABQ_XPL_PARTITIONSIZE',
                         'ABQLMHANGLIMIT',
                         'ABQLMQUEUE',
                         'ABQLMUSER',
                         'CCI_RENDEZVOUS',
                         'DOMAIN',
                         'DOMAIN_CPUS',
                         'DOUBLE_PRECISION',
                         'FLEXLM_DIAGNOSTICS',
                         'FOR0006',
                         'FOR0064',
                         'FOR_IGNORE_EXCEPTIONS',
                         'FOR_DISABLE_DIAGNOSTIC_DISPLAY',
                         'LD_PRELOAD',
                         'MP_NUMBER_OF_THREADS',
                         'MPC_GANG',
                         'MPI_FLAGS',
                         'MPI_FLUSH_FCACHE',
                         'MPI_RDMA_NENVELOPE',
                         'MPI_SOCKBUFSIZE',
                         'MPI_USE_MALLOPT_MMAP_MAX',
                         'MPI_USE_MALLOPT_MMAP_THRESHOLD',
                         'MPI_USE_MALLOPT_SBRK_PROTECTION',
                         'MPI_WORKDIR',
                         'MPCCI_DEBUG',
                         'MPCCI_CODEID',
                         'MPCCI_JOBID',
                         'MPCCI_NETDEVICE',
                         'MPCCI_TINFO',
                         'MPCCI_SERVER',
                         'MPIEXEC_AFFINITY_TABLE',
                         'ABAQUS_CCI_DEBUG',
                         'NCPUS',
                         'OMP_DYNAMIC',
                         'OMP_NUM_THREADS', 
                         'OUTDIR',
                         'PAIDUP',
                         'PARALLEL_METHOD',
                         'RAIDEV_NDREG_LAZYMEM',
                         'ABA_SYMBOLIC_GENERALCOLLAPSE',
                         'ABA_SYMBOLIC_GENERAL_MAXCLIQUERANK',
                         'ABA_ADM_MINIMUMINCREASE',
                         'ABA_ADM_MINIMUMDECREASE',
                         'IPATH_NO_CPUAFFINITY',
                         'MALLOC_MMAP_THRESHOLD_',
                         'ABA_EXT_SIMOUTPUT',
                         'SMA_WS',
                         'SMA_PARENT',
                         'SMA_PLATFORM',
                         'ABA_PRE_DECOMPOSITION',
                         'ACML_FAST_MALLOC',
                         'ACML_FAST_MALLOC_CHUNK_SIZE',
                         'ACML_FAST_MALLOC_MAX_CHUNKS',
                         'ACML_FAST_MALLOC_DEBUG',
                         'MKL_NUM_THREADS',
                         'MKL_DYNAMIC')

import driverUtils, os
#-*- mode: python -*-

#############################################################################
#                                                                           #
#    Compile and Link command settings for the Linux 64 Platform            #
#                 ( AMD Opteron / Intel EM64T )                             #
#                                                                           #
#############################################################################

import os, re, glob, driverUtils

#MPI implementation handling
mpiCppImpl = ''
mp_rsh_command = 'ssh -n -l %U %H %C'
mp_mpi_implementation = IMPI
#mp_mpi_implementation = IMPI #<--- Uncomment this line and comment above line to switch to IMPI
if mp_mpi_implementation == PMPI:
    pmpipath = driverUtils.locateFile(os.environ.get('ABA_PATH', ''), 'pmpi-9.1.2/bin', 'mpirun')
    mp_mpirun_path = {PMPI: pmpipath}
    mpiCppImpl = '-DABQ_MPI_PMPI'
if mp_mpi_implementation == IMPI:
    #impipath = driverUtils.locateFile(os.environ.get('ABA_PATH', ''), 'impi-4.1.1/bin', 'mpiexec.hydra')
    impipath = driverUtils.locateFile(os.environ.get('PATH', ''), '/cm/shared/apps/intel/impi_5.0.3/intel64/bin', 'mpiexec.hydra')
    mp_mpirun_path = {IMPI: impipath}
    mpiCppImpl = '-DABQ_MPI_IMPI'
    
fortCmd = "ifort"   # <-- Fortran compiler
cppCmd  = "g++"     # <-- C++ compiler

# Avoid signal trapping by the Fortran RTE
os.environ["FOR_IGNORE_EXCEPTIONS"] = "1"
# Disable messages from the Fotran RTE
os.environ["FOR_DISABLE_STACK_TRACE"] = "1"

# Do not let QLogic InfiniPath driver set processor affinity.
os.environ["IPATH_NO_CPUAFFINITY"] = "1"

# Add the flag "-free" to the compile_fortran command below to use free-format FORTRAN 90 syntax.

compile_fortran = [fortCmd,
                   '-V',
                   '-c', '-fPIC', '-auto', '-mP2OPT_hpo_vec_divbyzero=F', '-extend_source',
                   '-fpp', '-WB', '-I%I']

# Additional command-line options for the Intel C/C++ Compilers:
# '-cxxlib', '-Kc++eh', '-Krtti', '-Kc++', '-pc64', '-restrict', '-i-dynamic',
# '-we1011', '-we120',  '-we117', '-we556', '-we144', '-we268', '-we1224', '-we167', '-we880'

compile_cpp = [cppCmd,
               '-c', '-fPIC', '-w', '-Wno-deprecated', '-DTYPENAME=typename',
               '-D_LINUX_SOURCE', '-DABQ_LINUX', '-DABQ_LNX86_64', '-DSMA_GNUC',
               '-DFOR_TRAIL', '-DHAS_BOOL', '-DASSERT_ENABLED',
               '-D_BSD_TYPES', '-D_BSD_SOURCE', '-D_GNU_SOURCE',
               '-D_POSIX_SOURCE', '-D_XOPEN_SOURCE_EXTENDED', '-D_XOPEN_SOURCE',
               '-DHAVE_OPENGL', '-DHKS_OPEN_GL',  '-DGL_GLEXT_PROTOTYPES',
               '-DMULTI_THREADING_ENABLED', '-D_REENTRANT',
               '-DABQ_MPI_SUPPORT', '-DBIT64', '-D_LARGEFILE64_SOURCE', '-D_FILE_OFFSET_BITS=64',                
               mpiCppImpl,
               # '-O0', # <-- Optimization level
               # '-g',  # <-- Debug symbols
               '-I%I']

compile_fmu = [cppCmd,
               '-c', '-fPIC','-I%I']

link_fmu = [cppCmd,
            '-fPIC', '-shared', '-o', '%J', '%M']

link_sl = [fortCmd,
           '-V',           
           '-cxxlib', '-fPIC', '-threads', '-shared',
           '%E', '-Wl,-soname,%U', '-o', '%U', '%F', '%A', '%L', '%B', '-parallel',           
           '-Wl,-Bdynamic', '-i-dynamic', '-lifport', '-lifcoremt', '-lmpi']

link_exe = [cppCmd,
            '-fPIC',
            '-Wl,-Bdynamic', '-o', '%J', '%F', '%M', '%L', '%B', '%O', '-lpthread', '-lm', '-lifcoremt']

# Remove the temporary names from the namespace
del cppCmd
del fortCmd
del mpiCppImpl
if mp_mpi_implementation == PMPI:
    del pmpipath
if mp_mpi_implementation == IMPI:    
    del impipath


graphicsEnv = driverUtils.locateFile(os.environ['ABA_PATH'],'site','graphicsConfig','env')
if graphicsEnv:
    execfile(graphicsEnv)
else:
    raise 'Cannot find the graphics configuration environment file (graphicsConfig.env)'

del driverUtils, os, graphicsEnv
license_server_type=FLEXNET

run_mode=INTERACTIVE

mp_host_list=(('n16-021',16),('n16-026',16),)
  • doku/abaqus6.txt
  • Last modified: 2015/09/30 13:04
  • by markus