[molpro-user] 2012.1 build with slurm/mvapich2

Sun Dec 2 16:51:11 GMT 2012

Dear all,

we would like to use molpro in conjunction with the SLURM batch queueing 
system, that is, in particular we are trying to build Molpro-2012.1 on an 
AMD Opteron QDR-Infiniband Linux-cluster (managed by the SLURM batch 
queueing system) using the following configuration:

  INTEL ifort/icc 12.1
  MKL 10.3
  GLOBAL ARRAYS 5-1-1
  MVAPICH2-1.8
  SLURM (2.4.1)

To exploit both, the fast Infiniband interconnect as well as the 
intra-node shared-memory multiple CPU architecture, we were aiming at an 
OpenMP/MPI Hybrid build.

Compilation of the code works fine, and MPI/OpenMP-hybrid molpro jobs 
start to run correctly; e.g. "molpro -n 4 -t 2 test.inp" runs on 4 CPUs, 
each spawning 2 threads = total of 8 processes, and produces some regular 
output. They do, however, cause the nodes to crash just before the 
variable memory is released (by starting a second instance of slurmstepd, 
i.e., more precisely a second set of molpro.exe's, which consume all of 
the resident memory and lead to a blow-up).

Can anyone confirm this or has anyone succeeded in a similar environment? 
I attach the current CONFIG with this email.

Cheers,
Martin

PS: After several many trials, the latest configuration we used is the 
following (utilising the nightly build of molpro):

MVAPICH2-1.8:
=============
./configure \
   --prefix=${PREFIX}/mvapich2 \
   --with-device=ch3:mrail \
   --with-rdma=gen2 \
   --enable-shared \
   --disable-xrc \
   --with-pm=no \
   --with-pmi=slurm \
   --with-limic2=/opt/limic2-0.5.5 \
   --with-hwloc=/cm/shared/apps/hwloc/1.4.2

GA-5-1-1:
=========
LIBMPI="-I${SLURM_HOME}/include -I${MPI_HOME}/include -I/opt/limic2-0.5.5/include \
  -L${SLURM_HOME}/lib -L${MPI_HOME}/lib -L/opt/limic2-0.5.5/lib \
  -lmpichf90 -lmpich -lopa -lmpl -lpmi -lpthread -llimic2 -lrdmacm -libverbs -libumad -ldl -lrt -lnuma"
BLAS_LIBS="-L${MKLROOT}/lib/intel64 -lmkl_rt -lpthread -lm"
./configure \
     MPIEXEC="srun -n %NP%" \
     CC=icc F77=ifort CXX=icpc \
     CFLAGS="-DLARGE_FILES -DMKL_ILP64 -openmp" \
     FFLAGS="-openmp" \
     --with-blas8="${BLAS_LIBS}" \
     --with-mpi="${LIBMPI}" \
     --with-tcgmsg \
     --with-openib \
     --prefix=${PREFIX}/ga

MOLPRO-2012.1:
==============
./configure \
   -batch \
   -prefix "${PREFIX}" \
   -blaspath "${MKLROOT}/lib/intel64" \
   -openmp \
   -mpp \
   -mppbase "../ga-5-1-1" \
   -ifort -icc

The CONFIG file is then edited/adapted to SLURM and thread-safe MKL by 
changing the two corresponding variables into
  LAUNCHER="srun -n %n %x"
  BLASLIB="-L${MKLROOT}/lib/intel64 -lmkl_rt -lpthread -lm"

Also, molpro.sh is edited to include "OMP_NUM_THREADS MV2_ENABLE_AFFINITY" 
with $envlist.

-- 
Dr. Martin Diefenbach
Institut für Anorganische und Analytische Chemie
der Johann Wolfgang Goethe-Universität Frankfurt
Max-von-Laue-Straße 7, D-60438 Frankfurt am Main
Campus Riedberg, N160/207, Tel: +49-69-798-29445
E-Mail: diefenbach at chemie.uni-frankfurt.de
-------------- next part --------------
# MOLPRO CONFIG generated at Sun Dec  2 16:34:06 CET 2012, for host node2-037

CONFIGURE_OPTIONS= "-prefix" "/home/compuchem/diefenbach/chemsoft/molpro/2012.1-nightly-ga-mvapich-intel-mkl-smp" "-blaspath" "/cm/shared/apps/intel/composer_xe/2011_sp1.11.339/mkl/lib/intel64" "-openmp" "-mpp" "-mppbase" "../ga-5-1-1" "-ifort" "-icc"

AR=ar
ARCHNAME=Linux/x86_64
ARFLAGS=-rS
AWK=awk
BIBTEX=/usr/bin/bibtex
BLASLIB=-L/cm/shared/apps/intel/composer_xe/2011_sp1.11.339/mkl/lib/intel64 -lmkl_rt -lpthread -lm
BUILD=p
CAT=cat
CC=/cm/shared/apps/intel/composer_xe/2011_sp1.11.339/bin/intel64/icc
CCVERSION=12.1.5
CC_FRONT=
CDEBUG=-g $(addprefix $(CDEFINE),_DEBUG)
CDEFINE=-D
CFLAGS=-ftz -vec-report0 -I/cm/shared/apps/slurm/2.4.1/include -I/opt/limic2-0.5.5/include -I/cm/shared/apps/mvapich2/intel-12.1/1.8-r5435/include -openmp
CLDFLAGS=
CLEAN=echo 'target clean only available with git cloned versions, please unpack the tarball again'
CLEARSPEEDLIB=
CMPPINCLUDE=/home/compuchem/diefenbach/chemsoft/molpro/2012.1-nightly-ga-mvapich-intel-mkl-smp/ga/include
COPT=-O2
COPT0=-O0
COPT1=-O1
COPT2=-O2
COPT3=-O3
CP=cp -p
CPROFILE=-p
CSCN=cscn
CSFLAGS=-O3 -I. --dynamic
CUDACC=
CUDACCVERSION=
CUDACDEBUG=-g $(addprefix $(CUDACDEFINE),_DEBUG)
CUDACDEFINE=-D
CUDACFLAGS=
CUDACOPT=
CUDACOPT0=-O0
CUDACOPT1=-O1
CUDACOPT2=-O2
CUDACOPT3=-O3
CUDACPROFILE=-p
CXX=/cm/shared/apps/intel/composer_xe/2011_sp1.11.339/bin/intel64/icpc
CXXFLAGS=$(CFLAGS)
DOXYGEN=
ECHO=echo
EXPORT=export
F90FLAGS=-stand f03
FC=/cm/shared/apps/intel/composer_xe/2011_sp1.11.339/bin/intel64/ifort
FCVERSION=12.1.5
FDEBUG=-g $(addprefix $(FDEFINE),_DEBUG)
FDEFINE=-D
FFLAGS=-i8 -r8 -pc64 -auto -vec-report0 -warn nousage -cxxlib -openmp
FLDFLAGS=
FOPT=-O3
FOPT0=-O0
FOPT1=-O1
FOPT2=-O2
FOPT3=-O3
FPP=-fpp
FPROFILE=-p
FSTATIC=
HOSTFILE_FORMAT=%N
INSTBIN=
INST_PL=0
INTEGER=8
LAPACKLIB=
LATEX2HTML=
LAUNCHER=/cm/shared/apps/slurm/2.4.1/bin/srun -n %n %x
LD_ENV=/cm/shared/apps/intel/composer_xe/2011_sp1.11.339/compiler/lib/intel64:/cm/shared/apps/intel/composer_xe/2011_sp1.11.339/mkl/lib/intel64
LD_ENVNAME=LD_LIBRARY_PATH
LIBRARY_SUFFIX=a
LIBS=-lrt -libverbs -lpthread -L/home/compuchem/diefenbach/build/IFC/mvapich-intel-slurm/Molpro/src/boost-install/lib -lboost_system -lboost_thread
LN=ln -s
MACROS=MOLPRO MOLPRO_f2003 GA_TOOLS GA_TCGMSGMPI GA_VERSION_GE_5 HAVE_BOOST_THREADS HAVE_SSE2 _I8_ BLAS_INT=8 LAPACK_INT=8 MOLPRO_FORCE_VECTOR MOLPRO_HPDIR MOLPRO_NEXTSCALAR MOLPRO_NO_RECURRENCE MOLPRO_NOVECTOR MOLPRO_SCALAR MOLPRO_SHORTLOOP OMP _MOLCAS_MPP_ MOLPRO_BLAS MOLPRO_LAPACK
MAKEDEPEND_OPTIONS=
MAKEINDEX=/usr/bin/makeindex
MAPLE=
MAX_INCREMENT_LIBRARY=0
MKDIR=mkdir -p
MODULE_FLAG=-I
MODULE_SUFFIX=mod
MPILIB=-L/cm/shared/apps/slurm/2.4.1/lib -L/opt/limic2-0.5.5/lib -L/cm/shared/apps/mvapich2/intel-12.1/1.8-r5435/lib -lmpichf90 -lmpich -lopa -lmpl -lpmi -lpthread -llimic2 -lrdmacm -libverbs -libumad -ldl -lrt -lnuma
MPPLIB=-L/home/compuchem/diefenbach/chemsoft/molpro/2012.1-nightly-ga-mvapich-intel-mkl-smp/ga/lib -lga -larmci
OBJECT_SUFFIX=o
OPT0=copyc6.F
OPT1=explicit_util.F cclocal.F utlmar.F matrix1.F optconical.F avcc.F koopro4.F
OPT2=tstfnc.F dftgrid.F lmo_symmetry.F mrf12_singles.F90 basis_integral_shells.F integrals.F90 geminal.F
OPT3=
PAPER=a4paper
PARSE=parse-Linux-x86_64-i8.o
PDFLATEX=/usr/bin/pdflatex -halt-on-error
PNAME=molprop_2012_1_Linux_x86_64_i8
PREFIX=/home/compuchem/diefenbach/chemsoft/molpro/2012.1-nightly-ga-mvapich-intel-mkl-smp/molprop_2012_1_Linux_x86_64_i8
PTSIZE=11
PYTHON=/usr/bin/python
RANLIB=ranlib
RM=rm -rf
SHELL=/bin/sh
STRIP=strip
SUFFIXES=F F90 c cpp
TAR=tar -cf
UNTAR=tar -xf
VERBOSE=@
VERSION=2012.1
XSD=/usr/bin/xmllint --noout --schema
XSLT=/usr/bin/xsltproc
YACC=bison -b y

.SUFFIXES:
MAKEFLAGS+=-r
ifneq ($(LD_ENVNAME),)
$(LD_ENVNAME):=$(LD_ENV):$($(LD_ENVNAME))
endif