Skip to content
[Jenkins]Jenkins
Success

Console Output

Download View as plain text
Started by user arnoldg
Building on master in workspace /var/lib/jenkins/workspace/awp-odc-spp
executing script:

set -v

#rm -rf awp-odc-spp
mkdir awp-odc-spp
cd awp-odc-spp

 tar xvfz /projects/bluewaters/benchmarking/SPP/AWP-ODC/awp-odc_gpu.tgz
 cd awp-odc-os/src/
 module unload darshan
 module swap $( module list | grep -o PrgEnv-.*$ ) PrgEnv-gnu
 module load cudatoolkit
~/bin/myenv.sh

# for topomapping run:
# module load topomapping
# cp ~/awp-odc-spp/Makefile.bluewaters.gnu.topomapping Makefile.bluewaters.gnu
 make -f Makefile.bluewaters.gnu

cd ~/awp-odc-spp/awp-odc-os/run

cat run_small.pbs | sed -e "s/aprun/time aprun/g" > runs.pbs
cp runs.pbs run_small.pbs
cat -n run_small.pbs

rm awp-odc_gpu_small.*

MYJOBID=`qsub run_small.pbs`
echo MYJOBID=$MYJOBID
~/bin//wait_for.sh $MYJOBID

tail -50 awp-odc_gpu_small.e* > ~/awp-odc-spp/stderr.txt
MYREALMIN=`grep real awp-odc_gpu_small.e* | cut -dl -f2 | cut -dm -f1`
echo "YVALUE=$MYREALMIN" > ~/awp-odc-spp/awp.realtime
cat -n ~/awp-odc-spp/awp.realtime


#rm -rf awp-odc-spp
mkdir awp-odc-spp
mkdir: cannot create directory `awp-odc-spp': File exists
cd awp-odc-spp

 tar xvfz /projects/bluewaters/benchmarking/SPP/AWP-ODC/awp-odc_gpu.tgz
awp-odc-os/
awp-odc-os/input/
awp-odc-os/input/source
awp-odc-os/input/mesh
 cd awp-odc-os/src/
 module unload darshan
 module swap $( module list | grep -o PrgEnv-.*$ ) PrgEnv-gnu
 module list | grep -o PrgEnv-.*$ 
awp-odc-os/AUTHORS
awp-odc-os/run/
awp-odc-os/run/run.bluewaters.gnu.pbs
awp-odc-os/run/run_medium.pbs
awp-odc-os/run/env_clean.sh
awp-odc-os/run/env.sh
awp-odc-os/run/run_small.pbs
awp-odc-os/README.md
awp-odc-os/LICENSE
awp-odc-os/submodules/
awp-odc-os/submodules/wiki/
awp-odc-os/src/
awp-odc-os/src/cerjan.c
awp-odc-os/src/kernel.cu
awp-odc-os/src/Makefile.bluewaters.cray
awp-odc-os/src/Makefile.bluewaters.gnu
awp-odc-os/src/grid.c
awp-odc-os/src/kernel.h
awp-odc-os/src/pmcl3d.c
awp-odc-os/src/command.c
awp-odc-os/src/io.c
awp-odc-os/src/swap.c
awp-odc-os/src/mesh.c
awp-odc-os/src/pmcl3d.h
awp-odc-os/src/source.c
awp-odc-os/src/pmcl3d_cons.h
awp-odc-os/CHANGELOG
${MODULESHOME}/bin/modulecmd bash $*
LD_LIBRARY_PATH=/usr/local/globus-5.2.5/lib64:/opt/torque/6.0.1-end/lib:/usr/local/globus/lib64 ;export LD_LIBRARY_PATH;LOADEDMODULES=modules/3.2.10.4:eswrap/1.3.3-1.020200.1278.0:cce/8.4.6:craype-network-gemini:craype/2.5.4:cray-libsci/16.03.1:udreg/2.3.2-1.0502.10518.2.17.gem:ugni/6.0-1.0502.10863.8.28.gem:pmi/5.0.10-1.0000.11050.179.3.gem:dmapp/7.0.1-1.0502.11080.8.74.gem:gni-headers/4.0-1.0502.10859.7.8.gem:xpmem/0.1-2.0502.64982.5.3.gem:dvs/2.5_0.9.0-1.0502.2188.1.113.gem:alps/5.2.4-2.0502.9774.31.12.gem:rca/1.0.0-2.0502.60530.1.63.gem:atp/2.0.1:PrgEnv-cray/5.2.82:cray-mpich/7.3.3:craype-interlagos:torque/6.0.1:moab/9.0.2.TAS2-1477409647_6706986-sles11:java/jdk1.8.0_51:globus/5.2.5:gsissh/6.2p2:scripts:user-paths:xalt/0.7.6 ;export LOADEDMODULES;PATH=/sw/xe_xk_cle5.2UP02_pe2.3.0/xalt/0.7.6/sles11.3/libexec:/sw/xe_xk_cle5.2UP02_pe2.3.0/xalt/0.7.6/sles11.3/bin:/sw/admin/scripts:/sw/user/scripts:/usr/local/gsi-openssh-6.2p2-2/bin:/opt/java/jdk1.8.0_51/bin:/usr/local/globus-5.2.5/bin:/usr/local/globus-5.2.5/sbin:/opt/moab/9.0.2t2/sbin:/opt/torque/6.0.1-end/sbin:/opt/torque/6.0.1-end/bin:/opt/cray/mpt/7.3.3/gni/bin:/opt/cray/rca/1.0.0-2.0502.60530.1.63.gem/bin:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/sbin:/opt/cray/dvs/2.5_0.9.0-1.0502.2188.1.113.gem/bin:/opt/cray/xpmem/0.1-2.0502.64982.5.3.gem/bin:/opt/cray/pmi/5.0.10-1.0000.11050.179.3.gem/bin:/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/bin:/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/bin:/opt/cray/craype/2.5.4/bin:/opt/cray/cce/8.4.6/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.4.6/craylibs/x86-64/bin:/opt/cray/cce/8.4.6/cftn/bin:/opt/cray/cce/8.4.6/CC/bin:/opt/cray/eslogin/eswrap/1.3.3-1.020200.1278.0/bin:/opt/modules/3.2.10.4/bin:/opt/moab/9.0.2t2/bin:/u/system/bwjenkins/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin ;export PATH;PE_PKGCONFIG_LIBS=mpich:AtpSigHandler:cray-rca:libsci_mpi:libsci ;export PE_PKGCONFIG_LIBS;PKG_CONFIG_PATH=/opt/cray/rca/1.0.0-2.0502.60530.1.63.gem/lib64/pkgconfig:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/lib64/pkgconfig:/opt/cray/xpmem/0.1-2.0502.64982.5.3.gem/lib64/pkgconfig:/opt/cray/gni-headers/4.0-1.0502.10859.7.8.gem/lib64/pkgconfig:/opt/cray/dmapp/7.0.1-1.0502.11080.8.74.gem/lib64/pkgconfig:/opt/cray/pmi/5.0.10-1.0000.11050.179.3.gem/lib64/pkgconfig:/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/lib64/pkgconfig:/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/lib64/pkgconfig:/opt/cray/craype/2.5.4/pkg-config:/opt/cray/iobuf/2.0.6/lib/pkgconfig:/opt/cray/atp/2.0.1/lib/pkgconfig ;export PKG_CONFIG_PATH;_LMFILES_=/opt/modulefiles/modules/3.2.10.4:/opt/modulefiles/eswrap/1.3.3-1.020200.1278.0:/opt/modulefiles/cce/8.4.6:/opt/cray/craype/2.5.4/modulefiles/craype-network-gemini:/opt/cray/modulefiles/craype/2.5.4:/opt/cray/modulefiles/cray-libsci/16.03.1:/opt/cray/gem/modulefiles/udreg/2.3.2-1.0502.10518.2.17.gem:/opt/cray/gem/modulefiles/ugni/6.0-1.0502.10863.8.28.gem:/opt/cray/gem/modulefiles/pmi/5.0.10-1.0000.11050.179.3.gem:/opt/cray/gem/modulefiles/dmapp/7.0.1-1.0502.11080.8.74.gem:/opt/cray/gem/modulefiles/gni-headers/4.0-1.0502.10859.7.8.gem:/opt/cray/gem/modulefiles/xpmem/0.1-2.0502.64982.5.3.gem:/opt/cray/gem/modulefiles/dvs/2.5_0.9.0-1.0502.2188.1.113.gem:/opt/cray/gem/modulefiles/alps/5.2.4-2.0502.9774.31.12.gem:/opt/cray/gem/modulefiles/rca/1.0.0-2.0502.60530.1.63.gem:/opt/cray/modulefiles/atp/2.0.1:/opt/cray/modulefiles/PrgEnv-cray/5.2.82:/opt/cray/modulefiles/cray-mpich/7.3.3:/opt/cray/craype/2.5.4/modulefiles/craype-interlagos:/opt/modulefiles/torque/6.0.1:/opt/modulefiles/moab/9.0.2.TAS2-1477409647_6706986-sles11:/opt/modulefiles/java/jdk1.8.0_51:/usr/local/modulefiles/globus/5.2.5:/usr/local/modulefiles/gsissh/6.2p2:/sw/xe/modulefiles/scripts:/sw/xe/modulefiles/user-paths:/sw/bw/modulefiles/xalt/0.7.6 ;export _LMFILES_;unset DARSHAN_LOGPATH;
${MODULESHOME}/bin/modulecmd bash $*
ATP_HOME=/opt/cray/atp/2.0.1 ;export ATP_HOME;ATP_IGNORE_SIGTERM=1 ;export ATP_IGNORE_SIGTERM;ATP_MRNET_COMM_PATH=/opt/cray/atp/2.0.1/libexec/atp_mrnet_commnode_wrapper ;export ATP_MRNET_COMM_PATH;ATP_POST_LINK_OPTS=-Wl,-L/opt/cray/atp/2.0.1/libApp/\ ;export ATP_POST_LINK_OPTS;CRAYOS_VERSION=5.2.82 ;export CRAYOS_VERSION;CRAYPE_DIR=/opt/cray/craype/2.5.4 ;export CRAYPE_DIR;CRAYPE_NETWORK_TARGET=gemini ;export CRAYPE_NETWORK_TARGET;CRAYPE_VERSION=2.5.4 ;export CRAYPE_VERSION;CRAY_ALPS_INCLUDE_OPTS=-I/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/include ;export CRAY_ALPS_INCLUDE_OPTS;CRAY_ALPS_POST_LINK_OPTS=-L/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/lib64 ;export CRAY_ALPS_POST_LINK_OPTS;CRAY_CPU_TARGET=interlagos ;export CRAY_CPU_TARGET;CRAY_DMAPP_INCLUDE_OPTS=-I/opt/cray/dmapp/7.0.1-1.0502.11080.8.74.gem/include\ -I/opt/cray/gni-headers/4.0-1.0502.10859.7.8.gem/include\ ;export CRAY_DMAPP_INCLUDE_OPTS;CRAY_DMAPP_POST_LINK_OPTS=-L/opt/cray/dmapp/7.0.1-1.0502.11080.8.74.gem/lib64 ;export CRAY_DMAPP_POST_LINK_OPTS;CRAY_GNI_HEADERS_INCLUDE_OPTS=-I/opt/cray/gni-headers/4.0-1.0502.10859.7.8.gem/include ;export CRAY_GNI_HEADERS_INCLUDE_OPTS;CRAY_LD_LIBRARY_PATH=/opt/cray/rca/1.0.0-2.0502.60530.1.63.gem/lib64:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/lib64:/opt/cray/xpmem/0.1-2.0502.64982.5.3.gem/lib64:/opt/cray/dmapp/7.0.1-1.0502.11080.8.74.gem/lib64:/opt/cray/pmi/5.0.10-1.0000.11050.179.3.gem/lib64:/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/lib64:/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/lib64:/opt/cray/libsci/16.03.1/GNU/4.9/x86_64/lib:/opt/cray/mpt/7.3.3/gni/mpich-gnu/4.9/lib ;export CRAY_LD_LIBRARY_PATH;CRAY_LIBSCI_BASE_DIR=/opt/cray/libsci/16.03.1 ;export CRAY_LIBSCI_BASE_DIR;CRAY_LIBSCI_DIR=/opt/cray/libsci/16.03.1 ;export CRAY_LIBSCI_DIR;CRAY_LIBSCI_PREFIX_DIR=/opt/cray/libsci/16.03.1/GNU/4.9/x86_64 ;export CRAY_LIBSCI_PREFIX_DIR;CRAY_LIBSCI_VERSION=16.03.1 ;export CRAY_LIBSCI_VERSION;CRAY_MPICH2_DIR=/opt/cray/mpt/7.3.3/gni/mpich-gnu/4.9 ;export CRAY_MPICH2_DIR;CRAY_MPICH2_VER=7.3.3 ;export CRAY_MPICH2_VER;CRAY_MPICH_BASEDIR=/opt/cray/mpt/7.3.3/gni ;export CRAY_MPICH_BASEDIR;CRAY_MPICH_DIR=/opt/cray/mpt/7.3.3/gni/mpich-gnu/4.9 ;export CRAY_MPICH_DIR;CRAY_MPICH_ROOTDIR=/opt/cray/mpt/7.3.3 ;export CRAY_MPICH_ROOTDIR;CRAY_PMI_INCLUDE_OPTS=-I/opt/cray/pmi/5.0.10-1.0000.11050.179.3.gem/include ;export CRAY_PMI_INCLUDE_OPTS;CRAY_PMI_POST_LINK_OPTS=-L/opt/cray/pmi/5.0.10-1.0000.11050.179.3.gem/lib64 ;export CRAY_PMI_POST_LINK_OPTS;CRAY_PRE_COMPILE_OPTS=-hnetwork=gemini ;export CRAY_PRE_COMPILE_OPTS;CRAY_PRGENVGNU=loaded ;export CRAY_PRGENVGNU;CRAY_RCA_INCLUDE_OPTS=-I/opt/cray/rca/1.0.0-2.0502.60530.1.63.gem/include\ -I/opt/cray/krca/1.0.0-2.0502.63139.4.30.gem/include\ -I/opt/cray-hss-devel/7.2.0/include\ ;export CRAY_RCA_INCLUDE_OPTS;CRAY_RCA_POST_LINK_OPTS=-L/opt/cray/rca/1.0.0-2.0502.60530.1.63.gem/lib64\ -lrca\ ;export CRAY_RCA_POST_LINK_OPTS;CRAY_UDREG_INCLUDE_OPTS=-I/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/include ;export CRAY_UDREG_INCLUDE_OPTS;CRAY_UDREG_POST_LINK_OPTS=-L/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/lib64 ;export CRAY_UDREG_POST_LINK_OPTS;CRAY_UGNI_INCLUDE_OPTS=-I/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/include ;export CRAY_UGNI_INCLUDE_OPTS;CRAY_UGNI_POST_LINK_OPTS=-L/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/lib64 ;export CRAY_UGNI_POST_LINK_OPTS;CRAY_XPMEM_INCLUDE_OPTS=-I/opt/cray/xpmem/0.1-2.0502.64982.5.3.gem/include ;export CRAY_XPMEM_INCLUDE_OPTS;CRAY_XPMEM_POST_LINK_OPTS=-L/opt/cray/xpmem/0.1-2.0502.64982.5.3.gem/lib64 ;export CRAY_XPMEM_POST_LINK_OPTS;DMAPP_ABORT_ON_ERROR=1 ;export DMAPP_ABORT_ON_ERROR;DVS_INCLUDE_OPTS=-I/opt/cray/dvs/2.5_0.9.0-1.0502.2188.1.113.gem/include ;export DVS_INCLUDE_OPTS;DVS_VERSION=0.9.0 ;export DVS_VERSION;GCC_PATH=/opt/gcc/4.9.3 ;export GCC_PATH;GCC_VERSION=4.9.3 ;export GCC_VERSION;GNU_VERSION=4.9.3 ;export GNU_VERSION;INFOPATH=/opt/gcc/4.9.3/snos/share/info:/usr/local/info:/usr/share/info:/usr/info ;export INFOPATH;LD_LIBRARY_PATH=/opt/gcc/4.9.3/snos/lib64:/usr/local/globus-5.2.5/lib64:/opt/torque/6.0.1-end/lib:/usr/local/globus/lib64 ;export LD_LIBRARY_PATH;LIBSCI_BASE_DIR=/opt/cray/libsci/16.03.1 ;export LIBSCI_BASE_DIR;LIBSCI_VERSION=16.03.1 ;export LIBSCI_VERSION;LOADEDMODULES=modules/3.2.10.4:eswrap/1.3.3-1.020200.1278.0:gcc/4.9.3:craype-network-gemini:craype-interlagos:craype/2.5.4:cray-mpich/7.3.3:torque/6.0.1:moab/9.0.2.TAS2-1477409647_6706986-sles11:java/jdk1.8.0_51:globus/5.2.5:gsissh/6.2p2:scripts:user-paths:xalt/0.7.6:cray-libsci/16.03.1:udreg/2.3.2-1.0502.10518.2.17.gem:ugni/6.0-1.0502.10863.8.28.gem:pmi/5.0.10-1.0000.11050.179.3.gem:dmapp/7.0.1-1.0502.11080.8.74.gem:gni-headers/4.0-1.0502.10859.7.8.gem:xpmem/0.1-2.0502.64982.5.3.gem:dvs/2.5_0.9.0-1.0502.2188.1.113.gem:alps/5.2.4-2.0502.9774.31.12.gem:rca/1.0.0-2.0502.60530.1.63.gem:atp/2.0.1:PrgEnv-gnu/5.2.82 ;export LOADEDMODULES;MANPATH=/opt/cray/atp/2.0.1/man:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/man:/opt/cray/libsci/16.03.1/man:/opt/cray/man/csmlversion:/opt/gcc/4.9.3/snos/share/man:/sw/user/man:/opt/java/jdk1.8.0_51/man:/usr/local/globus-5.2.5/man:/opt/moab/9.0.2t2/share/man/:/opt/torque/6.0.1-end/share/man:/opt/cray/mpt/7.3.3/gni/man/mpich:/opt/cray/craype/2.5.4/man:/opt/cray/eslogin/eswrap/1.3.3-1.020200.1278.0/man:/opt/modules/3.2.10.4/share/man:/opt/moab/9.0.2t2/share/man::ignore:/usr/local/man:/usr/share/man:/usr/man:/opt/cray/share/man:/cm/local/apps/environment-modules/3.2.6/man:/opt/cray/man:/u/system/bwjenkins/man ;export MANPATH;MODULEPATH=/sw/bw/modulefiles:/sw/xe/modulefiles:/usr/local/modulefiles:/opt/cray/gem/modulefiles:/opt/cray/craype/2.5.4/modulefiles:/opt/cray/modulefiles:/opt/modulefiles ;export MODULEPATH;MPICH_ABORT_ON_ERROR=1 ;export MPICH_ABORT_ON_ERROR;MPICH_DIR=/opt/cray/mpt/7.3.3/gni/mpich-gnu/4.9 ;export MPICH_DIR;PATH=/opt/cray/rca/1.0.0-2.0502.60530.1.63.gem/bin:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/sbin:/opt/cray/dvs/2.5_0.9.0-1.0502.2188.1.113.gem/bin:/opt/cray/xpmem/0.1-2.0502.64982.5.3.gem/bin:/opt/cray/pmi/5.0.10-1.0000.11050.179.3.gem/bin:/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/bin:/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/bin:/opt/gcc/4.9.3/bin:/sw/xe_xk_cle5.2UP02_pe2.3.0/xalt/0.7.6/sles11.3/libexec:/sw/xe_xk_cle5.2UP02_pe2.3.0/xalt/0.7.6/sles11.3/bin:/sw/admin/scripts:/sw/user/scripts:/usr/local/gsi-openssh-6.2p2-2/bin:/opt/java/jdk1.8.0_51/bin:/usr/local/globus-5.2.5/bin:/usr/local/globus-5.2.5/sbin:/opt/moab/9.0.2t2/sbin:/opt/torque/6.0.1-end/sbin:/opt/torque/6.0.1-end/bin:/opt/cray/mpt/7.3.3/gni/bin:/opt/cray/craype/2.5.4/bin:/opt/cray/eslogin/eswrap/1.3.3-1.020200.1278.0/bin:/opt/modules/3.2.10.4/bin:/opt/moab/9.0.2t2/bin:/u/system/bwjenkins/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin ;export PATH;PE_ACML_DEFAULT_DIR_CRAY=gfortran64 ;export PE_ACML_DEFAULT_DIR_CRAY;PE_ACML_DEFAULT_DIR_DEFAULT64=_int64 ;export PE_ACML_DEFAULT_DIR_DEFAULT64;PE_ACML_DEFAULT_DIR_GNU=gfortran64 ;export PE_ACML_DEFAULT_DIR_GNU;PE_ACML_DEFAULT_DIR_INTEL=ifort64 ;export PE_ACML_DEFAULT_DIR_INTEL;PE_ACML_DEFAULT_DIR_OPENMP=_mp ;export PE_ACML_DEFAULT_DIR_OPENMP;PE_ACML_DEFAULT_DIR_PGI=pgi64 ;export PE_ACML_DEFAULT_DIR_PGI;PE_ACML_DEFAULT_TARGET_interlagos=_fma4 ;export PE_ACML_DEFAULT_TARGET_interlagos;PE_ACML_DEFAULT_TARGET_x86_64= ;export PE_ACML_DEFAULT_TARGET_x86_64;PE_ACML_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/acml/5.3.1/@PE_ACML_DEFAULT_DIR@@PE_ACML_DEFAULT_TARGET@@PE_ACML_DEFAULT_DIR_OPENMP@@PE_ACML_DEFAULT_DIR_DEFAULT64@/lib/pkgconfig ;export PE_ACML_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_CRAY_DEFAULT_FIXED_PKGCONFIG_PATH=/opt/cray/parallel-netcdf/1.7.0/CRAY/8.3/lib/pkgconfig:/opt/cray/netcdf-hdf5parallel/4.4.0/CRAY/8.3/lib/pkgconfig:/opt/cray/netcdf/4.4.0/CRAY/8.3/lib/pkgconfig:/opt/cray/hdf5-parallel/1.8.16/CRAY/8.3/lib/pkgconfig:/opt/cray/hdf5/1.8.16/CRAY/8.3/lib/pkgconfig:/opt/cray/ga/5.3.0.6/CRAY/8.3/lib/pkgconfig ;export PE_CRAY_DEFAULT_FIXED_PKGCONFIG_PATH;PE_CXX_PKGCONFIG_LIBS=mpichcxx ;export PE_CXX_PKGCONFIG_LIBS;PE_ENV=GNU ;export PE_ENV;PE_FFTW_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH ;export PE_FFTW_DEFAULT_REQUIRED_PRODUCTS;PE_FFTW_DEFAULT_TARGET_broadwell=broadwell ;export PE_FFTW_DEFAULT_TARGET_broadwell;PE_FFTW_DEFAULT_TARGET_haswell=haswell ;export PE_FFTW_DEFAULT_TARGET_haswell;PE_FFTW_DEFAULT_TARGET_interlagos=interlagos ;export PE_FFTW_DEFAULT_TARGET_interlagos;PE_FFTW_DEFAULT_TARGET_mic_knl=mic_knl ;export PE_FFTW_DEFAULT_TARGET_mic_knl;PE_FFTW_DEFAULT_TARGET_sandybridge=sandybridge ;export PE_FFTW_DEFAULT_TARGET_sandybridge;PE_FFTW_DEFAULT_TARGET_x86_64=x86_64 ;export PE_FFTW_DEFAULT_TARGET_x86_64;PE_FFTW_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/fftw/3.3.4.7/@PE_FFTW_DEFAULT_TARGET@/lib/pkgconfig ;export PE_FFTW_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_FORTRAN_PKGCONFIG_LIBS=mpichf90 ;export PE_FORTRAN_PKGCONFIG_LIBS;PE_GA_DEFAULT_FIXED_PRGENV=CRAY\ PGI\ INTEL ;export PE_GA_DEFAULT_FIXED_PRGENV;PE_GA_DEFAULT_GENCOMPILERS_GNU=5.1\ 4.9 ;export PE_GA_DEFAULT_GENCOMPILERS_GNU;PE_GA_DEFAULT_GENCOMPS_GNU=51\ 49 ;export PE_GA_DEFAULT_GENCOMPS_GNU;PE_GA_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/ga/5.3.0.6/@PRGENV@/@PE_GA_DEFAULT_GENCOMPS@/lib/pkgconfig ;export PE_GA_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_GA_DEFAULT_VOLATILE_PRGENV=GNU ;export PE_GA_DEFAULT_VOLATILE_PRGENV;PE_HDF5_DEFAULT_FIXED_PRGENV=CRAY\ PGI ;export PE_HDF5_DEFAULT_FIXED_PRGENV;PE_HDF5_DEFAULT_GENCOMPILERS_GNU=5.1\ 4.9 ;export PE_HDF5_DEFAULT_GENCOMPILERS_GNU;PE_HDF5_DEFAULT_GENCOMPILERS_INTEL=15.0\ 14.0 ;export PE_HDF5_DEFAULT_GENCOMPILERS_INTEL;PE_HDF5_DEFAULT_GENCOMPS_GNU=51\ 49 ;export PE_HDF5_DEFAULT_GENCOMPS_GNU;PE_HDF5_DEFAULT_GENCOMPS_INTEL=150\ 140 ;export PE_HDF5_DEFAULT_GENCOMPS_INTEL;PE_HDF5_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH ;export PE_HDF5_DEFAULT_REQUIRED_PRODUCTS;PE_HDF5_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/hdf5/1.8.16/@PRGENV@/@PE_HDF5_DEFAULT_GENCOMPS@/lib/pkgconfig ;export PE_HDF5_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_HDF5_DEFAULT_VOLATILE_PRGENV=GNU\ INTEL ;export PE_HDF5_DEFAULT_VOLATILE_PRGENV;PE_HDF5_PARALLEL_DEFAULT_FIXED_PRGENV=CRAY\ PGI ;export PE_HDF5_PARALLEL_DEFAULT_FIXED_PRGENV;PE_HDF5_PARALLEL_DEFAULT_GENCOMPILERS_GNU=5.1\ 4.9 ;export PE_HDF5_PARALLEL_DEFAULT_GENCOMPILERS_GNU;PE_HDF5_PARALLEL_DEFAULT_GENCOMPILERS_INTEL=15.0\ 14.0 ;export PE_HDF5_PARALLEL_DEFAULT_GENCOMPILERS_INTEL;PE_HDF5_PARALLEL_DEFAULT_GENCOMPS_GNU=51\ 49 ;export PE_HDF5_PARALLEL_DEFAULT_GENCOMPS_GNU;PE_HDF5_PARALLEL_DEFAULT_GENCOMPS_INTEL=150\ 140 ;export PE_HDF5_PARALLEL_DEFAULT_GENCOMPS_INTEL;PE_HDF5_PARALLEL_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH ;export PE_HDF5_PARALLEL_DEFAULT_REQUIRED_PRODUCTS;PE_HDF5_PARALLEL_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/hdf5-parallel/1.8.16/@PRGENV@/@PE_HDF5_PARALLEL_DEFAULT_GENCOMPS@/lib/pkgconfig ;export PE_HDF5_PARALLEL_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_HDF5_PARALLEL_DEFAULT_VOLATILE_PRGENV=GNU\ INTEL ;export PE_HDF5_PARALLEL_DEFAULT_VOLATILE_PRGENV;PE_INTEL_DEFAULT_FIXED_PKGCONFIG_PATH=/opt/cray/parallel-netcdf/1.7.0/INTEL/15.0/lib/pkgconfig:/opt/cray/netcdf-hdf5parallel/4.4.0/INTEL/15.0/lib/pkgconfig:/opt/cray/netcdf/4.4.0/INTEL/15.0/lib/pkgconfig:/opt/cray/mpt/7.3.3/gni/mpich-intel/15.0/lib/pkgconfig:/opt/cray/ga/5.3.0.6/INTEL/15.0/lib/pkgconfig ;export PE_INTEL_DEFAULT_FIXED_PKGCONFIG_PATH;PE_LIBSCI_ACC_DEFAULT_GENCOMPILERS_CRAY_x86_64=8.3 ;export PE_LIBSCI_ACC_DEFAULT_GENCOMPILERS_CRAY_x86_64;PE_LIBSCI_ACC_DEFAULT_GENCOMPILERS_GNU_x86_64=4.9 ;export PE_LIBSCI_ACC_DEFAULT_GENCOMPILERS_GNU_x86_64;PE_LIBSCI_ACC_DEFAULT_GENCOMPS_CRAY_x86_64=83 ;export PE_LIBSCI_ACC_DEFAULT_GENCOMPS_CRAY_x86_64;PE_LIBSCI_ACC_DEFAULT_GENCOMPS_GNU_x86_64=49 ;export PE_LIBSCI_ACC_DEFAULT_GENCOMPS_GNU_x86_64;PE_LIBSCI_ACC_DEFAULT_NV_SUFFIX_nvidia20=nv20 ;export PE_LIBSCI_ACC_DEFAULT_NV_SUFFIX_nvidia20;PE_LIBSCI_ACC_DEFAULT_NV_SUFFIX_nvidia35=nv35 ;export PE_LIBSCI_ACC_DEFAULT_NV_SUFFIX_nvidia35;PE_LIBSCI_ACC_DEFAULT_PKGCONFIG_VARIABLES=PE_LIBSCI_ACC_DEFAULT_NV_SUFFIX_@accelerator@ ;export PE_LIBSCI_ACC_DEFAULT_PKGCONFIG_VARIABLES;PE_LIBSCI_ACC_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH:PE_LIBSCI ;export PE_LIBSCI_ACC_DEFAULT_REQUIRED_PRODUCTS;PE_LIBSCI_ACC_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/libsci_acc/16.03.1/@PRGENV@/@PE_LIBSCI_ACC_DEFAULT_GENCOMPS@/@PE_LIBSCI_ACC_DEFAULT_TARGET@/lib/pkgconfig ;export PE_LIBSCI_ACC_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_LIBSCI_ACC_DEFAULT_VOLATILE_PRGENV=CRAY\ GNU ;export PE_LIBSCI_ACC_DEFAULT_VOLATILE_PRGENV;PE_LIBSCI_DEFAULT_GENCOMPILERS_CRAY_mic_knl=8.5 ;export PE_LIBSCI_DEFAULT_GENCOMPILERS_CRAY_mic_knl;PE_LIBSCI_DEFAULT_GENCOMPILERS_CRAY_x86_64=8.3 ;export PE_LIBSCI_DEFAULT_GENCOMPILERS_CRAY_x86_64;PE_LIBSCI_DEFAULT_GENCOMPILERS_GNU_mic_knl=5.1 ;export PE_LIBSCI_DEFAULT_GENCOMPILERS_GNU_mic_knl;PE_LIBSCI_DEFAULT_GENCOMPILERS_GNU_x86_64=5.1\ 4.9 ;export PE_LIBSCI_DEFAULT_GENCOMPILERS_GNU_x86_64;PE_LIBSCI_DEFAULT_GENCOMPILERS_INTEL_mic_knl=15.0 ;export PE_LIBSCI_DEFAULT_GENCOMPILERS_INTEL_mic_knl;PE_LIBSCI_DEFAULT_GENCOMPILERS_INTEL_x86_64=15.0 ;export PE_LIBSCI_DEFAULT_GENCOMPILERS_INTEL_x86_64;PE_LIBSCI_DEFAULT_GENCOMPILERS_PGI_x86_64=15.3 ;export PE_LIBSCI_DEFAULT_GENCOMPILERS_PGI_x86_64;PE_LIBSCI_DEFAULT_GENCOMPS_CRAY_mic_knl=85 ;export PE_LIBSCI_DEFAULT_GENCOMPS_CRAY_mic_knl;PE_LIBSCI_DEFAULT_GENCOMPS_CRAY_x86_64=83 ;export PE_LIBSCI_DEFAULT_GENCOMPS_CRAY_x86_64;PE_LIBSCI_DEFAULT_GENCOMPS_GNU_mic_knl=51 ;export PE_LIBSCI_DEFAULT_GENCOMPS_GNU_mic_knl;PE_LIBSCI_DEFAULT_GENCOMPS_GNU_x86_64=51\ 49 ;export PE_LIBSCI_DEFAULT_GENCOMPS_GNU_x86_64;PE_LIBSCI_DEFAULT_GENCOMPS_INTEL_mic_knl=150 ;export PE_LIBSCI_DEFAULT_GENCOMPS_INTEL_mic_knl;PE_LIBSCI_DEFAULT_GENCOMPS_INTEL_x86_64=150 ;export PE_LIBSCI_DEFAULT_GENCOMPS_INTEL_x86_64;PE_LIBSCI_DEFAULT_GENCOMPS_PGI_x86_64=153 ;export PE_LIBSCI_DEFAULT_GENCOMPS_PGI_x86_64;PE_LIBSCI_DEFAULT_OMP_REQUIRES= ;export PE_LIBSCI_DEFAULT_OMP_REQUIRES;PE_LIBSCI_DEFAULT_OMP_REQUIRES_openmp=_mp ;export PE_LIBSCI_DEFAULT_OMP_REQUIRES_openmp;PE_LIBSCI_DEFAULT_PKGCONFIG_VARIABLES=PE_LIBSCI_DEFAULT_OMP_REQUIRES_@openmp@ ;export PE_LIBSCI_DEFAULT_PKGCONFIG_VARIABLES;PE_LIBSCI_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH ;export PE_LIBSCI_DEFAULT_REQUIRED_PRODUCTS;PE_LIBSCI_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/libsci/16.03.1/@PRGENV@/@PE_LIBSCI_DEFAULT_GENCOMPS@/@PE_LIBSCI_DEFAULT_TARGET@/lib/pkgconfig ;export PE_LIBSCI_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_LIBSCI_DEFAULT_VOLATILE_PRGENV=CRAY\ GNU\ INTEL\ PGI ;export PE_LIBSCI_DEFAULT_VOLATILE_PRGENV;PE_LIBSCI_GENCOMPILERS_CRAY_mic_knl=8.5 ;export PE_LIBSCI_GENCOMPILERS_CRAY_mic_knl;PE_LIBSCI_GENCOMPILERS_CRAY_x86_64=8.3 ;export PE_LIBSCI_GENCOMPILERS_CRAY_x86_64;PE_LIBSCI_GENCOMPILERS_GNU_mic_knl=5.1 ;export PE_LIBSCI_GENCOMPILERS_GNU_mic_knl;PE_LIBSCI_GENCOMPILERS_GNU_x86_64=5.1\ 4.9 ;export PE_LIBSCI_GENCOMPILERS_GNU_x86_64;PE_LIBSCI_GENCOMPILERS_INTEL_mic_knl=15.0 ;export PE_LIBSCI_GENCOMPILERS_INTEL_mic_knl;PE_LIBSCI_GENCOMPILERS_INTEL_x86_64=15.0 ;export PE_LIBSCI_GENCOMPILERS_INTEL_x86_64;PE_LIBSCI_GENCOMPILERS_PGI_x86_64=15.3 ;export PE_LIBSCI_GENCOMPILERS_PGI_x86_64;PE_LIBSCI_GENCOMPS_CRAY_mic_knl=85 ;export PE_LIBSCI_GENCOMPS_CRAY_mic_knl;PE_LIBSCI_GENCOMPS_CRAY_x86_64=83 ;export PE_LIBSCI_GENCOMPS_CRAY_x86_64;PE_LIBSCI_GENCOMPS_GNU_mic_knl=51 ;export PE_LIBSCI_GENCOMPS_GNU_mic_knl;PE_LIBSCI_GENCOMPS_GNU_x86_64=51\ 49 ;export PE_LIBSCI_GENCOMPS_GNU_x86_64;PE_LIBSCI_GENCOMPS_INTEL_mic_knl=150 ;export PE_LIBSCI_GENCOMPS_INTEL_mic_knl;PE_LIBSCI_GENCOMPS_INTEL_x86_64=150 ;export PE_LIBSCI_GENCOMPS_INTEL_x86_64;PE_LIBSCI_GENCOMPS_PGI_x86_64=153 ;export PE_LIBSCI_GENCOMPS_PGI_x86_64;PE_LIBSCI_MODULE_NAME=cray-libsci/16.03.1 ;export PE_LIBSCI_MODULE_NAME;PE_LIBSCI_OMP_REQUIRES= ;export PE_LIBSCI_OMP_REQUIRES;PE_LIBSCI_OMP_REQUIRES_openmp=_mp ;export PE_LIBSCI_OMP_REQUIRES_openmp;PE_LIBSCI_PKGCONFIG_LIBS=libsci_mpi:libsci ;export PE_LIBSCI_PKGCONFIG_LIBS;PE_LIBSCI_PKGCONFIG_VARIABLES=PE_LIBSCI_OMP_REQUIRES_@openmp@ ;export PE_LIBSCI_PKGCONFIG_VARIABLES;PE_LIBSCI_REQUIRED_PRODUCTS=PE_MPICH ;export PE_LIBSCI_REQUIRED_PRODUCTS;PE_LIBSCI_VOLATILE_PKGCONFIG_PATH=/opt/cray/libsci/16.03.1/@PRGENV@/@PE_LIBSCI_GENCOMPS@/@PE_LIBSCI_TARGET@/lib/pkgconfig ;export PE_LIBSCI_VOLATILE_PKGCONFIG_PATH;PE_LIBSCI_VOLATILE_PRGENV=CRAY\ GNU\ INTEL\ PGI ;export PE_LIBSCI_VOLATILE_PRGENV;PE_MPICH_CXX_PKGCONFIG_LIBS=mpichcxx ;export PE_MPICH_CXX_PKGCONFIG_LIBS;PE_MPICH_DEFAULT_DIR_CRAY_DEFAULT64=64 ;export PE_MPICH_DEFAULT_DIR_CRAY_DEFAULT64;PE_MPICH_DEFAULT_DIR_PGI_DEFAULT64=64 ;export PE_MPICH_DEFAULT_DIR_PGI_DEFAULT64;PE_MPICH_DEFAULT_FIXED_PRGENV=INTEL ;export PE_MPICH_DEFAULT_FIXED_PRGENV;PE_MPICH_DEFAULT_GENCOMPILERS_CRAY=8.3 ;export PE_MPICH_DEFAULT_GENCOMPILERS_CRAY;PE_MPICH_DEFAULT_GENCOMPILERS_GNU=5.1\ 4.9 ;export PE_MPICH_DEFAULT_GENCOMPILERS_GNU;PE_MPICH_DEFAULT_GENCOMPILERS_PGI=15.3 ;export PE_MPICH_DEFAULT_GENCOMPILERS_PGI;PE_MPICH_DEFAULT_GENCOMPS_CRAY=83 ;export PE_MPICH_DEFAULT_GENCOMPS_CRAY;PE_MPICH_DEFAULT_GENCOMPS_GNU=51\ 49 ;export PE_MPICH_DEFAULT_GENCOMPS_GNU;PE_MPICH_DEFAULT_GENCOMPS_PGI=153 ;export PE_MPICH_DEFAULT_GENCOMPS_PGI;PE_MPICH_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/mpt/7.3.3/gni/mpich-@PRGENV@@PE_MPICH_DEFAULT_DIR_DEFAULT64@/@PE_MPICH_DEFAULT_GENCOMPS@/lib/pkgconfig ;export PE_MPICH_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_MPICH_DEFAULT_VOLATILE_PRGENV=CRAY\ GNU\ PGI ;export PE_MPICH_DEFAULT_VOLATILE_PRGENV;PE_MPICH_FORTRAN_PKGCONFIG_LIBS=mpichf90 ;export PE_MPICH_FORTRAN_PKGCONFIG_LIBS;PE_MPICH_MODULE_NAME=cray-mpich ;export PE_MPICH_MODULE_NAME;PE_MPICH_MULTITHREADED_LIBS_multithreaded=_mt ;export PE_MPICH_MULTITHREADED_LIBS_multithreaded;PE_MPICH_NV_LIBS= ;export PE_MPICH_NV_LIBS;PE_MPICH_NV_LIBS_nvidia20=-lcudart ;export PE_MPICH_NV_LIBS_nvidia20;PE_MPICH_NV_LIBS_nvidia35=-lcudart ;export PE_MPICH_NV_LIBS_nvidia35;PE_MPICH_PKGCONFIG_LIBS=mpich ;export PE_MPICH_PKGCONFIG_LIBS;PE_MPICH_PKGCONFIG_VARIABLES=PE_MPICH_NV_LIBS_@accelerator@:PE_MPICH_MULTITHREADED_LIBS_@multithreaded@ ;export PE_MPICH_PKGCONFIG_VARIABLES;PE_MPICH_TARGET_VAR_nvidia20=-lcudart ;export PE_MPICH_TARGET_VAR_nvidia20;PE_MPICH_TARGET_VAR_nvidia35=-lcudart ;export PE_MPICH_TARGET_VAR_nvidia35;PE_NETCDF_DEFAULT_FIXED_PRGENV=CRAY\ PGI\ INTEL ;export PE_NETCDF_DEFAULT_FIXED_PRGENV;PE_NETCDF_DEFAULT_GENCOMPILERS_GNU=5.1\ 4.9 ;export PE_NETCDF_DEFAULT_GENCOMPILERS_GNU;PE_NETCDF_DEFAULT_GENCOMPS_GNU=51\ 49 ;export PE_NETCDF_DEFAULT_GENCOMPS_GNU;PE_NETCDF_DEFAULT_REQUIRED_PRODUCTS=PE_HDF5 ;export PE_NETCDF_DEFAULT_REQUIRED_PRODUCTS;PE_NETCDF_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/netcdf/4.4.0/@PRGENV@/@PE_NETCDF_DEFAULT_GENCOMPS@/lib/pkgconfig ;export PE_NETCDF_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_NETCDF_DEFAULT_VOLATILE_PRGENV=GNU ;export PE_NETCDF_DEFAULT_VOLATILE_PRGENV;PE_NETCDF_HDF5PARALLEL_DEFAULT_FIXED_PRGENV=CRAY\ PGI\ INTEL ;export PE_NETCDF_HDF5PARALLEL_DEFAULT_FIXED_PRGENV;PE_NETCDF_HDF5PARALLEL_DEFAULT_GENCOMPILERS_GNU=5.1\ 4.9 ;export PE_NETCDF_HDF5PARALLEL_DEFAULT_GENCOMPILERS_GNU;PE_NETCDF_HDF5PARALLEL_DEFAULT_GENCOMPS_GNU=51\ 49 ;export PE_NETCDF_HDF5PARALLEL_DEFAULT_GENCOMPS_GNU;PE_NETCDF_HDF5PARALLEL_DEFAULT_REQUIRED_PRODUCTS=PE_HDF5_PARALLEL:PE_MPICH ;export PE_NETCDF_HDF5PARALLEL_DEFAULT_REQUIRED_PRODUCTS;PE_NETCDF_HDF5PARALLEL_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/netcdf-hdf5parallel/4.4.0/@PRGENV@/@PE_NETCDF_HDF5PARALLEL_DEFAULT_GENCOMPS@/lib/pkgconfig ;export PE_NETCDF_HDF5PARALLEL_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_NETCDF_HDF5PARALLEL_DEFAULT_VOLATILE_PRGENV=GNU ;export PE_NETCDF_HDF5PARALLEL_DEFAULT_VOLATILE_PRGENV;PE_PAPI_DEFAULT_ACCEL_LIBS= ;export PE_PAPI_DEFAULT_ACCEL_LIBS;PE_PAPI_DEFAULT_ACCEL_LIBS_nvidia35=,-lcupti,-lcudart,-lcuda ;export PE_PAPI_DEFAULT_ACCEL_LIBS_nvidia35;PE_PAPI_DEFAULT_PKGCONFIG_VARIABLES=PE_PAPI_ACCEL_LIBS_@accelerator@ ;export PE_PAPI_DEFAULT_PKGCONFIG_VARIABLES;PE_PAPI_DEFAULT_TARGET_VAR_nvidia35=,-lcupti,-lcudart,-lcuda ;export PE_PAPI_DEFAULT_TARGET_VAR_nvidia35;PE_PARALLEL_NETCDF_DEFAULT_FIXED_PRGENV=CRAY\ PGI\ INTEL ;export PE_PARALLEL_NETCDF_DEFAULT_FIXED_PRGENV;PE_PARALLEL_NETCDF_DEFAULT_GENCOMPILERS_GNU=5.1\ 4.9 ;export PE_PARALLEL_NETCDF_DEFAULT_GENCOMPILERS_GNU;PE_PARALLEL_NETCDF_DEFAULT_GENCOMPS_GNU=51\ 49 ;export PE_PARALLEL_NETCDF_DEFAULT_GENCOMPS_GNU;PE_PARALLEL_NETCDF_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/parallel-netcdf/1.7.0/@PRGENV@/@PE_PARALLEL_NETCDF_DEFAULT_GENCOMPS@/lib/pkgconfig ;export PE_PARALLEL_NETCDF_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_PARALLEL_NETCDF_DEFAULT_VOLATILE_PRGENV=GNU ;export PE_PARALLEL_NETCDF_DEFAULT_VOLATILE_PRGENV;PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_haswell=8.3 ;export PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_haswell;PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_interlagos=8.3 ;export PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_interlagos;PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_mic_knl=8.5 ;export PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_mic_knl;PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_sandybridge=8.3 ;export PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_sandybridge;PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_x86_64=8.3 ;export PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_x86_64;PE_PETSC_DEFAULT_GENCOMPILERS_GNU_haswell=5.1\ 4.9 ;export PE_PETSC_DEFAULT_GENCOMPILERS_GNU_haswell;PE_PETSC_DEFAULT_GENCOMPILERS_GNU_interlagos=5.1\ 4.9 ;export PE_PETSC_DEFAULT_GENCOMPILERS_GNU_interlagos;PE_PETSC_DEFAULT_GENCOMPILERS_GNU_sandybridge=5.1\ 4.9 ;export PE_PETSC_DEFAULT_GENCOMPILERS_GNU_sandybridge;PE_PETSC_DEFAULT_GENCOMPILERS_GNU_x86_64=5.1\ 4.9 ;export PE_PETSC_DEFAULT_GENCOMPILERS_GNU_x86_64;PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_haswell=15.0 ;export PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_haswell;PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_interlagos=15.0 ;export PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_interlagos;PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_sandybridge=15.0 ;export PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_sandybridge;PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_x86_64=15.0 ;export PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_x86_64;PE_PETSC_DEFAULT_GENCOMPILERS_PGI_interlagos=15.3 ;export PE_PETSC_DEFAULT_GENCOMPILERS_PGI_interlagos;PE_PETSC_DEFAULT_GENCOMPILERS_PGI_x86_64=15.3 ;export PE_PETSC_DEFAULT_GENCOMPILERS_PGI_x86_64;PE_PETSC_DEFAULT_GENCOMPS_CRAY_haswell=83 ;export PE_PETSC_DEFAULT_GENCOMPS_CRAY_haswell;PE_PETSC_DEFAULT_GENCOMPS_CRAY_interlagos=83 ;export PE_PETSC_DEFAULT_GENCOMPS_CRAY_interlagos;PE_PETSC_DEFAULT_GENCOMPS_CRAY_mic_knl=85 ;export PE_PETSC_DEFAULT_GENCOMPS_CRAY_mic_knl;PE_PETSC_DEFAULT_GENCOMPS_CRAY_sandybridge=83 ;export PE_PETSC_DEFAULT_GENCOMPS_CRAY_sandybridge;PE_PETSC_DEFAULT_GENCOMPS_CRAY_x86_64=83 ;export PE_PETSC_DEFAULT_GENCOMPS_CRAY_x86_64;PE_PETSC_DEFAULT_GENCOMPS_GNU_haswell=51\ 49 ;export PE_PETSC_DEFAULT_GENCOMPS_GNU_haswell;PE_PETSC_DEFAULT_GENCOMPS_GNU_interlagos=51\ 49 ;export PE_PETSC_DEFAULT_GENCOMPS_GNU_interlagos;PE_PETSC_DEFAULT_GENCOMPS_GNU_sandybridge=51\ 49 ;export PE_PETSC_DEFAULT_GENCOMPS_GNU_sandybridge;PE_PETSC_DEFAULT_GENCOMPS_GNU_x86_64=51\ 49 ;export PE_PETSC_DEFAULT_GENCOMPS_GNU_x86_64;PE_PETSC_DEFAULT_GENCOMPS_INTEL_haswell=150 ;export PE_PETSC_DEFAULT_GENCOMPS_INTEL_haswell;PE_PETSC_DEFAULT_GENCOMPS_INTEL_interlagos=150 ;export PE_PETSC_DEFAULT_GENCOMPS_INTEL_interlagos;PE_PETSC_DEFAULT_GENCOMPS_INTEL_sandybridge=150 ;export PE_PETSC_DEFAULT_GENCOMPS_INTEL_sandybridge;PE_PETSC_DEFAULT_GENCOMPS_INTEL_x86_64=150 ;export PE_PETSC_DEFAULT_GENCOMPS_INTEL_x86_64;PE_PETSC_DEFAULT_GENCOMPS_PGI_interlagos=153 ;export PE_PETSC_DEFAULT_GENCOMPS_PGI_interlagos;PE_PETSC_DEFAULT_GENCOMPS_PGI_x86_64=153 ;export PE_PETSC_DEFAULT_GENCOMPS_PGI_x86_64;PE_PETSC_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH:PE_LIBSCI:PE_HDF5_PARALLEL:PE_TPSL ;export PE_PETSC_DEFAULT_REQUIRED_PRODUCTS;PE_PETSC_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/petsc/3.6.3.0/complex/@PRGENV@/@PE_PETSC_DEFAULT_GENCOMPS@/@PE_PETSC_DEFAULT_TARGET@/lib/pkgconfig ;export PE_PETSC_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_PETSC_DEFAULT_VOLATILE_PRGENV=CRAY\ CRAY64\ GNU\ GNU64\ INTEL\ INTEL64\ PGI\ PGI64 ;export PE_PETSC_DEFAULT_VOLATILE_PRGENV;PE_PGI_DEFAULT_FIXED_PKGCONFIG_PATH=/opt/cray/parallel-netcdf/1.7.0/PGI/15.3/lib/pkgconfig:/opt/cray/netcdf-hdf5parallel/4.4.0/PGI/15.3/lib/pkgconfig:/opt/cray/netcdf/4.4.0/PGI/15.3/lib/pkgconfig:/opt/cray/hdf5-parallel/1.8.16/PGI/15.3/lib/pkgconfig:/opt/cray/hdf5/1.8.16/PGI/15.3/lib/pkgconfig:/opt/cray/ga/5.3.0.6/PGI/15.3/lib/pkgconfig ;export PE_PGI_DEFAULT_FIXED_PKGCONFIG_PATH;PE_PKGCONFIG_DEFAULT_PRODUCTS=PE_TRILINOS:PE_TPSL_64:PE_TPSL:PE_PETSC:PE_PARALLEL_NETCDF:PE_NETCDF_HDF5PARALLEL:PE_NETCDF:PE_MPICH:PE_LIBSCI_ACC:PE_LIBSCI:PE_HDF5_PARALLEL:PE_HDF5:PE_GA:PE_FFTW:PE_ACML ;export PE_PKGCONFIG_DEFAULT_PRODUCTS;PE_PKGCONFIG_LIBS=AtpSigHandler:cray-rca:libsci_mpi:libsci:mpich ;export PE_PKGCONFIG_LIBS;PE_PKGCONFIG_PRODUCTS=PE_LIBSCI:PE_MPICH ;export PE_PKGCONFIG_PRODUCTS;PE_PKGCONFIG_PRODUCTS_DEFAULT=PE_PAPI ;export PE_PKGCONFIG_PRODUCTS_DEFAULT;PE_PKG_CONFIG_PATH=/opt/cray/cti/1.0.4/lib/pkgconfig ;export PE_PKG_CONFIG_PATH;PE_PRODUCT_LIST=CRAY_RCA:CRAY_ALPS:DVS:CRAY_XPMEM:CRAY_DMAPP:CRAY_PMI:CRAY_UGNI:CRAY_UDREG:CRAY_LIBSCI:GNU:GCC:CRAYPE:CRAYPE_INTERLAGOS ;export PE_PRODUCT_LIST;PE_SMA_DEFAULT_COMPFLAG= ;export PE_SMA_DEFAULT_COMPFLAG;PE_SMA_DEFAULT_COMPFLAG_GNU=-fcray-pointer ;export PE_SMA_DEFAULT_COMPFLAG_GNU;PE_SMA_DEFAULT_DIR_CRAY_DEFAULT64=64 ;export PE_SMA_DEFAULT_DIR_CRAY_DEFAULT64;PE_SMA_DEFAULT_DIR_PGI_DEFAULT64=64 ;export PE_SMA_DEFAULT_DIR_PGI_DEFAULT64;PE_SMA_DEFAULT_PKGCONFIG_VARIABLES=PE_SMA_COMPFLAG_@prgenv@ ;export PE_SMA_DEFAULT_PKGCONFIG_VARIABLES;PE_SMA_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/mpt/7.3.3/gni/sma@PE_SMA_DEFAULT_DIR_DEFAULT64@/lib64/pkgconfig ;export PE_SMA_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_TPSL_64_DEFAULT_GENCOMPILERS_CRAY_haswell=8.3 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_CRAY_haswell;PE_TPSL_64_DEFAULT_GENCOMPILERS_CRAY_interlagos=8.3 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_CRAY_interlagos;PE_TPSL_64_DEFAULT_GENCOMPILERS_CRAY_mic_knl=8.5 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_CRAY_mic_knl;PE_TPSL_64_DEFAULT_GENCOMPILERS_CRAY_sandybridge=8.3 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_CRAY_sandybridge;PE_TPSL_64_DEFAULT_GENCOMPILERS_CRAY_x86_64=8.3 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_CRAY_x86_64;PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_haswell=5.1\ 4.9 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_haswell;PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_interlagos=5.1\ 4.9 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_interlagos;PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_sandybridge=5.1\ 4.9 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_sandybridge;PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_x86_64=5.1\ 4.9 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_x86_64;PE_TPSL_64_DEFAULT_GENCOMPILERS_INTEL_haswell=15.0 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_INTEL_haswell;PE_TPSL_64_DEFAULT_GENCOMPILERS_INTEL_interlagos=15.0 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_INTEL_interlagos;PE_TPSL_64_DEFAULT_GENCOMPILERS_INTEL_sandybridge=15.0 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_INTEL_sandybridge;PE_TPSL_64_DEFAULT_GENCOMPILERS_INTEL_x86_64=15.0 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_INTEL_x86_64;PE_TPSL_64_DEFAULT_GENCOMPILERS_PGI_interlagos=15.3 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_PGI_interlagos;PE_TPSL_64_DEFAULT_GENCOMPILERS_PGI_x86_64=15.3 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_PGI_x86_64;PE_TPSL_64_DEFAULT_GENCOMPS_CRAY_haswell=83 ;export PE_TPSL_64_DEFAULT_GENCOMPS_CRAY_haswell;PE_TPSL_64_DEFAULT_GENCOMPS_CRAY_interlagos=83 ;export PE_TPSL_64_DEFAULT_GENCOMPS_CRAY_interlagos;PE_TPSL_64_DEFAULT_GENCOMPS_CRAY_mic_knl=85 ;export PE_TPSL_64_DEFAULT_GENCOMPS_CRAY_mic_knl;PE_TPSL_64_DEFAULT_GENCOMPS_CRAY_sandybridge=83 ;export PE_TPSL_64_DEFAULT_GENCOMPS_CRAY_sandybridge;PE_TPSL_64_DEFAULT_GENCOMPS_CRAY_x86_64=83 ;export PE_TPSL_64_DEFAULT_GENCOMPS_CRAY_x86_64;PE_TPSL_64_DEFAULT_GENCOMPS_GNU_haswell=51\ 49 ;export PE_TPSL_64_DEFAULT_GENCOMPS_GNU_haswell;PE_TPSL_64_DEFAULT_GENCOMPS_GNU_interlagos=51\ 49 ;export PE_TPSL_64_DEFAULT_GENCOMPS_GNU_interlagos;PE_TPSL_64_DEFAULT_GENCOMPS_GNU_sandybridge=51\ 49 ;export PE_TPSL_64_DEFAULT_GENCOMPS_GNU_sandybridge;PE_TPSL_64_DEFAULT_GENCOMPS_GNU_x86_64=51\ 49 ;export PE_TPSL_64_DEFAULT_GENCOMPS_GNU_x86_64;PE_TPSL_64_DEFAULT_GENCOMPS_INTEL_haswell=150 ;export PE_TPSL_64_DEFAULT_GENCOMPS_INTEL_haswell;PE_TPSL_64_DEFAULT_GENCOMPS_INTEL_interlagos=150 ;export PE_TPSL_64_DEFAULT_GENCOMPS_INTEL_interlagos;PE_TPSL_64_DEFAULT_GENCOMPS_INTEL_sandybridge=150 ;export PE_TPSL_64_DEFAULT_GENCOMPS_INTEL_sandybridge;PE_TPSL_64_DEFAULT_GENCOMPS_INTEL_x86_64=150 ;export PE_TPSL_64_DEFAULT_GENCOMPS_INTEL_x86_64;PE_TPSL_64_DEFAULT_GENCOMPS_PGI_interlagos=153 ;export PE_TPSL_64_DEFAULT_GENCOMPS_PGI_interlagos;PE_TPSL_64_DEFAULT_GENCOMPS_PGI_x86_64=153 ;export PE_TPSL_64_DEFAULT_GENCOMPS_PGI_x86_64;PE_TPSL_64_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH:PE_LIBSCI ;export PE_TPSL_64_DEFAULT_REQUIRED_PRODUCTS;PE_TPSL_64_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/tpsl/16.03.1/@PRGENV@64/@PE_TPSL_64_DEFAULT_GENCOMPS@/@PE_TPSL_64_DEFAULT_TARGET@/lib/pkgconfig ;export PE_TPSL_64_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_TPSL_64_DEFAULT_VOLATILE_PRGENV=CRAY\ CRAY64\ GNU\ GNU64\ INTEL\ INTEL64\ PGI\ PGI64 ;export PE_TPSL_64_DEFAULT_VOLATILE_PRGENV;PE_TPSL_DEFAULT_GENCOMPILERS_CRAY_haswell=8.3 ;export PE_TPSL_DEFAULT_GENCOMPILERS_CRAY_haswell;PE_TPSL_DEFAULT_GENCOMPILERS_CRAY_interlagos=8.3 ;export PE_TPSL_DEFAULT_GENCOMPILERS_CRAY_interlagos;PE_TPSL_DEFAULT_GENCOMPILERS_CRAY_mic_knl=8.5 ;export PE_TPSL_DEFAULT_GENCOMPILERS_CRAY_mic_knl;PE_TPSL_DEFAULT_GENCOMPILERS_CRAY_sandybridge=8.3 ;export PE_TPSL_DEFAULT_GENCOMPILERS_CRAY_sandybridge;PE_TPSL_DEFAULT_GENCOMPILERS_CRAY_x86_64=8.3 ;export PE_TPSL_DEFAULT_GENCOMPILERS_CRAY_x86_64;PE_TPSL_DEFAULT_GENCOMPILERS_GNU_haswell=5.1\ 4.9 ;export PE_TPSL_DEFAULT_GENCOMPILERS_GNU_haswell;PE_TPSL_DEFAULT_GENCOMPILERS_GNU_interlagos=5.1\ 4.9 ;export PE_TPSL_DEFAULT_GENCOMPILERS_GNU_interlagos;PE_TPSL_DEFAULT_GENCOMPILERS_GNU_sandybridge=5.1\ 4.9 ;export PE_TPSL_DEFAULT_GENCOMPILERS_GNU_sandybridge;PE_TPSL_DEFAULT_GENCOMPILERS_GNU_x86_64=5.1\ 4.9 ;export PE_TPSL_DEFAULT_GENCOMPILERS_GNU_x86_64;PE_TPSL_DEFAULT_GENCOMPILERS_INTEL_haswell=15.0 ;export PE_TPSL_DEFAULT_GENCOMPILERS_INTEL_haswell;PE_TPSL_DEFAULT_GENCOMPILERS_INTEL_interlagos=15.0 ;export PE_TPSL_DEFAULT_GENCOMPILERS_INTEL_interlagos;PE_TPSL_DEFAULT_GENCOMPILERS_INTEL_sandybridge=15.0 ;export PE_TPSL_DEFAULT_GENCOMPILERS_INTEL_sandybridge;PE_TPSL_DEFAULT_GENCOMPILERS_INTEL_x86_64=15.0 ;export PE_TPSL_DEFAULT_GENCOMPILERS_INTEL_x86_64;PE_TPSL_DEFAULT_GENCOMPILERS_PGI_interlagos=15.3 ;export PE_TPSL_DEFAULT_GENCOMPILERS_PGI_interlagos;PE_TPSL_DEFAULT_GENCOMPILERS_PGI_x86_64=15.3 ;export PE_TPSL_DEFAULT_GENCOMPILERS_PGI_x86_64;PE_TPSL_DEFAULT_GENCOMPS_CRAY_haswell=83 ;export PE_TPSL_DEFAULT_GENCOMPS_CRAY_haswell;PE_TPSL_DEFAULT_GENCOMPS_CRAY_interlagos=83 ;export PE_TPSL_DEFAULT_GENCOMPS_CRAY_interlagos;PE_TPSL_DEFAULT_GENCOMPS_CRAY_mic_knl=85 ;export PE_TPSL_DEFAULT_GENCOMPS_CRAY_mic_knl;PE_TPSL_DEFAULT_GENCOMPS_CRAY_sandybridge=83 ;export PE_TPSL_DEFAULT_GENCOMPS_CRAY_sandybridge;PE_TPSL_DEFAULT_GENCOMPS_CRAY_x86_64=83 ;export PE_TPSL_DEFAULT_GENCOMPS_CRAY_x86_64;PE_TPSL_DEFAULT_GENCOMPS_GNU_haswell=51\ 49 ;export PE_TPSL_DEFAULT_GENCOMPS_GNU_haswell;PE_TPSL_DEFAULT_GENCOMPS_GNU_interlagos=51\ 49 ;export PE_TPSL_DEFAULT_GENCOMPS_GNU_interlagos;PE_TPSL_DEFAULT_GENCOMPS_GNU_sandybridge=51\ 49 ;export PE_TPSL_DEFAULT_GENCOMPS_GNU_sandybridge;PE_TPSL_DEFAULT_GENCOMPS_GNU_x86_64=51\ 49 ;export PE_TPSL_DEFAULT_GENCOMPS_GNU_x86_64;PE_TPSL_DEFAULT_GENCOMPS_INTEL_haswell=150 ;export PE_TPSL_DEFAULT_GENCOMPS_INTEL_haswell;PE_TPSL_DEFAULT_GENCOMPS_INTEL_interlagos=150 ;export PE_TPSL_DEFAULT_GENCOMPS_INTEL_interlagos;PE_TPSL_DEFAULT_GENCOMPS_INTEL_sandybridge=150 ;export PE_TPSL_DEFAULT_GENCOMPS_INTEL_sandybridge;PE_TPSL_DEFAULT_GENCOMPS_INTEL_x86_64=150 ;export PE_TPSL_DEFAULT_GENCOMPS_INTEL_x86_64;PE_TPSL_DEFAULT_GENCOMPS_PGI_interlagos=153 ;export PE_TPSL_DEFAULT_GENCOMPS_PGI_interlagos;PE_TPSL_DEFAULT_GENCOMPS_PGI_x86_64=153 ;export PE_TPSL_DEFAULT_GENCOMPS_PGI_x86_64;PE_TPSL_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH:PE_LIBSCI ;export PE_TPSL_DEFAULT_REQUIRED_PRODUCTS;PE_TPSL_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/tpsl/16.03.1/@PRGENV@/@PE_TPSL_DEFAULT_GENCOMPS@/@PE_TPSL_DEFAULT_TARGET@/lib/pkgconfig ;export PE_TPSL_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_TPSL_DEFAULT_VOLATILE_PRGENV=CRAY\ CRAY64\ GNU\ GNU64\ INTEL\ INTEL64\ PGI\ PGI64 ;export PE_TPSL_DEFAULT_VOLATILE_PRGENV;PE_TRILINOS_DEFAULT_GENCOMPILERS_CRAY_x86_64=8.4 ;export PE_TRILINOS_DEFAULT_GENCOMPILERS_CRAY_x86_64;PE_TRILINOS_DEFAULT_GENCOMPILERS_GNU_x86_64=5.1\ 4.9 ;export PE_TRILINOS_DEFAULT_GENCOMPILERS_GNU_x86_64;PE_TRILINOS_DEFAULT_GENCOMPILERS_INTEL_x86_64=15.0 ;export PE_TRILINOS_DEFAULT_GENCOMPILERS_INTEL_x86_64;PE_TRILINOS_DEFAULT_GENCOMPS_CRAY_x86_64=84 ;export PE_TRILINOS_DEFAULT_GENCOMPS_CRAY_x86_64;PE_TRILINOS_DEFAULT_GENCOMPS_GNU_x86_64=51\ 49 ;export PE_TRILINOS_DEFAULT_GENCOMPS_GNU_x86_64;PE_TRILINOS_DEFAULT_GENCOMPS_INTEL_x86_64=150 ;export PE_TRILINOS_ module load cudatoolkit
~/bin/myenv.sh

# for topomapping run:
# module load topomapping
# cp ~/awp-odc-spp/Makefile.bluewaters.gnu.topomapping Makefile.bluewaters.gnu
 make -f Makefile.bluewaters.gnu

cd ~/awp-odc-spp/awp-odc-os/run

cat run_small.pbs | sed -e "s/aprun/time aprun/g" > runs.pbs
cp runs.pbs run_small.pbs
cat -n run_small.pbs

rm awp-odc_gpu_small.*

MYJOBID=`qsub run_small.pbs`
qsub run_small.pbs
Job submitted to account: jnc
echo MYJOBID=$MYJOBID
~/bin//wait_for.sh $MYJOBID
DEFAULT_GENCOMPS_INTEL_x86_64;PE_TRILINOS_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH:PE_HDF5_PARALLEL:PE_NETCDF_HDF5PARALLEL:PE_LIBSCI:PE_TPSL ;export PE_TRILINOS_DEFAULT_REQUIRED_PRODUCTS;PE_TRILINOS_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/trilinos/12.2.1.0/@PRGENV@/@PE_TRILINOS_DEFAULT_GENCOMPS@/@PE_TRILINOS_DEFAULT_TARGET@/lib/pkgconfig ;export PE_TRILINOS_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_TRILINOS_DEFAULT_VOLATILE_PRGENV=CRAY\ GNU\ INTEL ;export PE_TRILINOS_DEFAULT_VOLATILE_PRGENV;PKGCONFIG_ENABLED=1 ;export PKGCONFIG_ENABLED;PKG_CONFIG_PATH=/opt/cray/rca/1.0.0-2.0502.60530.1.63.gem/lib64/pkgconfig:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/lib64/pkgconfig:/opt/cray/xpmem/0.1-2.0502.64982.5.3.gem/lib64/pkgconfig:/opt/cray/gni-headers/4.0-1.0502.10859.7.8.gem/lib64/pkgconfig:/opt/cray/dmapp/7.0.1-1.0502.11080.8.74.gem/lib64/pkgconfig:/opt/cray/pmi/5.0.10-1.0000.11050.179.3.gem/lib64/pkgconfig:/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/lib64/pkgconfig:/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/lib64/pkgconfig:/opt/cray/craype/2.5.4/pkg-config:/opt/cray/iobuf/2.0.6/lib/pkgconfig:/opt/cray/atp/2.0.1/lib/pkgconfig ;export PKG_CONFIG_PATH;PKG_CONFIG_PATH_DEFAULT=/opt/cray/papi/5.4.3.1/lib64/pkgconfig ;export PKG_CONFIG_PATH_DEFAULT;SHMEM_ABORT_ON_ERROR=1 ;export SHMEM_ABORT_ON_ERROR;XTOS_VERSION=5.2.82 ;export XTOS_VERSION;XTPE_NETWORK_TARGET=gemini ;export XTPE_NETWORK_TARGET;_LMFILES_=/opt/modulefiles/modules/3.2.10.4:/opt/modulefiles/eswrap/1.3.3-1.020200.1278.0:/opt/modulefiles/gcc/4.9.3:/opt/cray/craype/2.5.4/modulefiles/craype-network-gemini:/opt/cray/craype/2.5.4/modulefiles/craype-interlagos:/opt/cray/modulefiles/craype/2.5.4:/opt/cray/modulefiles/cray-mpich/7.3.3:/opt/modulefiles/torque/6.0.1:/opt/modulefiles/moab/9.0.2.TAS2-1477409647_6706986-sles11:/opt/modulefiles/java/jdk1.8.0_51:/usr/local/modulefiles/globus/5.2.5:/usr/local/modulefiles/gsissh/6.2p2:/sw/xe/modulefiles/scripts:/sw/xe/modulefiles/user-paths:/sw/bw/modulefiles/xalt/0.7.6:/opt/cray/modulefiles/cray-libsci/16.03.1:/opt/cray/gem/modulefiles/udreg/2.3.2-1.0502.10518.2.17.gem:/opt/cray/gem/modulefiles/ugni/6.0-1.0502.10863.8.28.gem:/opt/cray/gem/modulefiles/pmi/5.0.10-1.0000.11050.179.3.gem:/opt/cray/gem/modulefiles/dmapp/7.0.1-1.0502.11080.8.74.gem:/opt/cray/gem/modulefiles/gni-headers/4.0-1.0502.10859.7.8.gem:/opt/cray/gem/modulefiles/xpmem/0.1-2.0502.64982.5.3.gem:/opt/cray/gem/modulefiles/dvs/2.5_0.9.0-1.0502.2188.1.113.gem:/opt/cray/gem/modulefiles/alps/5.2.4-2.0502.9774.31.12.gem:/opt/cray/gem/modulefiles/rca/1.0.0-2.0502.60530.1.63.gem:/opt/cray/modulefiles/atp/2.0.1:/opt/cray/modulefiles/PrgEnv-gnu/5.2.82 ;export _LMFILES_;unset ASSEMBLER_X86_64;unset CC_X86_64;unset CRAYLIBS_X86_64;unset CRAYLMD_LICENSE_FILE;unset CRAY_BINUTILS_BIN;unset CRAY_BINUTILS_ROOT;unset CRAY_BINUTILS_VERSION;unset CRAY_CC_VERSION;unset CRAY_CXX_IPA_LIBS;unset CRAY_FTN_VERSION;unset CRAY_PE_TARGET;unset CRAY_PRGENVCRAY;unset FORTRAN_SYSTEM_MODULE_NAMES;unset FTN_X86_64;unset GCC_X86_64;unset INCLUDE_PATH_X86_64;unset LINKER_X86_64;unset NLSPATH;unset PE_LEVEL;
${MODULESHOME}/bin/modulecmd bash $*
CRAYPE_LINK_TYPE=dynamic ;export CRAYPE_LINK_TYPE;CRAY_CUDATOOLKIT_DIR=/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1 ;export CRAY_CUDATOOLKIT_DIR;CRAY_CUDATOOLKIT_INCLUDE_OPTS=-I/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1/include\ -I/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1/extras/CUPTI/include\ -I/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1/extras/Debugger/include ;export CRAY_CUDATOOLKIT_INCLUDE_OPTS;CRAY_CUDATOOLKIT_POST_LINK_OPTS=-L/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1/lib64\ -L/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1/extras/CUPTI/lib64\ -Wl,--as-needed\ -Wl,-lcupti\ -Wl,-lcudart\ -Wl,--no-as-needed\ -L/opt/cray/nvidia/default/lib64\ -lcuda ;export CRAY_CUDATOOLKIT_POST_LINK_OPTS;CRAY_CUDATOOLKIT_VERSION=7.5.18-1.0502.10743.2.1 ;export CRAY_CUDATOOLKIT_VERSION;CRAY_LD_LIBRARY_PATH=/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1/lib64:/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1/extras/CUPTI/lib64:/opt/cray/rca/1.0.0-2.0502.60530.1.63.gem/lib64:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/lib64:/opt/cray/xpmem/0.1-2.0502.64982.5.3.gem/lib64:/opt/cray/dmapp/7.0.1-1.0502.11080.8.74.gem/lib64:/opt/cray/pmi/5.0.10-1.0000.11050.179.3.gem/lib64:/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/lib64:/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/lib64:/opt/cray/libsci/16.03.1/GNU/4.9/x86_64/lib:/opt/cray/mpt/7.3.3/gni/mpich-gnu/4.9/lib ;export CRAY_LD_LIBRARY_PATH;CUDATOOLKIT_HOME=/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1 ;export CUDATOOLKIT_HOME;LOADEDMODULES=modules/3.2.10.4:eswrap/1.3.3-1.020200.1278.0:gcc/4.9.3:craype-network-gemini:craype-interlagos:craype/2.5.4:cray-mpich/7.3.3:torque/6.0.1:moab/9.0.2.TAS2-1477409647_6706986-sles11:java/jdk1.8.0_51:globus/5.2.5:gsissh/6.2p2:scripts:user-paths:xalt/0.7.6:cray-libsci/16.03.1:udreg/2.3.2-1.0502.10518.2.17.gem:ugni/6.0-1.0502.10863.8.28.gem:pmi/5.0.10-1.0000.11050.179.3.gem:dmapp/7.0.1-1.0502.11080.8.74.gem:gni-headers/4.0-1.0502.10859.7.8.gem:xpmem/0.1-2.0502.64982.5.3.gem:dvs/2.5_0.9.0-1.0502.2188.1.113.gem:alps/5.2.4-2.0502.9774.31.12.gem:rca/1.0.0-2.0502.60530.1.63.gem:atp/2.0.1:PrgEnv-gnu/5.2.82:cudatoolkit/7.5.18-1.0502.10743.2.1 ;export LOADEDMODULES;MANPATH=/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1/doc/man:/opt/cray/atp/2.0.1/man:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/man:/opt/cray/libsci/16.03.1/man:/opt/cray/man/csmlversion:/opt/gcc/4.9.3/snos/share/man:/sw/user/man:/opt/java/jdk1.8.0_51/man:/usr/local/globus-5.2.5/man:/opt/moab/9.0.2t2/share/man/:/opt/torque/6.0.1-end/share/man:/opt/cray/mpt/7.3.3/gni/man/mpich:/opt/cray/craype/2.5.4/man:/opt/cray/eslogin/eswrap/1.3.3-1.020200.1278.0/man:/opt/modules/3.2.10.4/share/man:/opt/moab/9.0.2t2/share/man::ignore:/usr/local/man:/usr/share/man:/usr/man:/opt/cray/share/man:/cm/local/apps/environment-modules/3.2.6/man:/opt/cray/man:/u/system/bwjenkins/man ;export MANPATH;PATH=/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1/bin:/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1/libnvvp:/opt/cray/rca/1.0.0-2.0502.60530.1.63.gem/bin:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/sbin:/opt/cray/dvs/2.5_0.9.0-1.0502.2188.1.113.gem/bin:/opt/cray/xpmem/0.1-2.0502.64982.5.3.gem/bin:/opt/cray/pmi/5.0.10-1.0000.11050.179.3.gem/bin:/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/bin:/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/bin:/opt/gcc/4.9.3/bin:/sw/xe_xk_cle5.2UP02_pe2.3.0/xalt/0.7.6/sles11.3/libexec:/sw/xe_xk_cle5.2UP02_pe2.3.0/xalt/0.7.6/sles11.3/bin:/sw/admin/scripts:/sw/user/scripts:/usr/local/gsi-openssh-6.2p2-2/bin:/opt/java/jdk1.8.0_51/bin:/usr/local/globus-5.2.5/bin:/usr/local/globus-5.2.5/sbin:/opt/moab/9.0.2t2/sbin:/opt/torque/6.0.1-end/sbin:/opt/torque/6.0.1-end/bin:/opt/cray/mpt/7.3.3/gni/bin:/opt/cray/craype/2.5.4/bin:/opt/cray/eslogin/eswrap/1.3.3-1.020200.1278.0/bin:/opt/modules/3.2.10.4/bin:/opt/moab/9.0.2t2/bin:/u/system/bwjenkins/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin ;export PATH;PE_PKGCONFIG_LIBS=cray-cudatoolkit:AtpSigHandler:cray-rca:libsci_mpi:libsci:mpich ;export PE_PKGCONFIG_LIBS;PKG_CONFIG_PATH=/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1/lib64/pkgconfig:/opt/cray/rca/1.0.0-2.0502.60530.1.63.gem/lib64/pkgconfig:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/lib64/pkgconfig:/opt/cray/xpmem/0.1-2.0502.64982.5.3.gem/lib64/pkgconfig:/opt/cray/gni-headers/4.0-1.0502.10859.7.8.gem/lib64/pkgconfig:/opt/cray/dmapp/7.0.1-1.0502.11080.8.74.gem/lib64/pkgconfig:/opt/cray/pmi/5.0.10-1.0000.11050.179.3.gem/lib64/pkgconfig:/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/lib64/pkgconfig:/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/lib64/pkgconfig:/opt/cray/craype/2.5.4/pkg-config:/opt/cray/iobuf/2.0.6/lib/pkgconfig:/opt/cray/atp/2.0.1/lib/pkgconfig ;export PKG_CONFIG_PATH;XTPE_LINK_TYPE=dynamic ;export XTPE_LINK_TYPE;_LMFILES_=/opt/modulefiles/modules/3.2.10.4:/opt/modulefiles/eswrap/1.3.3-1.020200.1278.0:/opt/modulefiles/gcc/4.9.3:/opt/cray/craype/2.5.4/modulefiles/craype-network-gemini:/opt/cray/craype/2.5.4/modulefiles/craype-interlagos:/opt/cray/modulefiles/craype/2.5.4:/opt/cray/modulefiles/cray-mpich/7.3.3:/opt/modulefiles/torque/6.0.1:/opt/modulefiles/moab/9.0.2.TAS2-1477409647_6706986-sles11:/opt/modulefiles/java/jdk1.8.0_51:/usr/local/modulefiles/globus/5.2.5:/usr/local/modulefiles/gsissh/6.2p2:/sw/xe/modulefiles/scripts:/sw/xe/modulefiles/user-paths:/sw/bw/modulefiles/xalt/0.7.6:/opt/cray/modulefiles/cray-libsci/16.03.1:/opt/cray/gem/modulefiles/udreg/2.3.2-1.0502.10518.2.17.gem:/opt/cray/gem/modulefiles/ugni/6.0-1.0502.10863.8.28.gem:/opt/cray/gem/modulefiles/pmi/5.0.10-1.0000.11050.179.3.gem:/opt/cray/gem/modulefiles/dmapp/7.0.1-1.0502.11080.8.74.gem:/opt/cray/gem/modulefiles/gni-headers/4.0-1.0502.10859.7.8.gem:/opt/cray/gem/modulefiles/xpmem/0.1-2.0502.64982.5.3.gem:/opt/cray/gem/modulefiles/dvs/2.5_0.9.0-1.0502.2188.1.113.gem:/opt/cray/gem/modulefiles/alps/5.2.4-2.0502.9774.31.12.gem:/opt/cray/gem/modulefiles/rca/1.0.0-2.0502.60530.1.63.gem:/opt/cray/modulefiles/atp/2.0.1:/opt/cray/modulefiles/PrgEnv-gnu/5.2.82:/opt/cray/modulefiles/cudatoolkit/7.5.18-1.0502.10743.2.1 ;export _LMFILES_;
=== RUNNING /u/system/bwjenkins/bin/myenv.sh ===
HOST=h2ologin2
HOME=/u/system/bwjenkins
SCRATCH=/u/system/bwjenkins/scratch
     ls SCRATCH:
mdtest
rmg-spp
rmg-spp-topo
setsm_benchmark
SPP-PPM
stripe8
sw_config
SWTools-BWCI
sw_workdir
tests
     my id:
uid=49294(bwjenkins) gid=16228(BW_System) groups=16228(BW_System),999(login_user),1000(ie_user),1001(hpss_user)
Currently Loaded Modulefiles:
  1) modules/3.2.10.4
  2) eswrap/1.3.3-1.020200.1278.0
  3) gcc/4.9.3
  4) craype-network-gemini
  5) craype-interlagos
  6) craype/2.5.4
  7) cray-mpich/7.3.3
  8) torque/6.0.1
  9) moab/9.0.2.TAS2-1477409647_6706986-sles11
 10) java/jdk1.8.0_51
 11) globus/5.2.5
 12) gsissh/6.2p2
 13) scripts
 14) user-paths
 15) xalt/0.7.6
 16) cray-libsci/16.03.1
 17) udreg/2.3.2-1.0502.10518.2.17.gem
 18) ugni/6.0-1.0502.10863.8.28.gem
 19) pmi/5.0.10-1.0000.11050.179.3.gem
 20) dmapp/7.0.1-1.0502.11080.8.74.gem
 21) gni-headers/4.0-1.0502.10859.7.8.gem
 22) xpmem/0.1-2.0502.64982.5.3.gem
 23) dvs/2.5_0.9.0-1.0502.2188.1.113.gem
 24) alps/5.2.4-2.0502.9774.31.12.gem
 25) rca/1.0.0-2.0502.60530.1.63.gem
 26) atp/2.0.1
 27) PrgEnv-gnu/5.2.82
 28) cudatoolkit/7.5.18-1.0502.10743.2.1
make: `pmcl3d' is up to date.
     1	#!/bin/bash -login
     2	#PBS -N awp-odc_gpu_small
     3	#PBS -l walltime=00:30:00,nodes=4:ppn=1:xk
     4	#PBS -j eo
     5	
     6	module unload darshan
     7	module swap $( module list | grep -o PrgEnv-.*$ ) PrgEnv-gnu
     8	module load cudatoolkit
     9	
    10	mkdir $PBS_O_WORKDIR/$PBS_JOBID
    11	cd $PBS_O_WORKDIR/$PBS_JOBID
    12	
    13	ln -fs ../../src/pmcl3d
    14	mkdir input
    15	mkdir input_rst
    16	mkdir output_ckp
    17	mkdir output_sfc
    18	cp ../../input/source input/.
    19	lfs setstripe -s 600m -c 4 input/mesh
    20	cp ../../input/mesh input/.
    21	lfs setstripe -s 3m -c 4 output_sfc
    22	lfs setstripe -s 600m -c 1 input_rst
    23	
    24	start_time=$(date +%s)
    25	
    26	time aprun -n $PBS_NP -N1 ./pmcl3d \
    27	 -X 320 -Y 320 -Z 2048 -x 2 -y 2 \
    28	 --TMAX 4.0   --DH 40.0 --DT 0.002 \
    29	 --NSRC 1 --NST 101  \
    30	--IFAULT 1 --MEDIASTART 2 --READ_STEP 101 --WRITE_STEP 100 \
    31	--NSKPX 2 --NSKPY 2   --NTISKP 20   -c output_ckp/ckp   -o output_sfc   --INSRC input/source   --INVEL input/mesh
    32	
    33	end_time=$(date +%s)
    34	echo Completed in $(( end_time - start_time )) seconds
    35	
MYJOBID=5933208.bw
=== RUNNING /u/system/bwjenkins/bin//wait_for.sh ===
Fri Dec 23 09:07:18 CST 2016: waiting for 5933208.bw to finish
Fri Dec 23 09:12:18 CST 2016: waiting for 5933208.bw to finish
Fri Dec 23 09:17:18 CST 2016: waiting for 5933208.bw to finish
Fri Dec 23 09:22:18 CST 2016: waiting for 5933208.bw to finish
qstat: Unknown Job Id Error 5933208.bw

tail -50 awp-odc_gpu_small.e* > ~/awp-odc-spp/stderr.txt
MYREALMIN=`grep real awp-odc_gpu_small.e* | cut -dl -f2 | cut -dm -f1`
grep real awp-odc_gpu_small.e* | cut -dl -f2 | cut -dm -f1
echo "YVALUE=$MYREALMIN" > ~/awp-odc-spp/awp.realtime
cat -n ~/awp-odc-spp/awp.realtime
5933208.bw finished
     1	YVALUE=	6
[SSH] exit-status: 0
[awp-odc-spp] $ /bin/sh -xe /tmp/hudson6375270894070246299.sh
+ scp 'BW_REMOTE:~/awp-odc-spp/*.txt' /var/lib/jenkins/workspace/awp-odc-spp

Access by OTP or Two Factor Certificate Authority only.
Use myproxy-logon -s tfca.ncsa.illinois.edu -p 7512 for gsissh access.
gsissh or ssh -o PreferredAuthentications=keyboard-interactive for otp access.

Blue Water Admin Team

+ scp 'BW_REMOTE:~/awp-odc-spp/awp.realtime' /var/lib/jenkins/workspace/awp-odc-spp

Access by OTP or Two Factor Certificate Authority only.
Use myproxy-logon -s tfca.ncsa.illinois.edu -p 7512 for gsissh access.
gsissh or ssh -o PreferredAuthentications=keyboard-interactive for otp access.

Blue Water Admin Team

Recording plot data
Saving plot series data from: /var/lib/jenkins/workspace/awp-odc-spp/awp.realtime
Finished: SUCCESS