![[Jenkins]](/static/755cd4af/images/svgs/logo.svg){kind=logo}
Started by user Greg Bauer Building on master in workspace /var/lib/jenkins/workspace/awp-odc-spp executing script: set -v #rm -rf awp-odc-spp mkdir awp-odc-spp cd awp-odc-spp tar xvfz /projects/bluewaters/benchmarking/SPP/AWP-ODC/awp-odc_gpu.tgz cd awp-odc-os/src/ module unload darshan module swap $( module list | grep -o PrgEnv-.*$ ) PrgEnv-gnu module load cudatoolkit ~/bin/myenv.sh # for topomapping run: # module load topomapping # cp ~/awp-odc-spp/Makefile.bluewaters.gnu.topomapping Makefile.bluewaters.gnu make -f Makefile.bluewaters.gnu cd ~/awp-odc-spp/awp-odc-os/run cat run_small.pbs | sed -e "s/aprun/time aprun/g" > runs.pbs cp runs.pbs run_small.pbs cat -n run_small.pbs rm awp-odc_gpu_small.* MYJOBID=`qsub run_small.pbs` echo MYJOBID=$MYJOBID ~/bin//wait_for.sh $MYJOBID tail -50 awp-odc_gpu_small.e* > ~/awp-odc-spp/stderr.txt MYREALMIN=`grep real awp-odc_gpu_small.e* | cut -dl -f2 | cut -dm -f1` echo "YVALUE=$MYREALMIN" > ~/awp-odc-spp/awp.realtime cat -n ~/awp-odc-spp/awp.realtime #rm -rf awp-odc-spp mkdir awp-odc-spp mkdir: cannot create directory `awp-odc-spp': File exists cd awp-odc-spp tar xvfz /projects/bluewaters/benchmarking/SPP/AWP-ODC/awp-odc_gpu.tgz awp-odc-os/ awp-odc-os/input/ awp-odc-os/input/source awp-odc-os/input/mesh cd awp-odc-os/src/ module unload darshan module swap $( module list | grep -o PrgEnv-.*$ ) PrgEnv-gnu module list | grep -o PrgEnv-.*$ awp-odc-os/AUTHORS awp-odc-os/run/ awp-odc-os/run/run.bluewaters.gnu.pbs awp-odc-os/run/run_medium.pbs awp-odc-os/run/env_clean.sh awp-odc-os/run/env.sh awp-odc-os/run/run_small.pbs awp-odc-os/README.md awp-odc-os/LICENSE awp-odc-os/submodules/ awp-odc-os/submodules/wiki/ awp-odc-os/src/ awp-odc-os/src/cerjan.c awp-odc-os/src/kernel.cu awp-odc-os/src/Makefile.bluewaters.cray awp-odc-os/src/Makefile.bluewaters.gnu awp-odc-os/src/grid.c awp-odc-os/src/kernel.h awp-odc-os/src/pmcl3d.c awp-odc-os/src/command.c awp-odc-os/src/io.c awp-odc-os/src/swap.c awp-odc-os/src/mesh.c awp-odc-os/src/pmcl3d.h awp-odc-os/src/source.c awp-odc-os/src/pmcl3d_cons.h awp-odc-os/CHANGELOG ${MODULESHOME}/bin/modulecmd bash $* LD_LIBRARY_PATH=/usr/local/globus-5.2.5/lib64:/usr/local/globus/lib64 ;export LD_LIBRARY_PATH;LOADEDMODULES=modules/3.2.10.4:eswrap/1.3.3-1.020200.1280.0:cce/8.4.6:craype-network-gemini:craype/2.5.8:cray-libsci/16.11.1:udreg/2.3.2-1.0502.10518.2.17.gem:ugni/6.0-1.0502.10863.8.28.gem:pmi/5.0.10-1.0000.11050.179.3.gem:dmapp/7.0.1-1.0502.11080.8.74.gem:gni-headers/4.0-1.0502.10859.7.8.gem:xpmem/0.1-2.0502.64982.5.3.gem:dvs/2.5_0.9.0-1.0502.2188.1.113.gem:alps/5.2.4-2.0502.9774.31.12.gem:rca/1.0.0-2.0502.60530.1.63.gem:atp/2.0.4:PrgEnv-cray/5.2.82:cray-mpich/7.5.0:craype-interlagos:torque/6.0.4:moab/9.0.2.TAS7-1490665589_71674cf-sles11:java/jdk1.8.0_51:globus/5.2.5:gsissh/6.2p2:xalt/0.7.6.local:scripts:user-paths ;export LOADEDMODULES;MANPATH=/sw/user/man:/opt/java/jdk1.8.0_51/man:/usr/local/globus-5.2.5/man:/opt/moab/9.0.2t7/share/man/:/opt/torque/6.0.4/share/man:/opt/cray/mpt/7.5.0/gni/man/mpich:/opt/cray/atp/2.0.4/man:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/man:/opt/cray/libsci/16.11.1/man:/opt/cray/man/csmlversion:/opt/cray/craype/2.5.8/man:/opt/cray/cce/8.4.6/man:/opt/cray/cce/8.4.6/craylibs/man:/opt/cray/cce/8.4.6/CC/man:/opt/cray/cce/8.4.6/cftn/man:/opt/cray/eslogin/eswrap/1.3.3-1.020200.1280.0/man:/opt/modules/3.2.10.4/share/man:/opt/moab/9.0.2t7/share/man::ignore:/usr/local/man:/usr/share/man:/usr/man:/opt/cray/share/man:/cm/local/apps/environment-modules/3.2.6/man:/opt/cray/man:/u/system/bwjenkins/man ;export MANPATH;PATH=/sw/admin/scripts:/sw/user/scripts:/opt/xalt/0.7.6/sles11.3/libexec:/opt/xalt/0.7.6/sles11.3/bin:/usr/local/gsi-openssh-6.2p2-2/bin:/opt/java/jdk1.8.0_51/bin:/usr/local/globus-5.2.5/bin:/usr/local/globus-5.2.5/sbin:/opt/moab/9.0.2t7/sbin:/opt/torque/6.0.4/sbin:/opt/torque/6.0.4/bin:/opt/cray/mpt/7.5.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.60530.1.63.gem/bin:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/sbin:/opt/cray/dvs/2.5_0.9.0-1.0502.2188.1.113.gem/bin:/opt/cray/xpmem/0.1-2.0502.64982.5.3.gem/bin:/opt/cray/pmi/5.0.10-1.0000.11050.179.3.gem/bin:/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/bin:/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/bin:/opt/cray/craype/2.5.8/bin:/opt/cray/cce/8.4.6/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.4.6/craylibs/x86-64/bin:/opt/cray/cce/8.4.6/cftn/bin:/opt/cray/cce/8.4.6/CC/bin:/opt/cray/eslogin/eswrap/1.3.3-1.020200.1280.0/bin:/opt/modules/3.2.10.4/bin:/opt/moab/9.0.2t7/bin:/u/system/bwjenkins/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin ;export PATH;PE_PKGCONFIG_LIBS=mpich:AtpSigHandler:cray-rca:libsci_mpi:libsci ;export PE_PKGCONFIG_LIBS;PKG_CONFIG_PATH=/opt/cray/rca/1.0.0-2.0502.60530.1.63.gem/lib64/pkgconfig:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/lib64/pkgconfig:/opt/cray/xpmem/0.1-2.0502.64982.5.3.gem/lib64/pkgconfig:/opt/cray/gni-headers/4.0-1.0502.10859.7.8.gem/lib64/pkgconfig:/opt/cray/dmapp/7.0.1-1.0502.11080.8.74.gem/lib64/pkgconfig:/opt/cray/pmi/5.0.10-1.0000.11050.179.3.gem/lib64/pkgconfig:/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/lib64/pkgconfig:/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/lib64/pkgconfig:/opt/cray/craype/2.5.8/pkg-config:/opt/cray/iobuf/2.0.7/lib/pkgconfig:/opt/cray/atp/2.0.4/lib/pkgconfig ;export PKG_CONFIG_PATH;_LMFILES_=/opt/modulefiles/modules/3.2.10.4:/opt/modulefiles/eswrap/1.3.3-1.020200.1280.0:/opt/modulefiles/cce/8.4.6:/opt/cray/craype/2.5.8/modulefiles/craype-network-gemini:/opt/cray/modulefiles/craype/2.5.8:/opt/cray/modulefiles/cray-libsci/16.11.1:/opt/cray/gem/modulefiles/udreg/2.3.2-1.0502.10518.2.17.gem:/opt/cray/gem/modulefiles/ugni/6.0-1.0502.10863.8.28.gem:/opt/cray/gem/modulefiles/pmi/5.0.10-1.0000.11050.179.3.gem:/opt/cray/gem/modulefiles/dmapp/7.0.1-1.0502.11080.8.74.gem:/opt/cray/gem/modulefiles/gni-headers/4.0-1.0502.10859.7.8.gem:/opt/cray/gem/modulefiles/xpmem/0.1-2.0502.64982.5.3.gem:/opt/cray/gem/modulefiles/dvs/2.5_0.9.0-1.0502.2188.1.113.gem:/opt/cray/gem/modulefiles/alps/5.2.4-2.0502.9774.31.12.gem:/opt/cray/gem/modulefiles/rca/1.0.0-2.0502.60530.1.63.gem:/opt/cray/modulefiles/atp/2.0.4:/opt/cray/modulefiles/PrgEnv-cray/5.2.82:/opt/cray/modulefiles/cray-mpich/7.5.0:/opt/cray/craype/2.5.8/modulefiles/craype-interlagos:/opt/modulefiles/torque/6.0.4:/opt/modulefiles/moab/9.0.2.TAS7-1490665589_71674cf-sles11:/opt/modulefiles/java/jdk1.8.0_51:/usr/local/modulefiles/globus/5.2.5:/usr/local/modulefiles/gsissh/6.2p2:/opt/modulefiles/xalt/0.7.6.local:/sw/xe/modulefiles/scripts:/sw/xe/modulefiles/user-paths ;export _LMFILES_;unset DARSHAN_LOGPATH;unset EBDEVELGNUPLOT;unset EBROOTGNUPLOT;unset EBVERSIONGNUPLOT; ${MODULESHOME}/bin/modulecmd bash $* ATP_HOME=/opt/cray/atp/2.0.4 ;export ATP_HOME;ATP_IGNORE_SIGTERM=1 ;export ATP_IGNORE_SIGTERM;ATP_MRNET_COMM_PATH=/opt/cray/atp/2.0.4/libexec/atp_mrnet_commnode_wrapper ;export ATP_MRNET_COMM_PATH;ATP_POST_LINK_OPTS=-Wl,-L/opt/cray/atp/2.0.4/libApp/\ ;export ATP_POST_LINK_OPTS;CRAYOS_VERSION=5.2.82 ;export CRAYOS_VERSION;CRAYPE_DIR=/opt/cray/craype/2.5.8 ;export CRAYPE_DIR;CRAYPE_NETWORK_TARGET=gemini ;export CRAYPE_NETWORK_TARGET;CRAYPE_VERSION=2.5.8 ;export CRAYPE_VERSION;CRAY_ALPS_INCLUDE_OPTS=-I/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/include ;export CRAY_ALPS_INCLUDE_OPTS;CRAY_ALPS_POST_LINK_OPTS=-L/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/lib64 ;export CRAY_ALPS_POST_LINK_OPTS;CRAY_CPU_TARGET=interlagos ;export CRAY_CPU_TARGET;CRAY_DMAPP_INCLUDE_OPTS=-I/opt/cray/dmapp/7.0.1-1.0502.11080.8.74.gem/include\ -I/opt/cray/gni-headers/4.0-1.0502.10859.7.8.gem/include\ ;export CRAY_DMAPP_INCLUDE_OPTS;CRAY_DMAPP_POST_LINK_OPTS=-L/opt/cray/dmapp/7.0.1-1.0502.11080.8.74.gem/lib64 ;export CRAY_DMAPP_POST_LINK_OPTS;CRAY_GNI_HEADERS_INCLUDE_OPTS=-I/opt/cray/gni-headers/4.0-1.0502.10859.7.8.gem/include ;export CRAY_GNI_HEADERS_INCLUDE_OPTS;CRAY_LD_LIBRARY_PATH=/opt/cray/rca/1.0.0-2.0502.60530.1.63.gem/lib64:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/lib64:/opt/cray/xpmem/0.1-2.0502.64982.5.3.gem/lib64:/opt/cray/dmapp/7.0.1-1.0502.11080.8.74.gem/lib64:/opt/cray/pmi/5.0.10-1.0000.11050.179.3.gem/lib64:/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/lib64:/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/lib64:/opt/cray/libsci/16.11.1/GNU/4.9/x86_64/lib:/opt/cray/mpt/7.5.0/gni/mpich-gnu/4.9/lib ;export CRAY_LD_LIBRARY_PATH;CRAY_LIBSCI_BASE_DIR=/opt/cray/libsci/16.11.1 ;export CRAY_LIBSCI_BASE_DIR;CRAY_LIBSCI_DIR=/opt/cray/libsci/16.11.1 ;export CRAY_LIBSCI_DIR;CRAY_LIBSCI_PREFIX_DIR=/opt/cray/libsci/16.11.1/GNU/4.9/x86_64 ;export CRAY_LIBSCI_PREFIX_DIR;CRAY_LIBSCI_VERSION=16.11.1 ;export CRAY_LIBSCI_VERSION;CRAY_MPICH2_DIR=/opt/cray/mpt/7.5.0/gni/mpich-gnu/4.9 ;export CRAY_MPICH2_DIR;CRAY_MPICH2_VER=7.5.0 ;export CRAY_MPICH2_VER;CRAY_MPICH_BASEDIR=/opt/cray/mpt/7.5.0/gni ;export CRAY_MPICH_BASEDIR;CRAY_MPICH_DIR=/opt/cray/mpt/7.5.0/gni/mpich-gnu/4.9 ;export CRAY_MPICH_DIR;CRAY_MPICH_ROOTDIR=/opt/cray/mpt/7.5.0 ;export CRAY_MPICH_ROOTDIR;CRAY_PMI_INCLUDE_OPTS=-I/opt/cray/pmi/5.0.10-1.0000.11050.179.3.gem/include ;export CRAY_PMI_INCLUDE_OPTS;CRAY_PMI_POST_LINK_OPTS=-L/opt/cray/pmi/5.0.10-1.0000.11050.179.3.gem/lib64 ;export CRAY_PMI_POST_LINK_OPTS;CRAY_PRE_COMPILE_OPTS=-hnetwork=gemini ;export CRAY_PRE_COMPILE_OPTS;CRAY_PRGENVGNU=loaded ;export CRAY_PRGENVGNU;CRAY_RCA_INCLUDE_OPTS=-I/opt/cray/rca/1.0.0-2.0502.60530.1.63.gem/include\ -I/opt/cray/krca/1.0.0-2.0502.63139.4.30.gem/include\ -I/opt/cray-hss-devel/7.2.0/include\ ;export CRAY_RCA_INCLUDE_OPTS;CRAY_RCA_POST_LINK_OPTS=-L/opt/cray/rca/1.0.0-2.0502.60530.1.63.gem/lib64\ -lrca\ ;export CRAY_RCA_POST_LINK_OPTS;CRAY_UDREG_INCLUDE_OPTS=-I/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/include ;export CRAY_UDREG_INCLUDE_OPTS;CRAY_UDREG_POST_LINK_OPTS=-L/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/lib64 ;export CRAY_UDREG_POST_LINK_OPTS;CRAY_UGNI_INCLUDE_OPTS=-I/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/include ;export CRAY_UGNI_INCLUDE_OPTS;CRAY_UGNI_POST_LINK_OPTS=-L/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/lib64 ;export CRAY_UGNI_POST_LINK_OPTS;CRAY_XPMEM_INCLUDE_OPTS=-I/opt/cray/xpmem/0.1-2.0502.64982.5.3.gem/include ;export CRAY_XPMEM_INCLUDE_OPTS;CRAY_XPMEM_POST_LINK_OPTS=-L/opt/cray/xpmem/0.1-2.0502.64982.5.3.gem/lib64 ;export CRAY_XPMEM_POST_LINK_OPTS;DMAPP_ABORT_ON_ERROR=1 ;export DMAPP_ABORT_ON_ERROR;DVS_INCLUDE_OPTS=-I/opt/cray/dvs/2.5_0.9.0-1.0502.2188.1.113.gem/include ;export DVS_INCLUDE_OPTS;DVS_VERSION=0.9.0 ;export DVS_VERSION;GCC_PATH=/opt/gcc/4.9.3 ;export GCC_PATH;GCC_VERSION=4.9.3 ;export GCC_VERSION;GNU_VERSION=4.9.3 ;export GNU_VERSION;INFOPATH=/opt/gcc/4.9.3/snos/share/info:/usr/local/info:/usr/share/info:/usr/info ;export INFOPATH;LD_LIBRARY_PATH=/opt/gcc/4.9.3/snos/lib64:/usr/local/globus-5.2.5/lib64:/usr/local/globus/lib64 ;export LD_LIBRARY_PATH;LIBSCI_BASE_DIR=/opt/cray/libsci/16.11.1 ;export LIBSCI_BASE_DIR;LIBSCI_VERSION=16.11.1 ;export LIBSCI_VERSION;LOADEDMODULES=modules/3.2.10.4:eswrap/1.3.3-1.020200.1280.0:gcc/4.9.3:craype-interlagos:craype-network-gemini:craype/2.5.8:cray-mpich/7.5.0:torque/6.0.4:moab/9.0.2.TAS7-1490665589_71674cf-sles11:java/jdk1.8.0_51:globus/5.2.5:gsissh/6.2p2:xalt/0.7.6.local:scripts:user-paths:cray-libsci/16.11.1:udreg/2.3.2-1.0502.10518.2.17.gem:ugni/6.0-1.0502.10863.8.28.gem:pmi/5.0.10-1.0000.11050.179.3.gem:dmapp/7.0.1-1.0502.11080.8.74.gem:gni-headers/4.0-1.0502.10859.7.8.gem:xpmem/0.1-2.0502.64982.5.3.gem:dvs/2.5_0.9.0-1.0502.2188.1.113.gem:alps/5.2.4-2.0502.9774.31.12.gem:rca/1.0.0-2.0502.60530.1.63.gem:atp/2.0.4:PrgEnv-gnu/5.2.82 ;export LOADEDMODULES;MANPATH=/opt/cray/atp/2.0.4/man:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/man:/opt/cray/libsci/16.11.1/man:/opt/cray/man/csmlversion:/opt/gcc/4.9.3/snos/share/man:/sw/user/man:/opt/java/jdk1.8.0_51/man:/usr/local/globus-5.2.5/man:/opt/moab/9.0.2t7/share/man/:/opt/torque/6.0.4/share/man:/opt/cray/mpt/7.5.0/gni/man/mpich:/opt/cray/craype/2.5.8/man:/opt/cray/eslogin/eswrap/1.3.3-1.020200.1280.0/man:/opt/modules/3.2.10.4/share/man:/opt/moab/9.0.2t7/share/man::ignore:/usr/local/man:/usr/share/man:/usr/man:/opt/cray/share/man:/cm/local/apps/environment-modules/3.2.6/man:/opt/cray/man:/u/system/bwjenkins/man ;export MANPATH;MODULEPATH=/sw/EasyBuild/modules/all:/sw/bw/modulefiles:/sw/xe/modulefiles:/usr/local/modulefiles:/opt/cray/gem/modulefiles:/opt/cray/craype/2.5.8/modulefiles:/opt/cray/modulefiles:/opt/modulefiles ;export MODULEPATH;MPICH_ABORT_ON_ERROR=1 ;export MPICH_ABORT_ON_ERROR;MPICH_DIR=/opt/cray/mpt/7.5.0/gni/mpich-gnu/4.9 ;export MPICH_DIR;PATH=/opt/cray/rca/1.0.0-2.0502.60530.1.63.gem/bin:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/sbin:/opt/cray/dvs/2.5_0.9.0-1.0502.2188.1.113.gem/bin:/opt/cray/xpmem/0.1-2.0502.64982.5.3.gem/bin:/opt/cray/pmi/5.0.10-1.0000.11050.179.3.gem/bin:/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/bin:/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/bin:/opt/gcc/4.9.3/bin:/sw/admin/scripts:/sw/user/scripts:/opt/xalt/0.7.6/sles11.3/libexec:/opt/xalt/0.7.6/sles11.3/bin:/usr/local/gsi-openssh-6.2p2-2/bin:/opt/java/jdk1.8.0_51/bin:/usr/local/globus-5.2.5/bin:/usr/local/globus-5.2.5/sbin:/opt/moab/9.0.2t7/sbin:/opt/torque/6.0.4/sbin:/opt/torque/6.0.4/bin:/opt/cray/mpt/7.5.0/gni/bin:/opt/cray/craype/2.5.8/bin:/opt/cray/eslogin/eswrap/1.3.3-1.020200.1280.0/bin:/opt/modules/3.2.10.4/bin:/opt/moab/9.0.2t7/bin:/u/system/bwjenkins/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin ;export PATH;PE_ACML_DEFAULT_DIR_CRAY=gfortran64 ;export PE_ACML_DEFAULT_DIR_CRAY;PE_ACML_DEFAULT_DIR_DEFAULT64=_int64 ;export PE_ACML_DEFAULT_DIR_DEFAULT64;PE_ACML_DEFAULT_DIR_GNU=gfortran64 ;export PE_ACML_DEFAULT_DIR_GNU;PE_ACML_DEFAULT_DIR_INTEL=ifort64 ;export PE_ACML_DEFAULT_DIR_INTEL;PE_ACML_DEFAULT_DIR_OPENMP=_mp ;export PE_ACML_DEFAULT_DIR_OPENMP;PE_ACML_DEFAULT_DIR_PGI=pgi64 ;export PE_ACML_DEFAULT_DIR_PGI;PE_ACML_DEFAULT_TARGET_interlagos=_fma4 ;export PE_ACML_DEFAULT_TARGET_interlagos;PE_ACML_DEFAULT_TARGET_x86_64= ;export PE_ACML_DEFAULT_TARGET_x86_64;PE_ACML_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/acml/5.3.1/@PE_ACML_DEFAULT_DIR@@PE_ACML_DEFAULT_TARGET@@PE_ACML_DEFAULT_DIR_OPENMP@@PE_ACML_DEFAULT_DIR_DEFAULT64@/lib/pkgconfig ;export PE_ACML_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_CRAY_DEFAULT_FIXED_PKGCONFIG_PATH=/opt/cray/parallel-netcdf/1.7.0/CRAY/8.3/lib/pkgconfig:/opt/cray/netcdf-hdf5parallel/4.4.1/CRAY/8.3/lib/pkgconfig:/opt/cray/netcdf/4.4.1/CRAY/8.3/lib/pkgconfig:/opt/cray/hdf5-parallel/1.10.0/CRAY/8.3/lib/pkgconfig:/opt/cray/hdf5/1.10.0/CRAY/8.3/lib/pkgconfig:/opt/cray/ga/5.3.0.7/CRAY/8.4/lib/pkgconfig ;export PE_CRAY_DEFAULT_FIXED_PKGCONFIG_PATH;PE_CXX_PKGCONFIG_LIBS=mpichcxx ;export PE_CXX_PKGCONFIG_LIBS;PE_ENV=GNU ;export PE_ENV;PE_FFTW_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH ;export PE_FFTW_DEFAULT_REQUIRED_PRODUCTS;PE_FFTW_DEFAULT_TARGET_abudhabi=abudhabi ;export PE_FFTW_DEFAULT_TARGET_abudhabi;PE_FFTW_DEFAULT_TARGET_broadwell=broadwell ;export PE_FFTW_DEFAULT_TARGET_broadwell;PE_FFTW_DEFAULT_TARGET_haswell=haswell ;export PE_FFTW_DEFAULT_TARGET_haswell;PE_FFTW_DEFAULT_TARGET_interlagos=interlagos ;export PE_FFTW_DEFAULT_TARGET_interlagos;PE_FFTW_DEFAULT_TARGET_ivybridge=ivybridge ;export PE_FFTW_DEFAULT_TARGET_ivybridge;PE_FFTW_DEFAULT_TARGET_mic_knl=mic_knl ;export PE_FFTW_DEFAULT_TARGET_mic_knl;PE_FFTW_DEFAULT_TARGET_sandybridge=sandybridge ;export PE_FFTW_DEFAULT_TARGET_sandybridge;PE_FFTW_DEFAULT_TARGET_share=share ;export PE_FFTW_DEFAULT_TARGET_share;PE_FFTW_DEFAULT_TARGET_x86_64=x86_64 ;export PE_FFTW_DEFAULT_TARGET_x86_64;PE_FFTW_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/fftw/3.3.4.10/@PE_FFTW_DEFAULT_TARGET@/lib/pkgconfig ;export PE_FFTW_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_FORTRAN_PKGCONFIG_LIBS=mpichf90 ;export PE_FORTRAN_PKGCONFIG_LIBS;PE_GA_DEFAULT_FIXED_PRGENV=CRAY\ PGI\ INTEL ;export PE_GA_DEFAULT_FIXED_PRGENV;PE_GA_DEFAULT_GENCOMPILERS_GNU=5.1\ 4.9 ;export PE_GA_DEFAULT_GENCOMPILERS_GNU;PE_GA_DEFAULT_GENCOMPS_GNU=51\ 49 ;export PE_GA_DEFAULT_GENCOMPS_GNU;PE_GA_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/ga/5.3.0.7/@PRGENV@/@PE_GA_DEFAULT_GENCOMPS@/lib/pkgconfig ;export PE_GA_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_GA_DEFAULT_VOLATILE_PRGENV=GNU ;export PE_GA_DEFAULT_VOLATILE_PRGENV;PE_HDF5_DEFAULT_FIXED_PRGENV=CRAY\ PGI\ INTEL ;export PE_HDF5_DEFAULT_FIXED_PRGENV;PE_HDF5_DEFAULT_GENCOMPILERS_GNU=5.1\ 4.9 ;export PE_HDF5_DEFAULT_GENCOMPILERS_GNU;PE_HDF5_DEFAULT_GENCOMPS_GNU=51\ 49 ;export PE_HDF5_DEFAULT_GENCOMPS_GNU;PE_HDF5_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH ;export PE_HDF5_DEFAULT_REQUIRED_PRODUCTS;PE_HDF5_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/hdf5/1.10.0/@PRGENV@/@PE_HDF5_DEFAULT_GENCOMPS@/lib/pkgconfig ;export PE_HDF5_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_HDF5_DEFAULT_VOLATILE_PRGENV=GNU ;export PE_HDF5_DEFAULT_VOLATILE_PRGENV;PE_HDF5_PARALLEL_DEFAULT_FIXED_PRGENV=CRAY\ PGI\ INTEL ;export PE_HDF5_PARALLEL_DEFAULT_FIXED_PRGENV;PE_HDF5_PARALLEL_DEFAULT_GENCOMPILERS_GNU=5.1\ 4.9 ;export PE_HDF5_PARALLEL_DEFAULT_GENCOMPILERS_GNU;PE_HDF5_PARALLEL_DEFAULT_GENCOMPS_GNU=51\ 49 ;export PE_HDF5_PARALLEL_DEFAULT_GENCOMPS_GNU;PE_HDF5_PARALLEL_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH ;export PE_HDF5_PARALLEL_DEFAULT_REQUIRED_PRODUCTS;PE_HDF5_PARALLEL_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/hdf5-parallel/1.10.0/@PRGENV@/@PE_HDF5_PARALLEL_DEFAULT_GENCOMPS@/lib/pkgconfig ;export PE_HDF5_PARALLEL_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_HDF5_PARALLEL_DEFAULT_VOLATILE_PRGENV=GNU ;export PE_HDF5_PARALLEL_DEFAULT_VOLATILE_PRGENV;PE_INTEL_DEFAULT_FIXED_PKGCONFIG_PATH=/opt/cray/parallel-netcdf/1.7.0/INTEL/15.0/lib/pkgconfig:/opt/cray/netcdf-hdf5parallel/4.4.1/INTEL/15.0/lib/pkgconfig:/opt/cray/netcdf/4.4.1/INTEL/15.0/lib/pkgconfig:/opt/cray/mpt/7.5.0/gni/mpich-intel/16.0/lib/pkgconfig:/opt/cray/hdf5-parallel/1.10.0/INTEL/15.0/lib/pkgconfig:/opt/cray/hdf5/1.10.0/INTEL/15.0/lib/pkgconfig:/opt/cray/ga/5.3.0.7/INTEL/15.0/lib/pkgconfig ;export PE_INTEL_DEFAULT_FIXED_PKGCONFIG_PATH;PE_LIBSCI_ACC_DEFAULT_GENCOMPILERS_CRAY_x86_64=8.3 ;export PE_LIBSCI_ACC_DEFAULT_GENCOMPILERS_CRAY_x86_64;PE_LIBSCI_ACC_DEFAULT_GENCOMPILERS_GNU_x86_64=4.9 ;export PE_LIBSCI_ACC_DEFAULT_GENCOMPILERS_GNU_x86_64;PE_LIBSCI_ACC_DEFAULT_GENCOMPS_CRAY_x86_64=83 ;export PE_LIBSCI_ACC_DEFAULT_GENCOMPS_CRAY_x86_64;PE_LIBSCI_ACC_DEFAULT_GENCOMPS_GNU_x86_64=49 ;export PE_LIBSCI_ACC_DEFAULT_GENCOMPS_GNU_x86_64;PE_LIBSCI_ACC_DEFAULT_NV_SUFFIX_nvidia20=nv20 ;export PE_LIBSCI_ACC_DEFAULT_NV_SUFFIX_nvidia20;PE_LIBSCI_ACC_DEFAULT_NV_SUFFIX_nvidia35=nv35 ;export PE_LIBSCI_ACC_DEFAULT_NV_SUFFIX_nvidia35;PE_LIBSCI_ACC_DEFAULT_PKGCONFIG_VARIABLES=PE_LIBSCI_ACC_DEFAULT_NV_SUFFIX_@accelerator@ ;export PE_LIBSCI_ACC_DEFAULT_PKGCONFIG_VARIABLES;PE_LIBSCI_ACC_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH:PE_LIBSCI ;export PE_LIBSCI_ACC_DEFAULT_REQUIRED_PRODUCTS;PE_LIBSCI_ACC_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/libsci_acc/16.03.1/@PRGENV@/@PE_LIBSCI_ACC_DEFAULT_GENCOMPS@/@PE_LIBSCI_ACC_DEFAULT_TARGET@/lib/pkgconfig ;export PE_LIBSCI_ACC_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_LIBSCI_ACC_DEFAULT_VOLATILE_PRGENV=CRAY\ GNU ;export PE_LIBSCI_ACC_DEFAULT_VOLATILE_PRGENV;PE_LIBSCI_DEFAULT_GENCOMPILERS_CRAY_mic_knl=8.5 ;export PE_LIBSCI_DEFAULT_GENCOMPILERS_CRAY_mic_knl;PE_LIBSCI_DEFAULT_GENCOMPILERS_CRAY_x86_64=8.3 ;export PE_LIBSCI_DEFAULT_GENCOMPILERS_CRAY_x86_64;PE_LIBSCI_DEFAULT_GENCOMPILERS_GNU_mic_knl=5.1 ;export PE_LIBSCI_DEFAULT_GENCOMPILERS_GNU_mic_knl;PE_LIBSCI_DEFAULT_GENCOMPILERS_GNU_x86_64=5.1\ 4.9 ;export PE_LIBSCI_DEFAULT_GENCOMPILERS_GNU_x86_64;PE_LIBSCI_DEFAULT_GENCOMPILERS_INTEL_mic_knl=15.0 ;export PE_LIBSCI_DEFAULT_GENCOMPILERS_INTEL_mic_knl;PE_LIBSCI_DEFAULT_GENCOMPILERS_INTEL_x86_64=15.0 ;export PE_LIBSCI_DEFAULT_GENCOMPILERS_INTEL_x86_64;PE_LIBSCI_DEFAULT_GENCOMPILERS_PGI_x86_64=15.3 ;export PE_LIBSCI_DEFAULT_GENCOMPILERS_PGI_x86_64;PE_LIBSCI_DEFAULT_GENCOMPS_CRAY_mic_knl=85 ;export PE_LIBSCI_DEFAULT_GENCOMPS_CRAY_mic_knl;PE_LIBSCI_DEFAULT_GENCOMPS_CRAY_x86_64=83 ;export PE_LIBSCI_DEFAULT_GENCOMPS_CRAY_x86_64;PE_LIBSCI_DEFAULT_GENCOMPS_GNU_mic_knl=51 ;export PE_LIBSCI_DEFAULT_GENCOMPS_GNU_mic_knl;PE_LIBSCI_DEFAULT_GENCOMPS_GNU_x86_64=51\ 49 ;export PE_LIBSCI_DEFAULT_GENCOMPS_GNU_x86_64;PE_LIBSCI_DEFAULT_GENCOMPS_INTEL_mic_knl=150 ;export PE_LIBSCI_DEFAULT_GENCOMPS_INTEL_mic_knl;PE_LIBSCI_DEFAULT_GENCOMPS_INTEL_x86_64=150 ;export PE_LIBSCI_DEFAULT_GENCOMPS_INTEL_x86_64;PE_LIBSCI_DEFAULT_GENCOMPS_PGI_x86_64=153 ;export PE_LIBSCI_DEFAULT_GENCOMPS_PGI_x86_64;PE_LIBSCI_DEFAULT_OMP_REQUIRES= ;export PE_LIBSCI_DEFAULT_OMP_REQUIRES;PE_LIBSCI_DEFAULT_OMP_REQUIRES_openmp=_mp ;export PE_LIBSCI_DEFAULT_OMP_REQUIRES_openmp;PE_LIBSCI_DEFAULT_PKGCONFIG_VARIABLES=PE_LIBSCI_DEFAULT_OMP_REQUIRES_@openmp@ ;export PE_LIBSCI_DEFAULT_PKGCONFIG_VARIABLES;PE_LIBSCI_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH ;export PE_LIBSCI_DEFAULT_REQUIRED_PRODUCTS;PE_LIBSCI_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/libsci/16.11.1/@PRGENV@/@PE_LIBSCI_DEFAULT_GENCOMPS@/@PE_LIBSCI_DEFAULT_TARGET@/lib/pkgconfig ;export PE_LIBSCI_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_LIBSCI_DEFAULT_VOLATILE_PRGENV=CRAY\ GNU\ INTEL\ PGI ;export PE_LIBSCI_DEFAULT_VOLATILE_PRGENV;PE_LIBSCI_GENCOMPILERS_CRAY_mic_knl=8.5 ;export PE_LIBSCI_GENCOMPILERS_CRAY_mic_knl;PE_LIBSCI_GENCOMPILERS_CRAY_x86_64=8.3 ;export PE_LIBSCI_GENCOMPILERS_CRAY_x86_64;PE_LIBSCI_GENCOMPILERS_GNU_mic_knl=5.1 ;export PE_LIBSCI_GENCOMPILERS_GNU_mic_knl;PE_LIBSCI_GENCOMPILERS_GNU_x86_64=5.1\ 4.9 ;export PE_LIBSCI_GENCOMPILERS_GNU_x86_64;PE_LIBSCI_GENCOMPILERS_INTEL_mic_knl=15.0 ;export PE_LIBSCI_GENCOMPILERS_INTEL_mic_knl;PE_LIBSCI_GENCOMPILERS_INTEL_x86_64=15.0 ;export PE_LIBSCI_GENCOMPILERS_INTEL_x86_64;PE_LIBSCI_GENCOMPILERS_PGI_x86_64=15.3 ;export PE_LIBSCI_GENCOMPILERS_PGI_x86_64;PE_LIBSCI_GENCOMPS_CRAY_mic_knl=85 ;export PE_LIBSCI_GENCOMPS_CRAY_mic_knl;PE_LIBSCI_GENCOMPS_CRAY_x86_64=83 ;export PE_LIBSCI_GENCOMPS_CRAY_x86_64;PE_LIBSCI_GENCOMPS_GNU_mic_knl=51 ;export PE_LIBSCI_GENCOMPS_GNU_mic_knl;PE_LIBSCI_GENCOMPS_GNU_x86_64=51\ 49 ;export PE_LIBSCI_GENCOMPS_GNU_x86_64;PE_LIBSCI_GENCOMPS_INTEL_mic_knl=150 ;export PE_LIBSCI_GENCOMPS_INTEL_mic_knl;PE_LIBSCI_GENCOMPS_INTEL_x86_64=150 ;export PE_LIBSCI_GENCOMPS_INTEL_x86_64;PE_LIBSCI_GENCOMPS_PGI_x86_64=153 ;export PE_LIBSCI_GENCOMPS_PGI_x86_64;PE_LIBSCI_MODULE_NAME=cray-libsci/16.11.1 ;export PE_LIBSCI_MODULE_NAME;PE_LIBSCI_OMP_REQUIRES= ;export PE_LIBSCI_OMP_REQUIRES;PE_LIBSCI_OMP_REQUIRES_openmp=_mp ;export PE_LIBSCI_OMP_REQUIRES_openmp;PE_LIBSCI_PKGCONFIG_LIBS=libsci_mpi:libsci ;export PE_LIBSCI_PKGCONFIG_LIBS;PE_LIBSCI_PKGCONFIG_VARIABLES=PE_LIBSCI_OMP_REQUIRES_@openmp@ ;export PE_LIBSCI_PKGCONFIG_VARIABLES;PE_LIBSCI_REQUIRED_PRODUCTS=PE_MPICH ;export PE_LIBSCI_REQUIRED_PRODUCTS;PE_LIBSCI_VOLATILE_PKGCONFIG_PATH=/opt/cray/libsci/16.11.1/@PRGENV@/@PE_LIBSCI_GENCOMPS@/@PE_LIBSCI_TARGET@/lib/pkgconfig ;export PE_LIBSCI_VOLATILE_PKGCONFIG_PATH;PE_LIBSCI_VOLATILE_PRGENV=CRAY\ GNU\ INTEL\ PGI ;export PE_LIBSCI_VOLATILE_PRGENV;PE_MPICH_CXX_PKGCONFIG_LIBS=mpichcxx ;export PE_MPICH_CXX_PKGCONFIG_LIBS;PE_MPICH_DEFAULT_DIR_CRAY_DEFAULT64=64 ;export PE_MPICH_DEFAULT_DIR_CRAY_DEFAULT64;PE_MPICH_DEFAULT_DIR_PGI_DEFAULT64=64 ;export PE_MPICH_DEFAULT_DIR_PGI_DEFAULT64;PE_MPICH_DEFAULT_FIXED_PRGENV=INTEL ;export PE_MPICH_DEFAULT_FIXED_PRGENV;PE_MPICH_DEFAULT_GENCOMPILERS_CRAY=8.4 ;export PE_MPICH_DEFAULT_GENCOMPILERS_CRAY;PE_MPICH_DEFAULT_GENCOMPILERS_GNU=5.1\ 4.9 ;export PE_MPICH_DEFAULT_GENCOMPILERS_GNU;PE_MPICH_DEFAULT_GENCOMPILERS_PGI=15.3 ;export PE_MPICH_DEFAULT_GENCOMPILERS_PGI;PE_MPICH_DEFAULT_GENCOMPS_CRAY=84 ;export PE_MPICH_DEFAULT_GENCOMPS_CRAY;PE_MPICH_DEFAULT_GENCOMPS_GNU=51\ 49 ;export PE_MPICH_DEFAULT_GENCOMPS_GNU;PE_MPICH_DEFAULT_GENCOMPS_PGI=153 ;export PE_MPICH_DEFAULT_GENCOMPS_PGI;PE_MPICH_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/mpt/7.5.0/gni/mpich-@PRGENV@@PE_MPICH_DEFAULT_DIR_DEFAULT64@/@PE_MPICH_DEFAULT_GENCOMPS@/lib/pkgconfig ;export PE_MPICH_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_MPICH_DEFAULT_VOLATILE_PRGENV=CRAY\ GNU\ PGI ;export PE_MPICH_DEFAULT_VOLATILE_PRGENV;PE_MPICH_FORTRAN_PKGCONFIG_LIBS=mpichf90 ;export PE_MPICH_FORTRAN_PKGCONFIG_LIBS;PE_MPICH_MODULE_NAME=cray-mpich ;export PE_MPICH_MODULE_NAME;PE_MPICH_MULTITHREADED_LIBS_multithreaded=_mt ;export PE_MPICH_MULTITHREADED_LIBS_multithreaded;PE_MPICH_NV_LIBS= ;export PE_MPICH_NV_LIBS;PE_MPICH_NV_LIBS_nvidia20=-lcudart ;export PE_MPICH_NV_LIBS_nvidia20;PE_MPICH_NV_LIBS_nvidia35=-lcudart ;export PE_MPICH_NV_LIBS_nvidia35;PE_MPICH_PKGCONFIG_LIBS=mpich ;export PE_MPICH_PKGCONFIG_LIBS;PE_MPICH_PKGCONFIG_VARIABLES=PE_MPICH_NV_LIBS_@accelerator@:PE_MPICH_MULTITHREADED_LIBS_@multithreaded@ ;export PE_MPICH_PKGCONFIG_VARIABLES;PE_MPICH_TARGET_VAR_nvidia20=-lcudart ;export PE_MPICH_TARGET_VAR_nvidia20;PE_MPICH_TARGET_VAR_nvidia35=-lcudart ;export PE_MPICH_TARGET_VAR_nvidia35;PE_NETCDF_DEFAULT_FIXED_PRGENV=CRAY\ PGI\ INTEL ;export PE_NETCDF_DEFAULT_FIXED_PRGENV;PE_NETCDF_DEFAULT_GENCOMPILERS_GNU=5.1\ 4.9 ;export PE_NETCDF_DEFAULT_GENCOMPILERS_GNU;PE_NETCDF_DEFAULT_GENCOMPS_GNU=51\ 49 ;export PE_NETCDF_DEFAULT_GENCOMPS_GNU;PE_NETCDF_DEFAULT_REQUIRED_PRODUCTS=PE_HDF5 ;export PE_NETCDF_DEFAULT_REQUIRED_PRODUCTS;PE_NETCDF_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/netcdf/4.4.1/@PRGENV@/@PE_NETCDF_DEFAULT_GENCOMPS@/lib/pkgconfig ;export PE_NETCDF_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_NETCDF_DEFAULT_VOLATILE_PRGENV=GNU ;export PE_NETCDF_DEFAULT_VOLATILE_PRGENV;PE_NETCDF_HDF5PARALLEL_DEFAULT_FIXED_PRGENV=CRAY\ PGI\ INTEL ;export PE_NETCDF_HDF5PARALLEL_DEFAULT_FIXED_PRGENV;PE_NETCDF_HDF5PARALLEL_DEFAULT_GENCOMPILERS_GNU=5.1\ 4.9 ;export PE_NETCDF_HDF5PARALLEL_DEFAULT_GENCOMPILERS_GNU;PE_NETCDF_HDF5PARALLEL_DEFAULT_GENCOMPS_GNU=51\ 49 ;export PE_NETCDF_HDF5PARALLEL_DEFAULT_GENCOMPS_GNU;PE_NETCDF_HDF5PARALLEL_DEFAULT_REQUIRED_PRODUCTS=PE_HDF5_PARALLEL:PE_MPICH ;export PE_NETCDF_HDF5PARALLEL_DEFAULT_REQUIRED_PRODUCTS;PE_NETCDF_HDF5PARALLEL_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/netcdf-hdf5parallel/4.4.1/@PRGENV@/@PE_NETCDF_HDF5PARALLEL_DEFAULT_GENCOMPS@/lib/pkgconfig ;export PE_NETCDF_HDF5PARALLEL_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_NETCDF_HDF5PARALLEL_DEFAULT_VOLATILE_PRGENV=GNU ;export PE_NETCDF_HDF5PARALLEL_DEFAULT_VOLATILE_PRGENV;PE_PAPI_DEFAULT_ACCEL_LIBS= ;export PE_PAPI_DEFAULT_ACCEL_LIBS;PE_PAPI_DEFAULT_ACCEL_LIBS_nvidia35=,-lcupti,-lcudart,-lcuda ;export PE_PAPI_DEFAULT_ACCEL_LIBS_nvidia35;PE_PAPI_DEFAULT_PKGCONFIG_VARIABLES=PE_PAPI_ACCEL_LIBS_@accelerator@ ;export PE_PAPI_DEFAULT_PKGCONFIG_VARIABLES;PE_PAPI_DEFAULT_TARGET_VAR_nvidia35=,-lcupti,-lcudart,-lcuda ;export PE_PAPI_DEFAULT_TARGET_VAR_nvidia35;PE_PARALLEL_NETCDF_DEFAULT_FIXED_PRGENV=CRAY\ PGI\ INTEL ;export PE_PARALLEL_NETCDF_DEFAULT_FIXED_PRGENV;PE_PARALLEL_NETCDF_DEFAULT_GENCOMPILERS_GNU=5.1\ 4.9 ;export PE_PARALLEL_NETCDF_DEFAULT_GENCOMPILERS_GNU;PE_PARALLEL_NETCDF_DEFAULT_GENCOMPS_GNU=51\ 49 ;export PE_PARALLEL_NETCDF_DEFAULT_GENCOMPS_GNU;PE_PARALLEL_NETCDF_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/parallel-netcdf/1.7.0/@PRGENV@/@PE_PARALLEL_NETCDF_DEFAULT_GENCOMPS@/lib/pkgconfig ;export PE_PARALLEL_NETCDF_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_PARALLEL_NETCDF_DEFAULT_VOLATILE_PRGENV=GNU ;export PE_PARALLEL_NETCDF_DEFAULT_VOLATILE_PRGENV;PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_haswell=8.3 ;export PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_haswell;PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_interlagos=8.3 ;export PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_interlagos;PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_mic_knl=8.5 ;export PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_mic_knl;PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_sandybridge=8.3 ;export PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_sandybridge;PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_x86_64=8.3 ;export PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_x86_64;PE_PETSC_DEFAULT_GENCOMPILERS_GNU_haswell=5.1\ 4.9 ;export PE_PETSC_DEFAULT_GENCOMPILERS_GNU_haswell;PE_PETSC_DEFAULT_GENCOMPILERS_GNU_interlagos=5.1\ 4.9 ;export PE_PETSC_DEFAULT_GENCOMPILERS_GNU_interlagos;PE_PETSC_DEFAULT_GENCOMPILERS_GNU_sandybridge=5.1\ 4.9 ;export PE_PETSC_DEFAULT_GENCOMPILERS_GNU_sandybridge;PE_PETSC_DEFAULT_GENCOMPILERS_GNU_x86_64=5.1\ 4.9 ;export PE_PETSC_DEFAULT_GENCOMPILERS_GNU_x86_64;PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_haswell=15.0 ;export PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_haswell;PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_interlagos=15.0 ;export PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_interlagos;PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_sandybridge=15.0 ;export PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_sandybridge;PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_x86_64=15.0 ;export PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_x86_64;PE_PETSC_DEFAULT_GENCOMPILERS_PGI_interlagos=15.3 ;export PE_PETSC_DEFAULT_GENCOMPILERS_PGI_interlagos;PE_PETSC_DEFAULT_GENCOMPILERS_PGI_x86_64=15.3 ;export PE_PETSC_DEFAULT_GENCOMPILERS_PGI_x86_64;PE_PETSC_DEFAULT_GENCOMPS_CRAY_haswell=83 ;export PE_PETSC_DEFAULT_GENCOMPS_CRAY_haswell;PE_PETSC_DEFAULT_GENCOMPS_CRAY_interlagos=83 ;export PE_PETSC_DEFAULT_GENCOMPS_CRAY_interlagos;PE_PETSC_DEFAULT_GENCOMPS_CRAY_mic_knl=85 ;export PE_PETSC_DEFAULT_GENCOMPS_CRAY_mic_knl;PE_PETSC_DEFAULT_GENCOMPS_CRAY_sandybridge=83 ;export PE_PETSC_DEFAULT_GENCOMPS_CRAY_sandybridge;PE_PETSC_DEFAULT_GENCOMPS_CRAY_x86_64=83 ;export PE_PETSC_DEFAULT_GENCOMPS_CRAY_x86_64;PE_PETSC_DEFAULT_GENCOMPS_GNU_haswell=51\ 49 ;export PE_PETSC_DEFAULT_GENCOMPS_GNU_haswell;PE_PETSC_DEFAULT_GENCOMPS_GNU_interlagos=51\ 49 ;export PE_PETSC_DEFAULT_GENCOMPS_GNU_interlagos;PE_PETSC_DEFAULT_GENCOMPS_GNU_sandybridge=51\ 49 ;export PE_PETSC_DEFAULT_GENCOMPS_GNU_sandybridge;PE_PETSC_DEFAULT_GENCOMPS_GNU_x86_64=51\ 49 ;export PE_PETSC_DEFAULT_GENCOMPS_GNU_x86_64;PE_PETSC_DEFAULT_GENCOMPS_INTEL_haswell=150 ;export PE_PETSC_DEFAULT_GENCOMPS_INTEL_haswell;PE_PETSC_DEFAULT_GENCOMPS_INTEL_interlagos=150 ;export PE_PETSC_DEFAULT_GENCOMPS_INTEL_interlagos;PE_PETSC_DEFAULT_GENCOMPS_INTEL_sandybridge=150 ;export PE_PETSC_DEFAULT_GENCOMPS_INTEL_sandybridge;PE_PETSC_DEFAULT_GENCOMPS_INTEL_x86_64=150 ;export PE_PETSC_DEFAULT_GENCOMPS_INTEL_x86_64;PE_PETSC_DEFAULT_GENCOMPS_PGI_interlagos=153 ;export PE_PETSC_DEFAULT_GENCOMPS_PGI_interlagos;PE_PETSC_DEFAULT_GENCOMPS_PGI_x86_64=153 ;export PE_PETSC_DEFAULT_GENCOMPS_PGI_x86_64;PE_PETSC_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH:PE_LIBSCI:PE_HDF5_PARALLEL:PE_TPSL ;export PE_PETSC_DEFAULT_REQUIRED_PRODUCTS;PE_PETSC_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/petsc/3.7.2.1/complex/@PRGENV@/@PE_PETSC_DEFAULT_GENCOMPS@/@PE_PETSC_DEFAULT_TARGET@/lib/pkgconfig ;export PE_PETSC_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_PETSC_DEFAULT_VOLATILE_PRGENV=CRAY\ CRAY64\ GNU\ GNU64\ INTEL\ INTEL64\ PGI\ PGI64 ;export PE_PETSC_DEFAULT_VOLATILE_PRGENV;PE_PGI_DEFAULT_FIXED_PKGCONFIG_PATH=/opt/cray/parallel-netcdf/1.7.0/PGI/15.3/lib/pkgconfig:/opt/cray/netcdf-hdf5parallel/4.4.1/PGI/15.3/lib/pkgconfig:/opt/cray/netcdf/4.4.1/PGI/15.3/lib/pkgconfig:/opt/cray/hdf5-parallel/1.10.0/PGI/15.3/lib/pkgconfig:/opt/cray/hdf5/1.10.0/PGI/15.3/lib/pkgconfig:/opt/cray/ga/5.3.0.7/PGI/15.3/lib/pkgconfig ;export PE_PGI_DEFAULT_FIXED_PKGCONFIG_PATH;PE_PKGCONFIG_DEFAULT_PRODUCTS=PE_TRILINOS:PE_TPSL_64:PE_TPSL:PE_PETSC:PE_PARALLEL_NETCDF:PE_NETCDF_HDF5PARALLEL:PE_NETCDF:PE_MPICH:PE_LIBSCI_ACC:PE_LIBSCI:PE_HDF5_PARALLEL:PE_HDF5:PE_GA:PE_FFTW:PE_ACML ;export PE_PKGCONFIG_DEFAULT_PRODUCTS;PE_PKGCONFIG_LIBS=AtpSigHandler:cray-rca:libsci_mpi:libsci:mpich ;export PE_PKGCONFIG_LIBS;PE_PKGCONFIG_PRODUCTS=PE_LIBSCI:PE_MPICH ;export PE_PKGCONFIG_PRODUCTS;PE_PKGCONFIG_PRODUCTS_DEFAULT=PE_PAPI ;export PE_PKGCONFIG_PRODUCTS_DEFAULT;PE_PKG_CONFIG_PATH=/opt/cray/cti/1.0.4/lib/pkgconfig ;export PE_PKG_CONFIG_PATH;PE_PRODUCT_LIST=CRAY_RCA:CRAY_ALPS:DVS:CRAY_XPMEM:CRAY_DMAPP:CRAY_PMI:CRAY_UGNI:CRAY_UDREG:CRAY_LIBSCI:GNU:GCC:CRAYPE:CRAYPE_INTERLAGOS ;export PE_PRODUCT_LIST;PE_SMA_DEFAULT_COMPFLAG= ;export PE_SMA_DEFAULT_COMPFLAG;PE_SMA_DEFAULT_COMPFLAG_GNU=-fcray-pointer ;export PE_SMA_DEFAULT_COMPFLAG_GNU;PE_SMA_DEFAULT_DIR_CRAY_DEFAULT64=64 ;export PE_SMA_DEFAULT_DIR_CRAY_DEFAULT64;PE_SMA_DEFAULT_DIR_PGI_DEFAULT64=64 ;export PE_SMA_DEFAULT_DIR_PGI_DEFAULT64;PE_SMA_DEFAULT_PKGCONFIG_VARIABLES=PE_SMA_COMPFLAG_@prgenv@ ;export PE_SMA_DEFAULT_PKGCONFIG_VARIABLES;PE_SMA_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/mpt/7.5.0/gni/sma@PE_SMA_DEFAULT_DIR_DEFAULT64@/lib64/pkgconfig ;export PE_SMA_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_TPSL_64_DEFAULT_GENCOMPILERS_CRAY_haswell=8.3 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_CRAY_haswell;PE_TPSL_64_DEFAULT_GENCOMPILERS_CRAY_interlagos=8.3 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_CRAY_interlagos;PE_TPSL_64_DEFAULT_GENCOMPILERS_CRAY_mic_knl=8.5 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_CRAY_mic_knl;PE_TPSL_64_DEFAULT_GENCOMPILERS_CRAY_sandybridge=8.3 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_CRAY_sandybridge;PE_TPSL_64_DEFAULT_GENCOMPILERS_CRAY_x86_64=8.3 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_CRAY_x86_64;PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_haswell=5.1\ 4.9 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_haswell;PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_interlagos=5.1\ 4.9 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_interlagos;PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_sandybridge=5.1\ 4.9 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_sandybridge;PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_x86_64=5.1\ 4.9 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_x86_64;PE_TPSL_64_DEFAULT_GENCOMPILERS_INTEL_haswell=15.0 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_INTEL_haswell;PE_TPSL_64_DEFAULT_GENCOMPILERS_INTEL_interlagos=15.0 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_INTEL_interlagos;PE_TPSL_64_DEFAULT_GENCOMPILERS_INTEL_sandybridge=15.0 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_INTEL_sandybridge;PE_TPSL_64_DEFAULT_GENCOMPILERS_INTEL_x86_64=15.0 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_INTEL_x86_64;PE_TPSL_64_DEFAULT_GENCOMPILERS_PGI_interlagos=15.3 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_PGI_interlagos;PE_TPSL_64_DEFAULT_GENCOMPILERS_PGI_x86_64=15.3 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_PGI_x86_64;PE_TPSL_64_DEFAULT_GENCOMPS_CRAY_haswell=83 ;export PE_TPSL_64_DEFAULT_GENCOMPS_CRAY_haswell;PE_TPSL_64_DEFAULT_GENCOMPS_CRAY_interlagos=83 ;export PE_TPSL_64_DEFAULT_GENCOMPS_CRAY_interlagos;PE_TPSL_64_DEFAULT_GENCOMPS_CRAY_mic_knl=85 ;export PE_TPSL_64_DEFAULT_GENCOMPS_CRAY_mic_knl;PE_TPSL_64_DEFAULT_GENCOMPS_CRAY_sandybridge=83 ;export PE_TPSL_64_DEFAULT_GENCOMPS_CRAY_sandybridge;PE_TPSL_64_DEFAULT_GENCOMPS_CRAY_x86_64=83 ;export PE_TPSL_64_DEFAULT_GENCOMPS_CRAY_x86_64;PE_TPSL_64_DEFAULT_GENCOMPS_GNU_haswell=51\ 49 ;export PE_TPSL_64_DEFAULT_GENCOMPS_GNU_haswell;PE_TPSL_64_DEFAULT_GENCOMPS_GNU_interlagos=51\ 49 ;export PE_TPSL_64_DEFAULT_GENCOMPS_GNU_interlagos;PE_TPSL_64_DEFAULT_GENCOMPS_GNU_sandybridge=51\ 49 ;export PE_TPSL_64_DEFAULT_GENCOMPS_GNU_sandybridge;PE_TPSL_64_DEFAULT_GENCOMPS_GNU_x86_64=51\ 49 ;export PE_TPSL_64_DEFAULT_GENCOMPS_GNU_x86_64;PE_TPSL_64_DEFAULT_GENCOMPS_INTEL_haswell=150 ;export PE_TPSL_64_DEFAULT_GENCOMPS_INTEL_haswell;PE_TPSL_64_DEFAULT_GENCOMPS_INTEL_interlagos=150 ;export PE_TPSL_64_DEFAULT_GENCOMPS_INTEL_interlagos;PE_TPSL_64_DEFAULT_GENCOMPS_INTEL_sandybridge=150 ;export PE_TPSL_64_DEFAULT_GENCOMPS_INTEL_sandybridge;PE_TPSL_64_DEFAULT_GENCOMPS_INTEL_x86_64=150 ;export PE_TPSL_64_DEFAULT_GENCOMPS_INTEL_x86_64;PE_TPSL_64_DEFAULT_GENCOMPS_PGI_interlagos=153 ;export PE_TPSL_64_DEFAULT_GENCOMPS_PGI_interlagos;PE_TPSL_64_DEFAULT_GENCOMPS_PGI_x86_64=153 ;export PE_TPSL_64_DEFAULT_GENCOMPS_PGI_x86_64;PE_TPSL_64_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH:PE_LIBSCI ;export PE_TPSL_64_DEFAULT_REQUIRED_PRODUCTS;PE_TPSL_64_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/tpsl/16.07.1/@PRGENV@64/@PE_TPSL_64_DEFAULT_GENCOMPS@/@PE_TPSL_64_DEFAULT_TARGET@/lib/pkgconfig ;export PE_TPSL_64_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_TPSL_64_DEFAULT_VOLATILE_PRGENV=CRAY\ CRAY64\ GNU\ GNU64\ INTEL\ INTEL64\ PGI\ PGI64 ;export PE_TPSL_64_DEFAULT_VOLATILE_PRGENV;PE_TPSL_DEFAULT_GENCOMPILERS_CRAY_haswell=8.3 ;export PE_TPSL_DEFAULT_GENCOMPILERS_CRAY_haswell;PE_TPSL_DEFAULT_GENCOMPILERS_CRAY_interlagos=8.3 ;export PE_TPSL_DEFAULT_GENCOMPILERS_CRAY_interlagos;PE_TPSL_DEFAULT_GENCOMPILERS_CRAY_mic_knl=8.5 ;export PE_TPSL_DEFAULT_GENCOMPILERS_CRAY_mic_knl;PE_TPSL_DEFAULT_GENCOMPILERS_CRAY_sandybridge=8.3 ;export PE_TPSL_DEFAULT_GENCOMPILERS_CRAY_sandybridge;PE_TPSL_DEFAULT_GENCOMPILERS_CRAY_x86_64=8.3 ;export PE_TPSL_DEFAULT_GENCOMPILERS_CRAY_x86_64;PE_TPSL_DEFAULT_GENCOMPILERS_GNU_haswell=5.1\ 4.9 ;export PE_TPSL_DEFAULT_GENCOMPILERS_GNU_haswell;PE_TPSL_DEFAULT_GENCOMPILERS_GNU_interlagos=5.1\ 4.9 ;export PE_TPSL_DEFAULT_GENCOMPILERS_GNU_interlagos;PE_TPSL_DEFAULT_GENCOMPILERS_GNU_sandybridge=5.1\ 4.9 ;export PE_TPSL_DEFAULT_GENCOMPILERS_GNU_sandybridge;PE_TPSL_DEFAULT_GENCOMPILERS_GNU_x86_64=5.1\ 4.9 ;export PE_TPSL_DEFAULT_GENCOMPILERS_GNU_x86_64;PE_TPSL_DEFAULT_GENCOMPILERS_INTEL_haswell=15.0 ;export PE_TPSL_DEFAULT_GENCOMPILERS_INTEL_haswell;PE_TPSL_DEFAULT_GENCOMPILERS_INTEL_interlagos=15.0 ;export PE_TPSL_DEFAULT_GENCOMPILERS_INTEL_interlagos;PE_TPSL_DEFAULT_GENCOMPILERS_INTEL_sandybridge=15.0 ;export PE_TPSL_DEFAULT_GENCOMPILERS_INTEL_sandybridge;PE_TPSL_DEFAULT_GENCOMPILERS_INTEL_x86_64=15.0 ;export PE_TPSL_DEFAULT_GENCOMPILERS_INTEL_x86_64;PE_TPSL_DEFAULT_GENCOMPILERS_PGI_interlagos=15.3 ;export PE_TPSL_DEFAULT_GENCOMPILERS_PGI_interlagos;PE_TPSL_DEFAULT_GENCOMPILERS_PGI_x86_64=15.3 ;export PE_TPSL_DEFAULT_GENCOMPILERS_PGI_x86_64;PE_TPSL_DEFAULT_GENCOMPS_CRAY_haswell=83 ;export PE_TPSL_DEFAULT_GENCOMPS_CRAY_haswell;PE_TPSL_DEFAULT_GENCOMPS_CRAY_interlagos=83 ;export PE_TPSL_DEFAULT_GENCOMPS_CRAY_interlagos;PE_TPSL_DEFAULT_GENCOMPS_CRAY_mic_knl=85 ;export PE_TPSL_DEFAULT_GENCOMPS_CRAY_mic_knl;PE_TPSL_DEFAULT_GENCOMPS_CRAY_sandybridge=83 ;export PE_TPSL_DEFAULT_GENCOMPS_CRAY_sandybridge;PE_TPSL_DEFAULT_GENCOMPS_CRAY_x86_64=83 ;export PE_TPSL_DEFAULT_GENCOMPS_CRAY_x86_64;PE_TPSL_DEFAULT_GENCOMPS_GNU_haswell=51\ 49 ;export PE_TPSL_DEFAULT_GENCOMPS_GNU_haswell;PE_TPSL_DEFAULT_GENCOMPS_GNU_interlagos=51\ 49 ;export PE_TPSL_DEFAULT_GENCOMPS_GNU_interlagos;PE_TPSL_DEFAULT_GENCOMPS_GNU_sandybridge=51\ 49 ;export PE_TPSL_DEFAULT_GENCOMPS_GNU_sandybridge;PE_TPSL_DEFAULT_GENCOMPS_GNU_x86_64=51\ 49 ;export PE_TPSL_DEFAULT_GENCOMPS_GNU_x86_64;PE_TPSL_DEFAULT_GENCOMPS_INTEL_haswell=150 ;export PE_TPSL_DEFAULT_GENCOMPS_INTEL_haswell;PE_TPSL_DEFAULT_GENCOMPS_INTEL_interlagos=150 ;export PE_TPSL_DEFAULT_GENCOMPS_INTEL_interlagos;PE_TPSL_DEFAULT_GENCOMPS_INTEL_sandybridge=150 ;export PE_TPSL_DEFAULT_GENCOMPS_INTEL_sandybridge;PE_TPSL_DEFAULT_GENCOMPS_INTEL_x86_64=150 ;export PE_TPSL_DEFAULT_GENCOMPS_INTEL_x86_64;PE_TPSL_DEFAULT_GENCOMPS_PGI_interlagos=153 ;export PE_TPSL_DEFAULT_GENCOMPS_PGI_interlagos;PE_TPSL_DEFAULT_GENCOMPS_PGI_x86_64=153 ;export PE_TPSL_DEFAULT_GENCOMPS_PGI_x86_64;PE_TPSL_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH:PE_LIBSCI ;export PE_TPSL_DEFAULT_REQUIRED_PRODUCTS;PE_TPSL_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/tpsl/16.07.1/@PRGENV@/@PE_TPSL_DEFAULT_GENCOMPS@/@PE_TPSL_DEFAULT_TARGET@/lib/pkgconfig ;export PE_TPSL_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_TPSL_DEFAULT_VOLATILE_PRGENV=CRAY\ CRAY64\ GNU\ GNU64\ INTEL\ INTEL64\ PGI\ PGI64 ;export PE_TPSL_DEFAULT_VOLATILE_PRGENV;PE_TRILINOS_DEFAULT_GENCOMPILERS_CRAY_x86_64=8.5 ;export PE_TRILINOS_DEFAULT_GENCOMPILERS_CRAY_x86_64;PE_TRILINOS_DEFAULT_GENCOMPILERS_GNU_x86_64=5.1\ 4.9 ;export PE_TRILINOS_DEFAULT_GENCOMPILERS_GNU_x86_64;PE_TRILINOS_DEFAULT_GENCOMPS_CRAY_x86_64=85 ;export PE_TRILINOS_DEFAULT_GENCOMPS_CRAY_x86_64;PE_TRILINOS_DEFAULT_GENCOMPS_GNU_x86_64=51\ 49 ;export PE_TRILINOS_DEFAULT_GENCOMPS_GNU_x86_64;PE_TRILINOS_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH:PE_HDF5_PARALLEL:PE_NETCDF_HDF5PARALLEL:PE_LIBSCI:PE_TPSL ;export PE_TRILINOS_DEFAULT_REQUIRED_PRODUCTS;PE_TRILINOS_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/trilinos/12.6.3.3/@PRGENV@/@PE_TRILINOS_DEFAULT_GEN module load cudatoolkit ~/bin/myenv.sh # for topomapping run: # module load topomapping # cp ~/awp-odc-spp/Makefile.bluewaters.gnu.topomapping Makefile.bluewaters.gnu make -f Makefile.bluewaters.gnu cd ~/awp-odc-spp/awp-odc-os/run cat run_small.pbs | sed -e "s/aprun/time aprun/g" > runs.pbs cp runs.pbs run_small.pbs cat -n run_small.pbs rm awp-odc_gpu_small.* MYJOBID=`qsub run_small.pbs` qsub run_small.pbs COMPS@/@PE_TRILINOS_DEFAULT_TARGET@/lib/pkgconfig ;export PE_TRILINOS_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_TRILINOS_DEFAULT_VOLATILE_PRGENV=CRAY\ GNU ;export PE_TRILINOS_DEFAULT_VOLATILE_PRGENV;PKGCONFIG_ENABLED=1 ;export PKGCONFIG_ENABLED;PKG_CONFIG_PATH=/opt/cray/rca/1.0.0-2.0502.60530.1.63.gem/lib64/pkgconfig:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/lib64/pkgconfig:/opt/cray/xpmem/0.1-2.0502.64982.5.3.gem/lib64/pkgconfig:/opt/cray/gni-headers/4.0-1.0502.10859.7.8.gem/lib64/pkgconfig:/opt/cray/dmapp/7.0.1-1.0502.11080.8.74.gem/lib64/pkgconfig:/opt/cray/pmi/5.0.10-1.0000.11050.179.3.gem/lib64/pkgconfig:/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/lib64/pkgconfig:/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/lib64/pkgconfig:/opt/cray/craype/2.5.8/pkg-config:/opt/cray/iobuf/2.0.7/lib/pkgconfig:/opt/cray/atp/2.0.4/lib/pkgconfig ;export PKG_CONFIG_PATH;PKG_CONFIG_PATH_DEFAULT=/opt/cray/papi/5.5.0.1/lib64/pkgconfig ;export PKG_CONFIG_PATH_DEFAULT;SHMEM_ABORT_ON_ERROR=1 ;export SHMEM_ABORT_ON_ERROR;XTOS_VERSION=5.2.82 ;export XTOS_VERSION;XTPE_NETWORK_TARGET=gemini ;export XTPE_NETWORK_TARGET;_LMFILES_=/opt/modulefiles/modules/3.2.10.4:/opt/modulefiles/eswrap/1.3.3-1.020200.1280.0:/opt/modulefiles/gcc/4.9.3:/opt/cray/craype/2.5.8/modulefiles/craype-interlagos:/opt/cray/craype/2.5.8/modulefiles/craype-network-gemini:/opt/cray/modulefiles/craype/2.5.8:/opt/cray/modulefiles/cray-mpich/7.5.0:/opt/modulefiles/torque/6.0.4:/opt/modulefiles/moab/9.0.2.TAS7-1490665589_71674cf-sles11:/opt/modulefiles/java/jdk1.8.0_51:/usr/local/modulefiles/globus/5.2.5:/usr/local/modulefiles/gsissh/6.2p2:/opt/modulefiles/xalt/0.7.6.local:/sw/xe/modulefiles/scripts:/sw/xe/modulefiles/user-paths:/opt/cray/modulefiles/cray-libsci/16.11.1:/opt/cray/gem/modulefiles/udreg/2.3.2-1.0502.10518.2.17.gem:/opt/cray/gem/modulefiles/ugni/6.0-1.0502.10863.8.28.gem:/opt/cray/gem/modulefiles/pmi/5.0.10-1.0000.11050.179.3.gem:/opt/cray/gem/modulefiles/dmapp/7.0.1-1.0502.11080.8.74.gem:/opt/cray/gem/modulefiles/gni-headers/4.0-1.0502.10859.7.8.gem:/opt/cray/gem/modulefiles/xpmem/0.1-2.0502.64982.5.3.gem:/opt/cray/gem/modulefiles/dvs/2.5_0.9.0-1.0502.2188.1.113.gem:/opt/cray/gem/modulefiles/alps/5.2.4-2.0502.9774.31.12.gem:/opt/cray/gem/modulefiles/rca/1.0.0-2.0502.60530.1.63.gem:/opt/cray/modulefiles/atp/2.0.4:/opt/cray/modulefiles/PrgEnv-gnu/5.2.82 ;export _LMFILES_;unset ASSEMBLER_X86_64;unset CC_X86_64;unset CRAYLIBS_X86_64;unset CRAYLMD_LICENSE_FILE;unset CRAY_BINUTILS_BIN;unset CRAY_BINUTILS_ROOT;unset CRAY_BINUTILS_VERSION;unset CRAY_CC_VERSION;unset CRAY_CXX_IPA_LIBS;unset CRAY_FTN_VERSION;unset CRAY_PE_TARGET;unset CRAY_PRGENVCRAY;unset FORTRAN_SYSTEM_MODULE_NAMES;unset FTN_X86_64;unset GCC_X86_64;unset INCLUDE_PATH_X86_64;unset LINKER_X86_64;unset NLSPATH;unset PE_LEVEL; ${MODULESHOME}/bin/modulecmd bash $* CRAYPE_LINK_TYPE=dynamic ;export CRAYPE_LINK_TYPE;CRAY_CUDATOOLKIT_DIR=/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1 ;export CRAY_CUDATOOLKIT_DIR;CRAY_CUDATOOLKIT_INCLUDE_OPTS=-I/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1/include\ -I/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1/extras/CUPTI/include\ -I/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1/extras/Debugger/include ;export CRAY_CUDATOOLKIT_INCLUDE_OPTS;CRAY_CUDATOOLKIT_POST_LINK_OPTS=-L/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1/lib64\ -L/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1/extras/CUPTI/lib64\ -Wl,--as-needed\ -Wl,-lcupti\ -Wl,-lcudart\ -Wl,--no-as-needed\ -L/opt/cray/nvidia/default/lib64\ -lcuda ;export CRAY_CUDATOOLKIT_POST_LINK_OPTS;CRAY_CUDATOOLKIT_VERSION=7.5.18-1.0502.10743.2.1 ;export CRAY_CUDATOOLKIT_VERSION;CRAY_LD_LIBRARY_PATH=/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1/lib64:/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1/extras/CUPTI/lib64:/opt/cray/rca/1.0.0-2.0502.60530.1.63.gem/lib64:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/lib64:/opt/cray/xpmem/0.1-2.0502.64982.5.3.gem/lib64:/opt/cray/dmapp/7.0.1-1.0502.11080.8.74.gem/lib64:/opt/cray/pmi/5.0.10-1.0000.11050.179.3.gem/lib64:/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/lib64:/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/lib64:/opt/cray/libsci/16.11.1/GNU/4.9/x86_64/lib:/opt/cray/mpt/7.5.0/gni/mpich-gnu/4.9/lib ;export CRAY_LD_LIBRARY_PATH;CUDATOOLKIT_HOME=/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1 ;export CUDATOOLKIT_HOME;LOADEDMODULES=modules/3.2.10.4:eswrap/1.3.3-1.020200.1280.0:gcc/4.9.3:craype-interlagos:craype-network-gemini:craype/2.5.8:cray-mpich/7.5.0:torque/6.0.4:moab/9.0.2.TAS7-1490665589_71674cf-sles11:java/jdk1.8.0_51:globus/5.2.5:gsissh/6.2p2:xalt/0.7.6.local:scripts:user-paths:cray-libsci/16.11.1:udreg/2.3.2-1.0502.10518.2.17.gem:ugni/6.0-1.0502.10863.8.28.gem:pmi/5.0.10-1.0000.11050.179.3.gem:dmapp/7.0.1-1.0502.11080.8.74.gem:gni-headers/4.0-1.0502.10859.7.8.gem:xpmem/0.1-2.0502.64982.5.3.gem:dvs/2.5_0.9.0-1.0502.2188.1.113.gem:alps/5.2.4-2.0502.9774.31.12.gem:rca/1.0.0-2.0502.60530.1.63.gem:atp/2.0.4:PrgEnv-gnu/5.2.82:cudatoolkit/7.5.18-1.0502.10743.2.1 ;export LOADEDMODULES;MANPATH=/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1/doc/man:/opt/cray/atp/2.0.4/man:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/man:/opt/cray/libsci/16.11.1/man:/opt/cray/man/csmlversion:/opt/gcc/4.9.3/snos/share/man:/sw/user/man:/opt/java/jdk1.8.0_51/man:/usr/local/globus-5.2.5/man:/opt/moab/9.0.2t7/share/man/:/opt/torque/6.0.4/share/man:/opt/cray/mpt/7.5.0/gni/man/mpich:/opt/cray/craype/2.5.8/man:/opt/cray/eslogin/eswrap/1.3.3-1.020200.1280.0/man:/opt/modules/3.2.10.4/share/man:/opt/moab/9.0.2t7/share/man::ignore:/usr/local/man:/usr/share/man:/usr/man:/opt/cray/share/man:/cm/local/apps/environment-modules/3.2.6/man:/opt/cray/man:/u/system/bwjenkins/man ;export MANPATH;PATH=/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1/bin:/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1/libnvvp:/opt/cray/rca/1.0.0-2.0502.60530.1.63.gem/bin:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/sbin:/opt/cray/dvs/2.5_0.9.0-1.0502.2188.1.113.gem/bin:/opt/cray/xpmem/0.1-2.0502.64982.5.3.gem/bin:/opt/cray/pmi/5.0.10-1.0000.11050.179.3.gem/bin:/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/bin:/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/bin:/opt/gcc/4.9.3/bin:/sw/admin/scripts:/sw/user/scripts:/opt/xalt/0.7.6/sles11.3/libexec:/opt/xalt/0.7.6/sles11.3/bin:/usr/local/gsi-openssh-6.2p2-2/bin:/opt/java/jdk1.8.0_51/bin:/usr/local/globus-5.2.5/bin:/usr/local/globus-5.2.5/sbin:/opt/moab/9.0.2t7/sbin:/opt/torque/6.0.4/sbin:/opt/torque/6.0.4/bin:/opt/cray/mpt/7.5.0/gni/bin:/opt/cray/craype/2.5.8/bin:/opt/cray/eslogin/eswrap/1.3.3-1.020200.1280.0/bin:/opt/modules/3.2.10.4/bin:/opt/moab/9.0.2t7/bin:/u/system/bwjenkins/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin ;export PATH;PE_PKGCONFIG_LIBS=cray-cudatoolkit:AtpSigHandler:cray-rca:libsci_mpi:libsci:mpich ;export PE_PKGCONFIG_LIBS;PKG_CONFIG_PATH=/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1/lib64/pkgconfig:/opt/cray/rca/1.0.0-2.0502.60530.1.63.gem/lib64/pkgconfig:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/lib64/pkgconfig:/opt/cray/xpmem/0.1-2.0502.64982.5.3.gem/lib64/pkgconfig:/opt/cray/gni-headers/4.0-1.0502.10859.7.8.gem/lib64/pkgconfig:/opt/cray/dmapp/7.0.1-1.0502.11080.8.74.gem/lib64/pkgconfig:/opt/cray/pmi/5.0.10-1.0000.11050.179.3.gem/lib64/pkgconfig:/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/lib64/pkgconfig:/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/lib64/pkgconfig:/opt/cray/craype/2.5.8/pkg-config:/opt/cray/iobuf/2.0.7/lib/pkgconfig:/opt/cray/atp/2.0.4/lib/pkgconfig ;export PKG_CONFIG_PATH;XTPE_LINK_TYPE=dynamic ;export XTPE_LINK_TYPE;_LMFILES_=/opt/modulefiles/modules/3.2.10.4:/opt/modulefiles/eswrap/1.3.3-1.020200.1280.0:/opt/modulefiles/gcc/4.9.3:/opt/cray/craype/2.5.8/modulefiles/craype-interlagos:/opt/cray/craype/2.5.8/modulefiles/craype-network-gemini:/opt/cray/modulefiles/craype/2.5.8:/opt/cray/modulefiles/cray-mpich/7.5.0:/opt/modulefiles/torque/6.0.4:/opt/modulefiles/moab/9.0.2.TAS7-1490665589_71674cf-sles11:/opt/modulefiles/java/jdk1.8.0_51:/usr/local/modulefiles/globus/5.2.5:/usr/local/modulefiles/gsissh/6.2p2:/opt/modulefiles/xalt/0.7.6.local:/sw/xe/modulefiles/scripts:/sw/xe/modulefiles/user-paths:/opt/cray/modulefiles/cray-libsci/16.11.1:/opt/cray/gem/modulefiles/udreg/2.3.2-1.0502.10518.2.17.gem:/opt/cray/gem/modulefiles/ugni/6.0-1.0502.10863.8.28.gem:/opt/cray/gem/modulefiles/pmi/5.0.10-1.0000.11050.179.3.gem:/opt/cray/gem/modulefiles/dmapp/7.0.1-1.0502.11080.8.74.gem:/opt/cray/gem/modulefiles/gni-headers/4.0-1.0502.10859.7.8.gem:/opt/cray/gem/modulefiles/xpmem/0.1-2.0502.64982.5.3.gem:/opt/cray/gem/modulefiles/dvs/2.5_0.9.0-1.0502.2188.1.113.gem:/opt/cray/gem/modulefiles/alps/5.2.4-2.0502.9774.31.12.gem:/opt/cray/gem/modulefiles/rca/1.0.0-2.0502.60530.1.63.gem:/opt/cray/modulefiles/atp/2.0.4:/opt/cray/modulefiles/PrgEnv-gnu/5.2.82:/opt/cray/modulefiles/cudatoolkit/7.5.18-1.0502.10743.2.1 ;export _LMFILES_; === RUNNING /u/system/bwjenkins/bin/myenv.sh === HOST=h2ologin3 HOME=/u/system/bwjenkins SCRATCH=/u/system/bwjenkins/scratch ls SCRATCH: mdtest rmg-spp rmg-spp-topo setsm_benchmark SPP-PPM stripe8 sw_config SWTools-BWCI sw_workdir tests wrf wrf-spp my id: uid=49294(bwjenkins) gid=16228(BW_System) groups=16228(BW_System),999(login_user),1000(ie_user),1001(hpss_user),1009(swtools),16237(jnd) Currently Loaded Modulefiles: 1) modules/3.2.10.4 2) eswrap/1.3.3-1.020200.1280.0 3) gcc/4.9.3 4) craype-interlagos 5) craype-network-gemini 6) craype/2.5.8 7) cray-mpich/7.5.0 8) torque/6.0.4 9) moab/9.0.2.TAS7-1490665589_71674cf-sles11 10) java/jdk1.8.0_51 11) globus/5.2.5 12) gsissh/6.2p2 13) xalt/0.7.6.local 14) scripts 15) user-paths 16) cray-libsci/16.11.1 17) udreg/2.3.2-1.0502.10518.2.17.gem 18) ugni/6.0-1.0502.10863.8.28.gem 19) pmi/5.0.10-1.0000.11050.179.3.gem 20) dmapp/7.0.1-1.0502.11080.8.74.gem 21) gni-headers/4.0-1.0502.10859.7.8.gem 22) xpmem/0.1-2.0502.64982.5.3.gem 23) dvs/2.5_0.9.0-1.0502.2188.1.113.gem 24) alps/5.2.4-2.0502.9774.31.12.gem 25) rca/1.0.0-2.0502.60530.1.63.gem 26) atp/2.0.4 27) PrgEnv-gnu/5.2.82 28) cudatoolkit/7.5.18-1.0502.10743.2.1 make: `pmcl3d' is up to date. 1 #!/bin/bash -login 2 #PBS -N awp-odc_gpu_small 3 #PBS -l walltime=00:30:00,nodes=4:ppn=1:xk 4 #PBS -j eo 5 6 module unload darshan 7 module swap $( module list | grep -o PrgEnv-.*$ ) PrgEnv-gnu 8 module load cudatoolkit 9 10 mkdir $PBS_O_WORKDIR/$PBS_JOBID 11 cd $PBS_O_WORKDIR/$PBS_JOBID 12 13 ln -fs ../../src/pmcl3d 14 mkdir input 15 mkdir input_rst 16 mkdir output_ckp 17 mkdir output_sfc 18 cp ../../input/source input/. 19 lfs setstripe -s 600m -c 4 input/mesh 20 cp ../../input/mesh input/. 21 lfs setstripe -s 3m -c 4 output_sfc 22 lfs setstripe -s 600m -c 1 input_rst 23 24 start_time=$(date +%s) 25 26 time aprun -n $PBS_NP -N1 ./pmcl3d \ 27 -X 320 -Y 320 -Z 2048 -x 2 -y 2 \ 28 --TMAX 4.0 --DH 40.0 --DT 0.002 \ 29 --NSRC 1 --NST 101 \ 30 --IFAULT 1 --MEDIASTART 2 --READ_STEP 101 --WRITE_STEP 100 \ 31 --NSKPX 2 --NSKPY 2 --NTISKP 20 -c output_ckp/ckp -o output_sfc --INSRC input/source --INVEL input/mesh 32 33 end_time=$(date +%s) 34 echo Completed in $(( end_time - start_time )) seconds 35 INFO: Job submitted to account: jnc echo MYJOBID=$MYJOBID ~/bin//wait_for.sh $MYJOBID MYJOBID=8250344.bw === RUNNING /u/system/bwjenkins/bin//wait_for.sh === Tue Feb 27 13:44:13 CST 2018: waiting for 8250344.bw to finish Tue Feb 27 13:49:13 CST 2018: waiting for 8250344.bw to finish Tue Feb 27 13:54:13 CST 2018: waiting for 8250344.bw to finish Tue Feb 27 13:59:13 CST 2018: waiting for 8250344.bw to finish qstat: Unknown Job Id Error 8250344.bw tail -50 awp-odc_gpu_small.e* > ~/awp-odc-spp/stderr.txt MYREALMIN=`grep real awp-odc_gpu_small.e* | cut -dl -f2 | cut -dm -f1` grep real awp-odc_gpu_small.e* | cut -dl -f2 | cut -dm -f1 echo "YVALUE=$MYREALMIN" > ~/awp-odc-spp/awp.realtime cat -n ~/awp-odc-spp/awp.realtime 8250344.bw finished 1 YVALUE= 6 [SSH] exit-status: 0 [awp-odc-spp] $ /bin/sh -xe /tmp/jenkins4215698645668208466.sh + rm -rf /var/lib/jenkins/workspace/awp-odc-spp/awp.realtime /var/lib/jenkins/workspace/awp-odc-spp/stderr.txt + scp 'BW_REMOTE:~/awp-odc-spp/*.txt' /var/lib/jenkins/workspace/awp-odc-spp Access by OTP or Two Factor Certificate Authority only. Use myproxy-logon -s tfca.ncsa.illinois.edu -p 7512 for gsissh access. gsissh or ssh -o PreferredAuthentications=keyboard-interactive for otp access. 02/08/18: H2ologin1 is access restricted and removed from the login pool for maintenance. Please use bw.ncsa.illinois.edu or h2ologin.ncsa.illinois.edu to access the available login nodes. Current login sessions that remain will be terminated 02/09/18 at 1PM and the system rebooted. Blue Waters Admin Team + scp 'BW_REMOTE:~/awp-odc-spp/awp.realtime' /var/lib/jenkins/workspace/awp-odc-spp Access by OTP or Two Factor Certificate Authority only. Use myproxy-logon -s tfca.ncsa.illinois.edu -p 7512 for gsissh access. gsissh or ssh -o PreferredAuthentications=keyboard-interactive for otp access. 02/08/18: H2ologin1 is access restricted and removed from the login pool for maintenance. Please use bw.ncsa.illinois.edu or h2ologin.ncsa.illinois.edu to access the available login nodes. Current login sessions that remain will be terminated 02/09/18 at 1PM and the system rebooted. Blue Waters Admin Team Recording plot data Saving plot series data from: /var/lib/jenkins/workspace/awp-odc-spp/awp.realtime Finished: SUCCESS