![[Jenkins]](/static/755cd4af/images/svgs/logo.svg){kind=logo}
Started by user arnoldg Building on master in workspace /var/lib/jenkins/workspace/awp-odc-spp-jyc executing script: set -v #rm -rf awp-odc-spp mkdir awp-odc-spp cd awp-odc-spp # tar xvfz /projects/bluewaters/benchmarking/SPP/AWP-ODC/awp-odc_gpu.tgz mkdir ~/bin #cp ~arnoldg/wait_for.sh ~/bin #cp ~arnoldg/myenv.sh ~/bin tar xvfz ~arnoldg/awp-odc_gpu.tgz cd awp-odc-os/src/ module unload darshan module swap $( module list | grep -o PrgEnv-.*$ ) PrgEnv-gnu #module unload PrgEnv-gnu # module use /u/vendor/cray/dalton/modulefiles # module load PrgEnv/gnu-7.3.0-cuda-9.1 module load cudatoolkit #module unload gcc #module load gcc/4.9.3 ~/bin/myenv.sh # for topomapping run: # module load topomapping # cp ~/awp-odc-spp/Makefile.bluewaters.gnu.topomapping Makefile.bluewaters.gnu make -f Makefile.bluewaters.gnu clean make -f Makefile.bluewaters.gnu cd ~/awp-odc-spp/awp-odc-os/run cat run_small.pbs | sed -e "s/aprun/time aprun/g" > runs.pbs cp runs.pbs run_small.pbs cat -n run_small.pbs rm awp-odc_gpu_small.* MYJOBID=`qsub run_small.pbs` echo MYJOBID=$MYJOBID ~/bin//wait_for.sh $MYJOBID tail -50 awp-odc_gpu_small.e* > ~/awp-odc-spp/stderr.txt MYREALMIN=`grep real awp-odc_gpu_small.e* | cut -dl -f2 | cut -dm -f1` echo "YVALUE=$MYREALMIN" > ~/awp-odc-spp/awp.realtime cat -n ~/awp-odc-spp/awp.realtime #rm -rf awp-odc-spp mkdir awp-odc-spp mkdir: cannot create directory `awp-odc-spp': File exists cd awp-odc-spp # tar xvfz /projects/bluewaters/benchmarking/SPP/AWP-ODC/awp-odc_gpu.tgz mkdir ~/bin mkdir: cannot create directory `/u/system/bwjenkins/bin': File exists #cp ~arnoldg/wait_for.sh ~/bin #cp ~arnoldg/myenv.sh ~/bin tar xvfz ~arnoldg/awp-odc_gpu.tgz awp-odc-os/ awp-odc-os/input/ awp-odc-os/input/source awp-odc-os/input/mesh cd awp-odc-os/src/ module unload darshan module swap $( module list | grep -o PrgEnv-.*$ ) PrgEnv-gnu module list | grep -o PrgEnv-.*$ awp-odc-os/AUTHORS awp-odc-os/run/ awp-odc-os/run/run.bluewaters.gnu.pbs awp-odc-os/run/run_medium.pbs awp-odc-os/run/env_clean.sh awp-odc-os/run/env.sh awp-odc-os/run/run_small.pbs awp-odc-os/README.md awp-odc-os/LICENSE awp-odc-os/submodules/ awp-odc-os/submodules/wiki/ awp-odc-os/src/ awp-odc-os/src/cerjan.c awp-odc-os/src/kernel.cu awp-odc-os/src/Makefile.bluewaters.cray awp-odc-os/src/Makefile.bluewaters.gnu awp-odc-os/src/grid.c awp-odc-os/src/kernel.h awp-odc-os/src/pmcl3d.c awp-odc-os/src/command.c awp-odc-os/src/io.c awp-odc-os/src/swap.c awp-odc-os/src/mesh.c awp-odc-os/src/pmcl3d.h awp-odc-os/src/source.c awp-odc-os/src/pmcl3d_cons.h awp-odc-os/CHANGELOG ${MODULESHOME}/bin/modulecmd bash $* LD_LIBRARY_PATH=/sw/EasyBuild/software/OpenSSL/1.0.2m/lib:/usr/local/globus-6.0/lib ;export LD_LIBRARY_PATH;LOADEDMODULES=modules/3.2.10.5:nodestat/2.2-1.0502.60539.1.31.gem:sdb/1.1-1.0502.63652.4.27.gem:alps/5.2.4-2.0502.9774.31.12.gem:lustre-cray_gem_s/2.5_3.0.101_0.46.1_1.0502.8871.44.1-1.0502.21728.75.3:udreg/2.3.2-1.0502.10518.2.17.gem:ugni/6.0-1.0502.10863.8.28.gem:gni-headers/4.0-1.0502.10859.9.26.gem:dmapp/7.0.1-1.0502.11080.8.74.gem:xpmem/0.1-2.0502.64982.7.26.gem:hss-llm/7.2.0:Base-opts/1.0.2-1.0502.60680.2.4.gem:cce/8.7.7:craype-network-gemini:craype/2.5.16:cray-libsci/18.12.1:pmi/5.0.11:rca/1.0.0-2.0502.60530.1.63.gem:atp/2.1.0:PrgEnv-cray/5.2.82:moab/9.1.2.h2-sles11:torque/6.1.2:craype-interlagos:cray-mpich/7.7.4:java/jdk1.8.0_51:globus/6.0:gsissh/7.5p1b:xalt/0.7.6.local:scripts:OpenSSL/1.0.2m:cURL/7.59.0:git/2.17.0:wget/1.19.4:user-paths ;export LOADEDMODULES;MANPATH=/sw/EasyBuild/software/wget/1.19.4/share/man:/sw/EasyBuild/software/cURL/7.59.0/share/man:/opt/java/jdk1.8.0_51/man:/usr/local/globus-6.0/man:/opt/cray/mpt/7.7.4/gni/man/mpich:/opt/torque/6.1.2/share/man:/opt/moab/9.1.2.h2/share/man/:/opt/cray/atp/2.1.0/man:/opt/cray/pmi/5.0.11/man:/opt/cray/libsci/18.12.1/man:/opt/cray/man/csmlversion:/opt/cray/craype/2.5.16/man:/opt/cray/cce/8.7.7/man:/opt/cray/llm/default/man:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.46.1_1.0502.8871.44.1-1.0502.21728.75.3/man:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/man:/opt/modules/3.2.10.5/share/man:/usr/local/man:/usr/share/man:/usr/man:/opt/cray/share/man:/opt/cray/man:/u/system/bwjenkins/man ;export MANPATH;PATH=/sw/EasyBuild/software/wget/1.19.4/bin:/sw/EasyBuild/software/git/2.17.0/bin:/sw/EasyBuild/software/cURL/7.59.0/bin:/sw/EasyBuild/software/OpenSSL/1.0.2m/bin:/sw/user/scripts:/sw/admin/scripts:/opt/xalt/0.7.6/sles11.3/libexec:/opt/xalt/0.7.6/sles11.3/bin:/usr/local/gsi-openssh-7.5p1b/bin:/usr/local/gsi-openssh-7.5p1b/libexec:/opt/java/jdk1.8.0_51/bin:/usr/local/globus-6.0/bin:/usr/local/globus-6.0/sbin:/opt/cray/mpt/7.7.4/gni/bin:/opt/torque/6.1.2/sbin:/opt/torque/6.1.2/bin:/opt/moab/9.1.2.h2/bin:/opt/moab/9.1.2.h2/sbin:/opt/cray/rca/1.0.0-2.0502.60530.1.63.gem/bin:/opt/cray/craype/2.5.16/bin:/opt/cray/cce/8.7.7/binutils/x86_64/x86_64-pc-linux-gnu/bin:/opt/cray/cce/8.7.7/binutils/cross/x86_64-aarch64/aarch64-linux-gnu/../bin:/opt/cray/cce/8.7.7/utils/x86_64/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.64982.7.26.gem/bin:/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/bin:/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.46.1_1.0502.8871.44.1-1.0502.21728.75.3/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.46.1_1.0502.8871.44.1-1.0502.21728.75.3/bin:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/sbin:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/bin:/opt/cray/sdb/1.1-1.0502.63652.4.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.60539.1.31.gem/bin:/opt/modules/3.2.10.5/bin:/u/system/bwjenkins/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin ;export PATH;PE_PKGCONFIG_LIBS=mpich:AtpSigHandler:cray-rca:libsci_mpi:libsci ;export PE_PKGCONFIG_LIBS;PKG_CONFIG_PATH=/sw/EasyBuild/software/cURL/7.59.0/lib/pkgconfig:/sw/EasyBuild/software/OpenSSL/1.0.2m/lib/pkgconfig:/opt/cray/rca/1.0.0-2.0502.60530.1.63.gem/lib64/pkgconfig:/opt/cray/pmi/5.0.11/lib64/pkgconfig:/opt/cray/craype/2.5.16/pkg-config:/opt/cray/iobuf/2.0.8/lib/pkgconfig:/opt/cray/xpmem/0.1-2.0502.64982.7.26.gem/lib64/pkgconfig:/opt/cray/dmapp/7.0.1-1.0502.11080.8.74.gem/lib64/pkgconfig:/opt/cray/gni-headers/4.0-1.0502.10859.9.26.gem/lib64/pkgconfig:/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/lib64/pkgconfig:/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/lib64/pkgconfig:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/lib64/pkgconfig:/opt/cray/atp/2.1.0/lib/pkgconfig ;export PKG_CONFIG_PATH;_LMFILES_=/opt/modulefiles/modules/3.2.10.5:/opt/cray/gem/modulefiles/nodestat/2.2-1.0502.60539.1.31.gem:/opt/cray/gem/modulefiles/sdb/1.1-1.0502.63652.4.27.gem:/opt/cray/gem/modulefiles/alps/5.2.4-2.0502.9774.31.12.gem:/opt/cray/modulefiles/lustre-cray_gem_s/2.5_3.0.101_0.46.1_1.0502.8871.44.1-1.0502.21728.75.3:/opt/cray/gem/modulefiles/udreg/2.3.2-1.0502.10518.2.17.gem:/opt/cray/gem/modulefiles/ugni/6.0-1.0502.10863.8.28.gem:/opt/cray/gem/modulefiles/gni-headers/4.0-1.0502.10859.9.26.gem:/opt/cray/gem/modulefiles/dmapp/7.0.1-1.0502.11080.8.74.gem:/opt/cray/gem/modulefiles/xpmem/0.1-2.0502.64982.7.26.gem:/opt/modulefiles/hss-llm/7.2.0:/opt/modulefiles/Base-opts/1.0.2-1.0502.60680.2.4.gem:/opt/modulefiles/cce/8.7.7:/opt/cray/craype/2.5.16/modulefiles/craype-network-gemini:/opt/cray/modulefiles/craype/2.5.16:/opt/cray/modulefiles/cray-libsci/18.12.1:/opt/cray/gem/modulefiles/pmi/5.0.11:/opt/cray/gem/modulefiles/rca/1.0.0-2.0502.60530.1.63.gem:/opt/cray/modulefiles/atp/2.1.0:/opt/cray/modulefiles/PrgEnv-cray/5.2.82:/opt/modulefiles/moab/9.1.2.h2-sles11:/opt/modulefiles/torque/6.1.2:/opt/cray/craype/2.5.16/modulefiles/craype-interlagos:/opt/cray/modulefiles/cray-mpich/7.7.4:/opt/modulefiles/java/jdk1.8.0_51:/usr/local/modulefiles/globus/6.0:/usr/local/modulefiles/gsissh/7.5p1b:/opt/modulefiles/xalt/0.7.6.local:/sw/xe/modulefiles/scripts:/sw/EasyBuild/modules/all/OpenSSL/1.0.2m:/sw/EasyBuild/modules/all/cURL/7.59.0:/sw/EasyBuild/modules/all/git/2.17.0:/sw/EasyBuild/modules/all/wget/1.19.4:/sw/xe/modulefiles/user-paths ;export _LMFILES_;unset DARSHAN_LOGPATH;unset EBDEVELGNUPLOT;unset EBROOTGNUPLOT;unset EBVERSIONGNUPLOT; ${MODULESHOME}/bin/modulecmd bash $* #module unload PrgEnv-gnu # module use /u/vendor/cray/dalton/modulefiles # module load PrgEnv/gnu-7.3.0-cuda-9.1 module load cudatoolkit ALLINEA_QUEUE_DLL=/opt/cray/mpt/7.7.4/gni/mpich-gnu/5.1/lib/libtvmpich.so.3.0.1 ;export ALLINEA_QUEUE_DLL;ATP_HOME=/opt/cray/atp/2.1.0 ;export ATP_HOME;ATP_IGNORE_SIGTERM=1 ;export ATP_IGNORE_SIGTERM;ATP_MRNET_COMM_PATH=/opt/cray/atp/2.1.0/libexec/atp_mrnet_commnode_wrapper ;export ATP_MRNET_COMM_PATH;ATP_POST_LINK_OPTS=-Wl,-L/opt/cray/atp/2.1.0/libApp/\ ;export ATP_POST_LINK_OPTS;CRAYOS_VERSION=5.2.82 ;export CRAYOS_VERSION;CRAYPE_DIR=/opt/cray/craype/2.5.16 ;export CRAYPE_DIR;CRAYPE_NETWORK_TARGET=gemini ;export CRAYPE_NETWORK_TARGET;CRAYPE_VERSION=2.5.16 ;export CRAYPE_VERSION;CRAY_CPU_TARGET=interlagos ;export CRAY_CPU_TARGET;CRAY_LD_LIBRARY_PATH=/opt/cray/pmi/5.0.11/lib64:/opt/cray/libsci/18.12.1/GNU/5.1/x86_64/lib:/opt/cray/mpt/7.7.4/gni/mpich-gnu/5.1/lib:/opt/cray/xpmem/0.1-2.0502.64982.7.26.gem/lib64:/opt/cray/dmapp/7.0.1-1.0502.11080.8.74.gem/lib64:/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/lib64:/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/lib64:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/lib64 ;export CRAY_LD_LIBRARY_PATH;CRAY_LIBSCI_BASE_DIR=/opt/cray/libsci/18.12.1 ;export CRAY_LIBSCI_BASE_DIR;CRAY_LIBSCI_DIR=/opt/cray/libsci/18.12.1 ;export CRAY_LIBSCI_DIR;CRAY_LIBSCI_PREFIX_DIR=/opt/cray/libsci/18.12.1/GNU/5.1/x86_64 ;export CRAY_LIBSCI_PREFIX_DIR;CRAY_LIBSCI_VERSION=18.12.1 ;export CRAY_LIBSCI_VERSION;CRAY_MPICH2_DIR=/opt/cray/mpt/7.7.4/gni/mpich-gnu/5.1 ;export CRAY_MPICH2_DIR;CRAY_MPICH2_VER=7.7.4 ;export CRAY_MPICH2_VER;CRAY_MPICH_BASEDIR=/opt/cray/mpt/7.7.4/gni ;export CRAY_MPICH_BASEDIR;CRAY_MPICH_DIR=/opt/cray/mpt/7.7.4/gni/mpich-gnu/5.1 ;export CRAY_MPICH_DIR;CRAY_MPICH_ROOTDIR=/opt/cray/mpt/7.7.4 ;export CRAY_MPICH_ROOTDIR;CRAY_PMI_INCLUDE_OPTS=-I/opt/cray/pmi/5.0.11/include ;export CRAY_PMI_INCLUDE_OPTS;CRAY_PMI_POST_LINK_OPTS=-L/opt/cray/pmi/5.0.11/lib64 ;export CRAY_PMI_POST_LINK_OPTS;CRAY_PRE_COMPILE_OPTS=-hnetwork=gemini ;export CRAY_PRE_COMPILE_OPTS;CRAY_PRGENVGNU=loaded ;export CRAY_PRGENVGNU;DMAPP_ABORT_ON_ERROR=1 ;export DMAPP_ABORT_ON_ERROR;GCC_PATH=/opt/gcc/6.3.0 ;export GCC_PATH;GCC_VERSION=6.3.0 ;export GCC_VERSION;GNU_VERSION=6.3.0 ;export GNU_VERSION;INFOPATH=/opt/gcc/6.3.0/snos/share/info:/usr/local/info:/usr/share/info:/usr/info ;export INFOPATH;LD_LIBRARY_PATH=/opt/gcc/6.3.0/snos/lib64:/sw/EasyBuild/software/OpenSSL/1.0.2m/lib:/usr/local/globus-6.0/lib ;export LD_LIBRARY_PATH;LIBSCI_BASE_DIR=/opt/cray/libsci/18.12.1 ;export LIBSCI_BASE_DIR;LIBSCI_VERSION=18.12.1 ;export LIBSCI_VERSION;LOADEDMODULES=modules/3.2.10.5:nodestat/2.2-1.0502.60539.1.31.gem:sdb/1.1-1.0502.63652.4.27.gem:alps/5.2.4-2.0502.9774.31.12.gem:lustre-cray_gem_s/2.5_3.0.101_0.46.1_1.0502.8871.44.1-1.0502.21728.75.3:udreg/2.3.2-1.0502.10518.2.17.gem:ugni/6.0-1.0502.10863.8.28.gem:gni-headers/4.0-1.0502.10859.9.26.gem:dmapp/7.0.1-1.0502.11080.8.74.gem:xpmem/0.1-2.0502.64982.7.26.gem:hss-llm/7.2.0:Base-opts/1.0.2-1.0502.60680.2.4.gem:gcc/6.3.0:craype-network-gemini:craype-interlagos:craype/2.5.16:moab/9.1.2.h2-sles11:torque/6.1.2:cray-mpich/7.7.4:java/jdk1.8.0_51:globus/6.0:gsissh/7.5p1b:xalt/0.7.6.local:scripts:OpenSSL/1.0.2m:cURL/7.59.0:git/2.17.0:wget/1.19.4:user-paths:cray-libsci/18.12.1:pmi/5.0.11:atp/2.1.0:PrgEnv-gnu/5.2.82 ;export LOADEDMODULES;MANPATH=/opt/cray/atp/2.1.0/man:/opt/cray/pmi/5.0.11/man:/opt/cray/libsci/18.12.1/man:/opt/cray/man/csmlversion:/opt/gcc/6.3.0/snos/share/man:/sw/EasyBuild/software/wget/1.19.4/share/man:/sw/EasyBuild/software/cURL/7.59.0/share/man:/opt/java/jdk1.8.0_51/man:/usr/local/globus-6.0/man:/opt/cray/mpt/7.7.4/gni/man/mpich:/opt/torque/6.1.2/share/man:/opt/moab/9.1.2.h2/share/man/:/opt/cray/craype/2.5.16/man:/opt/cray/llm/default/man:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.46.1_1.0502.8871.44.1-1.0502.21728.75.3/man:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/man:/opt/modules/3.2.10.5/share/man:/usr/local/man:/usr/share/man:/usr/man:/opt/cray/share/man:/opt/cray/man:/u/system/bwjenkins/man ;export MANPATH;MODULEPATH=/sw/EasyBuild/modules/all:/sw/bw/modulefiles:/sw/xe/modulefiles:/usr/local/modulefiles:/opt/cray/craype/2.5.16/modulefiles:/opt/cray/gem/modulefiles:/opt/cray/modulefiles:/opt/modulefiles ;export MODULEPATH;MPICH_ABORT_ON_ERROR=1 ;export MPICH_ABORT_ON_ERROR;MPICH_DIR=/opt/cray/mpt/7.7.4/gni/mpich-gnu/5.1 ;export MPICH_DIR;PATH=/opt/gcc/6.3.0/bin:/sw/EasyBuild/software/wget/1.19.4/bin:/sw/EasyBuild/software/git/2.17.0/bin:/sw/EasyBuild/software/cURL/7.59.0/bin:/sw/EasyBuild/software/OpenSSL/1.0.2m/bin:/sw/user/scripts:/sw/admin/scripts:/opt/xalt/0.7.6/sles11.3/libexec:/opt/xalt/0.7.6/sles11.3/bin:/usr/local/gsi-openssh-7.5p1b/bin:/usr/local/gsi-openssh-7.5p1b/libexec:/opt/java/jdk1.8.0_51/bin:/usr/local/globus-6.0/bin:/usr/local/globus-6.0/sbin:/opt/cray/mpt/7.7.4/gni/bin:/opt/torque/6.1.2/sbin:/opt/torque/6.1.2/bin:/opt/moab/9.1.2.h2/bin:/opt/moab/9.1.2.h2/sbin:/opt/cray/craype/2.5.16/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.64982.7.26.gem/bin:/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/bin:/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.46.1_1.0502.8871.44.1-1.0502.21728.75.3/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.46.1_1.0502.8871.44.1-1.0502.21728.75.3/bin:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/sbin:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/bin:/opt/cray/sdb/1.1-1.0502.63652.4.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.60539.1.31.gem/bin:/opt/modules/3.2.10.5/bin:/u/system/bwjenkins/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin ;export PATH;PE_ACML_DEFAULT_DIR_CRAY=gfortran64 ;export PE_ACML_DEFAULT_DIR_CRAY;PE_ACML_DEFAULT_DIR_DEFAULT64=_int64 ;export PE_ACML_DEFAULT_DIR_DEFAULT64;PE_ACML_DEFAULT_DIR_GNU=gfortran64 ;export PE_ACML_DEFAULT_DIR_GNU;PE_ACML_DEFAULT_DIR_INTEL=ifort64 ;export PE_ACML_DEFAULT_DIR_INTEL;PE_ACML_DEFAULT_DIR_OPENMP=_mp ;export PE_ACML_DEFAULT_DIR_OPENMP;PE_ACML_DEFAULT_DIR_PGI=pgi64 ;export PE_ACML_DEFAULT_DIR_PGI;PE_ACML_DEFAULT_TARGET_interlagos=_fma4 ;export PE_ACML_DEFAULT_TARGET_interlagos;PE_ACML_DEFAULT_TARGET_x86_64= ;export PE_ACML_DEFAULT_TARGET_x86_64;PE_ACML_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/acml/5.3.1/@PE_ACML_DEFAULT_DIR@@PE_ACML_DEFAULT_TARGET@@PE_ACML_DEFAULT_DIR_OPENMP@@PE_ACML_DEFAULT_DIR_DEFAULT64@/lib/pkgconfig ;export PE_ACML_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_CRAY_DEFAULT_FIXED_PKGCONFIG_PATH=/opt/cray/parallel-netcdf/1.8.1.4/CRAY/8.6/lib/pkgconfig:/opt/cray/netcdf-hdf5parallel/4.4.1.1/CRAY/8.3/lib/pkgconfig:/opt/cray/netcdf/4.4.1.1/CRAY/8.3/lib/pkgconfig:/opt/cray/hdf5-parallel/1.10.0.1/CRAY/8.3/lib/pkgconfig:/opt/cray/hdf5/1.10.0.1/CRAY/8.3/lib/pkgconfig:/opt/cray/ga/5.3.0.9/CRAY/8.6/lib/pkgconfig ;export PE_CRAY_DEFAULT_FIXED_PKGCONFIG_PATH;PE_CXX_PKGCONFIG_LIBS=mpichcxx ;export PE_CXX_PKGCONFIG_LIBS;PE_ENV=GNU ;export PE_ENV;PE_FFTW_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH ;export PE_FFTW_DEFAULT_REQUIRED_PRODUCTS;PE_FFTW_DEFAULT_TARGET_abudhabi=abudhabi ;export PE_FFTW_DEFAULT_TARGET_abudhabi;PE_FFTW_DEFAULT_TARGET_broadwell=broadwell ;export PE_FFTW_DEFAULT_TARGET_broadwell;PE_FFTW_DEFAULT_TARGET_haswell=haswell ;export PE_FFTW_DEFAULT_TARGET_haswell;PE_FFTW_DEFAULT_TARGET_interlagos=interlagos ;export PE_FFTW_DEFAULT_TARGET_interlagos;PE_FFTW_DEFAULT_TARGET_ivybridge=ivybridge ;export PE_FFTW_DEFAULT_TARGET_ivybridge;PE_FFTW_DEFAULT_TARGET_mic_knl=mic_knl ;export PE_FFTW_DEFAULT_TARGET_mic_knl;PE_FFTW_DEFAULT_TARGET_sandybridge=sandybridge ;export PE_FFTW_DEFAULT_TARGET_sandybridge;PE_FFTW_DEFAULT_TARGET_share=share ;export PE_FFTW_DEFAULT_TARGET_share;PE_FFTW_DEFAULT_TARGET_x86_64=x86_64 ;export PE_FFTW_DEFAULT_TARGET_x86_64;PE_FFTW_DEFAULT_TARGET_x86_skylake=x86_skylake ;export PE_FFTW_DEFAULT_TARGET_x86_skylake;PE_FFTW_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/fftw/3.3.6.5/@PE_FFTW_DEFAULT_TARGET@/lib/pkgconfig ;export PE_FFTW_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_FORTRAN_PKGCONFIG_LIBS=mpichf90 ;export PE_FORTRAN_PKGCONFIG_LIBS;PE_GA_DEFAULT_FIXED_PRGENV=CRAY\ PGI\ INTEL ;export PE_GA_DEFAULT_FIXED_PRGENV;PE_GA_DEFAULT_GENCOMPILERS_GNU=8.2\ 7.3\ 5.3 ;export PE_GA_DEFAULT_GENCOMPILERS_GNU;PE_GA_DEFAULT_GENCOMPS_GNU=82\ 73\ 53 ;export PE_GA_DEFAULT_GENCOMPS_GNU;PE_GA_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/ga/5.3.0.9/@PRGENV@/@PE_GA_DEFAULT_GENCOMPS@/lib/pkgconfig ;export PE_GA_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_GA_DEFAULT_VOLATILE_PRGENV=GNU ;export PE_GA_DEFAULT_VOLATILE_PRGENV;PE_HDF5_DEFAULT_FIXED_PRGENV=CRAY\ PGI\ INTEL ;export PE_HDF5_DEFAULT_FIXED_PRGENV;PE_HDF5_DEFAULT_GENCOMPILERS_GNU=5.1\ 4.9 ;export PE_HDF5_DEFAULT_GENCOMPILERS_GNU;PE_HDF5_DEFAULT_GENCOMPS_GNU=51\ 49 ;export PE_HDF5_DEFAULT_GENCOMPS_GNU;PE_HDF5_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH ;export PE_HDF5_DEFAULT_REQUIRED_PRODUCTS;PE_HDF5_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/hdf5/1.10.0.1/@PRGENV@/@PE_HDF5_DEFAULT_GENCOMPS@/lib/pkgconfig ;export PE_HDF5_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_HDF5_DEFAULT_VOLATILE_PRGENV=GNU ;export PE_HDF5_DEFAULT_VOLATILE_PRGENV;PE_HDF5_PARALLEL_DEFAULT_FIXED_PRGENV=CRAY\ PGI\ INTEL ;export PE_HDF5_PARALLEL_DEFAULT_FIXED_PRGENV;PE_HDF5_PARALLEL_DEFAULT_GENCOMPILERS_GNU=5.1\ 4.9 ;export PE_HDF5_PARALLEL_DEFAULT_GENCOMPILERS_GNU;PE_HDF5_PARALLEL_DEFAULT_GENCOMPS_GNU=51\ 49 ;export PE_HDF5_PARALLEL_DEFAULT_GENCOMPS_GNU;PE_HDF5_PARALLEL_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH ;export PE_HDF5_PARALLEL_DEFAULT_REQUIRED_PRODUCTS;PE_HDF5_PARALLEL_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/hdf5-parallel/1.10.0.1/@PRGENV@/@PE_HDF5_PARALLEL_DEFAULT_GENCOMPS@/lib/pkgconfig ;export PE_HDF5_PARALLEL_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_HDF5_PARALLEL_DEFAULT_VOLATILE_PRGENV=GNU ;export PE_HDF5_PARALLEL_DEFAULT_VOLATILE_PRGENV;PE_INTEL_DEFAULT_FIXED_PKGCONFIG_PATH=/opt/cray/parallel-netcdf/1.8.1.4/INTEL/18.0/lib/pkgconfig:/opt/cray/netcdf-hdf5parallel/4.4.1.1/INTEL/15.0/lib/pkgconfig:/opt/cray/netcdf/4.4.1.1/INTEL/15.0/lib/pkgconfig:/opt/cray/mpt/7.7.4/gni/mpich-intel/16.0/lib/pkgconfig:/opt/cray/hdf5-parallel/1.10.0.1/INTEL/15.0/lib/pkgconfig:/opt/cray/hdf5/1.10.0.1/INTEL/15.0/lib/pkgconfig:/opt/cray/ga/5.3.0.9/INTEL/18.0/lib/pkgconfig ;export PE_INTEL_DEFAULT_FIXED_PKGCONFIG_PATH;PE_LIBSCI_ACC_DEFAULT_GENCOMPILERS_CRAY_x86_64=8.3 ;export PE_LIBSCI_ACC_DEFAULT_GENCOMPILERS_CRAY_x86_64;PE_LIBSCI_ACC_DEFAULT_GENCOMPILERS_GNU_x86_64=4.9 ;export PE_LIBSCI_ACC_DEFAULT_GENCOMPILERS_GNU_x86_64;PE_LIBSCI_ACC_DEFAULT_GENCOMPS_CRAY_x86_64=83 ;export PE_LIBSCI_ACC_DEFAULT_GENCOMPS_CRAY_x86_64;PE_LIBSCI_ACC_DEFAULT_GENCOMPS_GNU_x86_64=49 ;export PE_LIBSCI_ACC_DEFAULT_GENCOMPS_GNU_x86_64;PE_LIBSCI_ACC_DEFAULT_NV_SUFFIX_nvidia20=nv20 ;export PE_LIBSCI_ACC_DEFAULT_NV_SUFFIX_nvidia20;PE_LIBSCI_ACC_DEFAULT_NV_SUFFIX_nvidia35=nv35 ;export PE_LIBSCI_ACC_DEFAULT_NV_SUFFIX_nvidia35;PE_LIBSCI_ACC_DEFAULT_PKGCONFIG_VARIABLES=PE_LIBSCI_ACC_DEFAULT_NV_SUFFIX_@accelerator@ ;export PE_LIBSCI_ACC_DEFAULT_PKGCONFIG_VARIABLES;PE_LIBSCI_ACC_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH:PE_LIBSCI ;export PE_LIBSCI_ACC_DEFAULT_REQUIRED_PRODUCTS;PE_LIBSCI_ACC_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/libsci_acc/16.03.1/@PRGENV@/@PE_LIBSCI_ACC_DEFAULT_GENCOMPS@/@PE_LIBSCI_ACC_DEFAULT_TARGET@/lib/pkgconfig ;export PE_LIBSCI_ACC_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_LIBSCI_ACC_DEFAULT_VOLATILE_PRGENV=CRAY\ GNU ;export PE_LIBSCI_ACC_DEFAULT_VOLATILE_PRGENV;PE_LIBSCI_DEFAULT_GENCOMPILERS_CRAY_x86_64=8.6 ;export PE_LIBSCI_DEFAULT_GENCOMPILERS_CRAY_x86_64;PE_LIBSCI_DEFAULT_GENCOMPILERS_GNU_x86_64=8.1\ 7.1\ 5.1 ;export PE_LIBSCI_DEFAULT_GENCOMPILERS_GNU_x86_64;PE_LIBSCI_DEFAULT_GENCOMPILERS_INTEL_x86_64=16.0 ;export PE_LIBSCI_DEFAULT_GENCOMPILERS_INTEL_x86_64;PE_LIBSCI_DEFAULT_GENCOMPILERS_PGI_x86_64=15.3 ;export PE_LIBSCI_DEFAULT_GENCOMPILERS_PGI_x86_64;PE_LIBSCI_DEFAULT_GENCOMPS_CRAY_x86_64=86 ;export PE_LIBSCI_DEFAULT_GENCOMPS_CRAY_x86_64;PE_LIBSCI_DEFAULT_GENCOMPS_GNU_x86_64=81\ 71\ 51 ;export PE_LIBSCI_DEFAULT_GENCOMPS_GNU_x86_64;PE_LIBSCI_DEFAULT_GENCOMPS_INTEL_x86_64=160 ;export PE_LIBSCI_DEFAULT_GENCOMPS_INTEL_x86_64;PE_LIBSCI_DEFAULT_GENCOMPS_PGI_x86_64=153 ;export PE_LIBSCI_DEFAULT_GENCOMPS_PGI_x86_64;PE_LIBSCI_DEFAULT_OMP_REQUIRES= ;export PE_LIBSCI_DEFAULT_OMP_REQUIRES;PE_LIBSCI_DEFAULT_OMP_REQUIRES_openmp=_mp ;export PE_LIBSCI_DEFAULT_OMP_REQUIRES_openmp;PE_LIBSCI_DEFAULT_PKGCONFIG_VARIABLES=PE_LIBSCI_DEFAULT_OMP_REQUIRES_@openmp@:PE_SCI_EXT_LIBPATH:PE_SCI_EXT_LIBNAME ;export PE_LIBSCI_DEFAULT_PKGCONFIG_VARIABLES;PE_LIBSCI_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH ;export PE_LIBSCI_DEFAULT_REQUIRED_PRODUCTS;PE_LIBSCI_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/libsci/18.12.1/@PRGENV@/@PE_LIBSCI_DEFAULT_GENCOMPS@/@PE_LIBSCI_DEFAULT_TARGET@/lib/pkgconfig ;export PE_LIBSCI_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_LIBSCI_DEFAULT_VOLATILE_PRGENV=CRAY\ GNU\ INTEL\ PGI ;export PE_LIBSCI_DEFAULT_VOLATILE_PRGENV;PE_LIBSCI_GENCOMPILERS_CRAY_x86_64=8.6 ;export PE_LIBSCI_GENCOMPILERS_CRAY_x86_64;PE_LIBSCI_GENCOMPILERS_GNU_x86_64=8.1\ 7.1\ 5.1 ;export PE_LIBSCI_GENCOMPILERS_GNU_x86_64;PE_LIBSCI_GENCOMPILERS_INTEL_x86_64=16.0 ;export PE_LIBSCI_GENCOMPILERS_INTEL_x86_64;PE_LIBSCI_GENCOMPILERS_PGI_x86_64=15.3 ;export PE_LIBSCI_GENCOMPILERS_PGI_x86_64;PE_LIBSCI_GENCOMPS_CRAY_x86_64=86 ;export PE_LIBSCI_GENCOMPS_CRAY_x86_64;PE_LIBSCI_GENCOMPS_GNU_x86_64=81\ 71\ 51 ;export PE_LIBSCI_GENCOMPS_GNU_x86_64;PE_LIBSCI_GENCOMPS_INTEL_x86_64=160 ;export PE_LIBSCI_GENCOMPS_INTEL_x86_64;PE_LIBSCI_GENCOMPS_PGI_x86_64=153 ;export PE_LIBSCI_GENCOMPS_PGI_x86_64;PE_LIBSCI_MODULE_NAME=cray-libsci/18.12.1 ;export PE_LIBSCI_MODULE_NAME;PE_LIBSCI_OMP_REQUIRES= ;export PE_LIBSCI_OMP_REQUIRES;PE_LIBSCI_OMP_REQUIRES_openmp=_mp ;export PE_LIBSCI_OMP_REQUIRES_openmp;PE_LIBSCI_PKGCONFIG_LIBS=libsci_mpi:libsci ;export PE_LIBSCI_PKGCONFIG_LIBS;PE_LIBSCI_PKGCONFIG_VARIABLES=PE_LIBSCI_OMP_REQUIRES_@openmp@:PE_SCI_EXT_LIBPATH:PE_SCI_EXT_LIBNAME ;export PE_LIBSCI_PKGCONFIG_VARIABLES;PE_LIBSCI_REQUIRED_PRODUCTS=PE_MPICH ;export PE_LIBSCI_REQUIRED_PRODUCTS;PE_LIBSCI_VOLATILE_PKGCONFIG_PATH=/opt/cray/libsci/18.12.1/@PRGENV@/@PE_LIBSCI_GENCOMPS@/@PE_LIBSCI_TARGET@/lib/pkgconfig ;export PE_LIBSCI_VOLATILE_PKGCONFIG_PATH;PE_LIBSCI_VOLATILE_PRGENV=CRAY\ GNU\ INTEL\ PGI ;export PE_LIBSCI_VOLATILE_PRGENV;PE_MPICH_ALTERNATE_LIBS_dpm=_dpm ;export PE_MPICH_ALTERNATE_LIBS_dpm;PE_MPICH_ALTERNATE_LIBS_multithreaded=_mt ;export PE_MPICH_ALTERNATE_LIBS_multithreaded;PE_MPICH_CXX_PKGCONFIG_LIBS=mpichcxx ;export PE_MPICH_CXX_PKGCONFIG_LIBS;PE_MPICH_DEFAULT_DIR_CRAY_DEFAULT64=64 ;export PE_MPICH_DEFAULT_DIR_CRAY_DEFAULT64;PE_MPICH_DEFAULT_DIR_PGI_DEFAULT64=64 ;export PE_MPICH_DEFAULT_DIR_PGI_DEFAULT64;PE_MPICH_DEFAULT_FIXED_PRGENV=INTEL ;export PE_MPICH_DEFAULT_FIXED_PRGENV;PE_MPICH_DEFAULT_GENCOMPILERS_CRAY=8.6 ;export PE_MPICH_DEFAULT_GENCOMPILERS_CRAY;PE_MPICH_DEFAULT_GENCOMPILERS_GNU=8.2\ 7.1\ 5.1 ;export PE_MPICH_DEFAULT_GENCOMPILERS_GNU;PE_MPICH_DEFAULT_GENCOMPILERS_PGI=15.3 ;export PE_MPICH_DEFAULT_GENCOMPILERS_PGI;PE_MPICH_DEFAULT_GENCOMPS_CRAY=86 ;export PE_MPICH_DEFAULT_GENCOMPS_CRAY;PE_MPICH_DEFAULT_GENCOMPS_GNU=82\ 71\ 51 ;export PE_MPICH_DEFAULT_GENCOMPS_GNU;PE_MPICH_DEFAULT_GENCOMPS_PGI=153 ;export PE_MPICH_DEFAULT_GENCOMPS_PGI;PE_MPICH_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/mpt/7.7.4/gni/mpich-@PRGENV@@PE_MPICH_DEFAULT_DIR_DEFAULT64@/@PE_MPICH_DEFAULT_GENCOMPS@/lib/pkgconfig ;export PE_MPICH_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_MPICH_DEFAULT_VOLATILE_PRGENV=CRAY\ GNU\ PGI ;export PE_MPICH_DEFAULT_VOLATILE_PRGENV;PE_MPICH_FORTRAN_PKGCONFIG_LIBS=mpichf90 ;export PE_MPICH_FORTRAN_PKGCONFIG_LIBS;PE_MPICH_MODULE_NAME=cray-mpich ;export PE_MPICH_MODULE_NAME;PE_MPICH_NV_LIBS= ;export PE_MPICH_NV_LIBS;PE_MPICH_NV_LIBS_nvidia20=-lcudart ;export PE_MPICH_NV_LIBS_nvidia20;PE_MPICH_NV_LIBS_nvidia35=-lcudart ;export PE_MPICH_NV_LIBS_nvidia35;PE_MPICH_NV_LIBS_nvidia60=-lcudart ;export PE_MPICH_NV_LIBS_nvidia60;PE_MPICH_PKGCONFIG_LIBS=mpich ;export PE_MPICH_PKGCONFIG_LIBS;PE_MPICH_PKGCONFIG_VARIABLES=PE_MPICH_NV_LIBS_@accelerator@:PE_MPICH_ALTERNATE_LIBS_@multithreaded@:PE_MPICH_ALTERNATE_LIBS_@dpm@ ;export PE_MPICH_PKGCONFIG_VARIABLES;PE_MPICH_TARGET_VAR_nvidia20=-lcudart ;export PE_MPICH_TARGET_VAR_nvidia20;PE_MPICH_TARGET_VAR_nvidia35=-lcudart ;export PE_MPICH_TARGET_VAR_nvidia35;PE_NETCDF_DEFAULT_FIXED_PRGENV=CRAY\ PGI\ INTEL ;export PE_NETCDF_DEFAULT_FIXED_PRGENV;PE_NETCDF_DEFAULT_GENCOMPILERS_GNU=5.1\ 4.9 ;export PE_NETCDF_DEFAULT_GENCOMPILERS_GNU;PE_NETCDF_DEFAULT_GENCOMPS_GNU=51\ 49 ;export PE_NETCDF_DEFAULT_GENCOMPS_GNU;PE_NETCDF_DEFAULT_REQUIRED_PRODUCTS=PE_HDF5 ;export PE_NETCDF_DEFAULT_REQUIRED_PRODUCTS;PE_NETCDF_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/netcdf/4.4.1.1/@PRGENV@/@PE_NETCDF_DEFAULT_GENCOMPS@/lib/pkgconfig ;export PE_NETCDF_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_NETCDF_DEFAULT_VOLATILE_PRGENV=GNU ;export PE_NETCDF_DEFAULT_VOLATILE_PRGENV;PE_NETCDF_HDF5PARALLEL_DEFAULT_FIXED_PRGENV=CRAY\ PGI\ INTEL ;export PE_NETCDF_HDF5PARALLEL_DEFAULT_FIXED_PRGENV;PE_NETCDF_HDF5PARALLEL_DEFAULT_GENCOMPILERS_GNU=5.1\ 4.9 ;export PE_NETCDF_HDF5PARALLEL_DEFAULT_GENCOMPILERS_GNU;PE_NETCDF_HDF5PARALLEL_DEFAULT_GENCOMPS_GNU=51\ 49 ;export PE_NETCDF_HDF5PARALLEL_DEFAULT_GENCOMPS_GNU;PE_NETCDF_HDF5PARALLEL_DEFAULT_REQUIRED_PRODUCTS=PE_HDF5_PARALLEL:PE_MPICH ;export PE_NETCDF_HDF5PARALLEL_DEFAULT_REQUIRED_PRODUCTS;PE_NETCDF_HDF5PARALLEL_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/netcdf-hdf5parallel/4.4.1.1/@PRGENV@/@PE_NETCDF_HDF5PARALLEL_DEFAULT_GENCOMPS@/lib/pkgconfig ;export PE_NETCDF_HDF5PARALLEL_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_NETCDF_HDF5PARALLEL_DEFAULT_VOLATILE_PRGENV=GNU ;export PE_NETCDF_HDF5PARALLEL_DEFAULT_VOLATILE_PRGENV;PE_PAPI_DEFAULT_ACCELL_FAMILY_LIBS= ;export PE_PAPI_DEFAULT_ACCELL_FAMILY_LIBS;PE_PAPI_DEFAULT_ACCEL_FAMILY_LIBS_nvidia=,-lcupti,-lcudart,-lcuda ;export PE_PAPI_DEFAULT_ACCEL_FAMILY_LIBS_nvidia;PE_PAPI_DEFAULT_ACCEL_LIBS= ;export PE_PAPI_DEFAULT_ACCEL_LIBS;PE_PAPI_DEFAULT_ACCEL_LIBS_nvidia35=,-lcupti,-lcudart,-lcuda ;export PE_PAPI_DEFAULT_ACCEL_LIBS_nvidia35;PE_PAPI_DEFAULT_PKGCONFIG_VARIABLES=PE_PAPI_ACCEL_LIBS_@accelerator@\ PE_PAPI_ACCEL_FAMILY_LIBS_@accelerator_family@ ;export PE_PAPI_DEFAULT_PKGCONFIG_VARIABLES;PE_PARALLEL_NETCDF_DEFAULT_FIXED_PRGENV=CRAY\ PGI\ INTEL ;export PE_PARALLEL_NETCDF_DEFAULT_FIXED_PRGENV;PE_PARALLEL_NETCDF_DEFAULT_GENCOMPILERS_GNU=8.2\ 5.1 ;export PE_PARALLEL_NETCDF_DEFAULT_GENCOMPILERS_GNU;PE_PARALLEL_NETCDF_DEFAULT_GENCOMPS_GNU=82\ 51 ;export PE_PARALLEL_NETCDF_DEFAULT_GENCOMPS_GNU;PE_PARALLEL_NETCDF_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/parallel-netcdf/1.8.1.4/@PRGENV@/@PE_PARALLEL_NETCDF_DEFAULT_GENCOMPS@/lib/pkgconfig ;export PE_PARALLEL_NETCDF_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_PARALLEL_NETCDF_DEFAULT_VOLATILE_PRGENV=GNU ;export PE_PARALLEL_NETCDF_DEFAULT_VOLATILE_PRGENV;PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_haswell=8.6 ;export PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_haswell;PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_interlagos=8.6 ;export PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_interlagos;PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_mic_knl=8.6 ;export PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_mic_knl;PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_sandybridge=8.6 ;export PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_sandybridge;PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_skylake=8.6 ;export PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_skylake;PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_x86_64=8.6 ;export PE_PETSC_DEFAULT_GENCOMPILERS_CRAY_x86_64;PE_PETSC_DEFAULT_GENCOMPILERS_GNU_haswell=8.2\ 5.3 ;export PE_PETSC_DEFAULT_GENCOMPILERS_GNU_haswell;PE_PETSC_DEFAULT_GENCOMPILERS_GNU_interlagos=8.2\ 5.3 ;export PE_PETSC_DEFAULT_GENCOMPILERS_GNU_interlagos;PE_PETSC_DEFAULT_GENCOMPILERS_GNU_mic_knl=5.3 ;export PE_PETSC_DEFAULT_GENCOMPILERS_GNU_mic_knl;PE_PETSC_DEFAULT_GENCOMPILERS_GNU_sandybridge=8.2\ 5.3 ;export PE_PETSC_DEFAULT_GENCOMPILERS_GNU_sandybridge;PE_PETSC_DEFAULT_GENCOMPILERS_GNU_skylake=8.2\ 7.3 ;export PE_PETSC_DEFAULT_GENCOMPILERS_GNU_skylake;PE_PETSC_DEFAULT_GENCOMPILERS_GNU_x86_64=8.2\ 5.3 ;export PE_PETSC_DEFAULT_GENCOMPILERS_GNU_x86_64;PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_haswell=16.0 ;export PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_haswell;PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_interlagos=16.0 ;export PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_interlagos;PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_mic_knl=16.0 ;export PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_mic_knl;PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_sandybridge=16.0 ;export PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_sandybridge;PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_skylake=16.0 ;export PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_skylake;PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_x86_64=16.0 ;export PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_x86_64;PE_PETSC_DEFAULT_GENCOMPILERS_PGI_interlagos=17.10 ;export PE_PETSC_DEFAULT_GENCOMPILERS_PGI_interlagos;PE_PETSC_DEFAULT_GENCOMPILERS_PGI_x86_64=17.10 ;export PE_PETSC_DEFAULT_GENCOMPILERS_PGI_x86_64;PE_PETSC_DEFAULT_GENCOMPS_CRAY_haswell=86 ;export PE_PETSC_DEFAULT_GENCOMPS_CRAY_haswell;PE_PETSC_DEFAULT_GENCOMPS_CRAY_interlagos=86 ;export PE_PETSC_DEFAULT_GENCOMPS_CRAY_interlagos;PE_PETSC_DEFAULT_GENCOMPS_CRAY_mic_knl=86 ;export PE_PETSC_DEFAULT_GENCOMPS_CRAY_mic_knl;PE_PETSC_DEFAULT_GENCOMPS_CRAY_sandybridge=86 ;export PE_PETSC_DEFAULT_GENCOMPS_CRAY_sandybridge;PE_PETSC_DEFAULT_GENCOMPS_CRAY_skylake=86 ;export PE_PETSC_DEFAULT_GENCOMPS_CRAY_skylake;PE_PETSC_DEFAULT_GENCOMPS_CRAY_x86_64=86 ;export PE_PETSC_DEFAULT_GENCOMPS_CRAY_x86_64;PE_PETSC_DEFAULT_GENCOMPS_GNU_haswell=82\ 53 ;export PE_PETSC_DEFAULT_GENCOMPS_GNU_haswell;PE_PETSC_DEFAULT_GENCOMPS_GNU_interlagos=82\ 53 ;export PE_PETSC_DEFAULT_GENCOMPS_GNU_interlagos;PE_PETSC_DEFAULT_GENCOMPS_GNU_mic_knl=53 ;export PE_PETSC_DEFAULT_GENCOMPS_GNU_mic_knl;PE_PETSC_DEFAULT_GENCOMPS_GNU_sandybridge=82\ 53 ;export PE_PETSC_DEFAULT_GENCOMPS_GNU_sandybridge;PE_PETSC_DEFAULT_GENCOMPS_GNU_skylake=82\ 73 ;export PE_PETSC_DEFAULT_GENCOMPS_GNU_skylake;PE_PETSC_DEFAULT_GENCOMPS_GNU_x86_64=82\ 53 ;export PE_PETSC_DEFAULT_GENCOMPS_GNU_x86_64;PE_PETSC_DEFAULT_GENCOMPS_INTEL_haswell=160 ;export PE_PETSC_DEFAULT_GENCOMPS_INTEL_haswell;PE_PETSC_DEFAULT_GENCOMPS_INTEL_interlagos=160 ;export PE_PETSC_DEFAULT_GENCOMPS_INTEL_interlagos;PE_PETSC_DEFAULT_GENCOMPS_INTEL_mic_knl=160 ;export PE_PETSC_DEFAULT_GENCOMPS_INTEL_mic_knl;PE_PETSC_DEFAULT_GENCOMPS_INTEL_sandybridge=160 ;export PE_PETSC_DEFAULT_GENCOMPS_INTEL_sandybridge;PE_PETSC_DEFAULT_GENCOMPS_INTEL_skylake=160 ;export PE_PETSC_DEFAULT_GENCOMPS_INTEL_skylake;PE_PETSC_DEFAULT_GENCOMPS_INTEL_x86_64=160 ;export PE_PETSC_DEFAULT_GENCOMPS_INTEL_x86_64;PE_PETSC_DEFAULT_GENCOMPS_PGI_interlagos=1710 ;export PE_PETSC_DEFAULT_GENCOMPS_PGI_interlagos;PE_PETSC_DEFAULT_GENCOMPS_PGI_x86_64=1710 ;export PE_PETSC_DEFAULT_GENCOMPS_PGI_x86_64;PE_PETSC_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH:PE_LIBSCI:PE_HDF5_PARALLEL:PE_TPSL ;export PE_PETSC_DEFAULT_REQUIRED_PRODUCTS;PE_PETSC_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/petsc/3.9.3.0/complex/@PRGENV@/@PE_PETSC_DEFAULT_GENCOMPS@/@PE_PETSC_DEFAULT_TARGET@/lib/pkgconfig ;export PE_PETSC_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_PETSC_DEFAULT_VOLATILE_PRGENV=CRAY\ CRAY64\ GNU\ GNU64\ INTEL\ INTEL64\ PGI\ PGI64 ;export PE_PETSC_DEFAULT_VOLATILE_PRGENV;PE_PGI_DEFAULT_FIXED_PKGCONFIG_PATH=/opt/cray/parallel-netcdf/1.8.1.4/PGI/17.10/lib/pkgconfig:/opt/cray/netcdf-hdf5parallel/4.4.1.1/PGI/15.3/lib/pkgconfig:/opt/cray/netcdf/4.4.1.1/PGI/15.3/lib/pkgconfig:/opt/cray/hdf5-parallel/1.10.0.1/PGI/15.3/lib/pkgconfig:/opt/cray/hdf5/1.10.0.1/PGI/15.3/lib/pkgconfig:/opt/cray/ga/5.3.0.9/PGI/17.10/lib/pkgconfig ;export PE_PGI_DEFAULT_FIXED_PKGCONFIG_PATH;PE_PKGCONFIG_DEFAULT_PRODUCTS=PE_TRILINOS:PE_TPSL_64:PE_TPSL:PE_PETSC:PE_PARALLEL_NETCDF:PE_NETCDF_HDF5PARALLEL:PE_NETCDF:PE_MPICH:PE_LIBSCI_ACC:PE_LIBSCI:PE_HDF5_PARALLEL:PE_HDF5:PE_GA:PE_FFTW:PE_ACML ;export PE_PKGCONFIG_DEFAULT_PRODUCTS;PE_PKGCONFIG_LIBS=AtpSigHandler:libsci_mpi:libsci:mpich:gcc ;export PE_PKGCONFIG_LIBS;PE_PKGCONFIG_PRODUCTS=PE_LIBSCI:PE_MPICH ;export PE_PKGCONFIG_PRODUCTS;PE_PKGCONFIG_PRODUCTS_DEFAULT=PE_PAPI ;export PE_PKGCONFIG_PRODUCTS_DEFAULT;PE_PKG_CONFIG_PATH=/opt/cray/cti/1.0.4/lib/pkgconfig:/opt/cray/cti/1.0.1/lib/pkgconfig ;export PE_PKG_CONFIG_PATH;PE_PRODUCT_LIST=CRAY_PMI:CRAY_LIBSCI:GNU:GCC:CRAYPE:CRAYPE_INTERLAGOS:CRAY_LLM:CRAY_XPMEM:CRAY_DMAPP:CRAY_UGNI:CRAY_UDREG:CRAY_ALPS ;export PE_PRODUCT_LIST;PE_SMA_DEFAULT_COMPFLAG= ;export PE_SMA_DEFAULT_COMPFLAG;PE_SMA_DEFAULT_COMPFLAG_GNU=-fcray-pointer ;export PE_SMA_DEFAULT_COMPFLAG_GNU;PE_SMA_DEFAULT_DIR_CRAY_DEFAULT64=64 ;export PE_SMA_DEFAULT_DIR_CRAY_DEFAULT64;PE_SMA_DEFAULT_DIR_PGI_DEFAULT64=64 ;export PE_SMA_DEFAULT_DIR_PGI_DEFAULT64;PE_SMA_DEFAULT_PKGCONFIG_VARIABLES=PE_SMA_COMPFLAG_@prgenv@ ;export PE_SMA_DEFAULT_PKGCONFIG_VARIABLES;PE_SMA_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/mpt/7.7.4/gni/sma@PE_SMA_DEFAULT_DIR_DEFAULT64@/lib64/pkgconfig ;export PE_SMA_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_haswell=8.2 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_haswell;PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_interlagos=8.2 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_interlagos;PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_sandybridge=8.2 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_sandybridge;PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_x86_64=8.2 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_x86_64;PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_x86_skylake=8.2 ;export PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_x86_skylake;PE_TPSL_64_DEFAULT_GENCOMPS_GNU_haswell=82 ;export PE_TPSL_64_DEFAULT_GENCOMPS_GNU_haswell;PE_TPSL_64_DEFAULT_GENCOMPS_GNU_interlagos=82 ;export PE_TPSL_64_DEFAULT_GENCOMPS_GNU_interlagos;PE_TPSL_64_DEFAULT_GENCOMPS_GNU_sandybridge=82 ;export PE_TPSL_64_DEFAULT_GENCOMPS_GNU_sandybridge;PE_TPSL_64_DEFAULT_GENCOMPS_GNU_x86_64=82 ;export PE_TPSL_64_DEFAULT_GENCOMPS_GNU_x86_64;PE_TPSL_64_DEFAULT_GENCOMPS_GNU_x86_skylake=82 ;export PE_TPSL_64_DEFAULT_GENCOMPS_GNU_x86_skylake;PE_TPSL_64_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH:PE_LIBSCI ;export PE_TPSL_64_DEFAULT_REQUIRED_PRODUCTS;PE_TPSL_64_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/tpsl/18.06.1/@PRGENV@64/@PE_TPSL_64_DEFAULT_GENCOMPS@/@PE_TPSL_64_DEFAULT_TARGET@/lib/pkgconfig ;export PE_TPSL_64_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_TPSL_64_DEFAULT_VOLATILE_PRGENV=GNU\ GNU64 ;export PE_TPSL_64_DEFAULT_VOLATILE_PRGENV;PE_TPSL_DEFAULT_GENCOMPILERS_GNU_haswell=8.2 ;export PE_TPSL_DEFAULT_GENCOMPILERS_GNU_haswell;PE_TPSL_DEFAULT_GENCOMPILERS_GNU_interlagos=8.2 ;export PE_TPSL_DEFAULT_GENCOMPILERS_GNU_interlagos;PE_TPSL_DEFAULT_GENCOMPILERS_GNU_sandybridge=8.2 ;export PE_TPSL_DEFAULT_GENCOMPILERS_GNU_sandybridge;PE_TPSL_DEFAULT_GENCOMPILERS_GNU_x86_64=8.2 ;export PE_TPSL_DEFAULT_GENCOMPILERS_GNU_x86_64;PE_TPSL_DEFAULT_GENCOMPILERS_GNU_x86_skylake=8.2 ;export PE_TPSL_DEFAULT_GENCOMPILERS_GNU_x86_skylake;PE_TPSL_DEFAULT_GENCOMPS_GNU_haswell=82 ;export PE_TPSL_DEFAULT_GENCOMPS_GNU_haswell;PE_TPSL_DEFAULT_GENCOMPS_GNU_interlagos=82 ;export PE_TPSL_DEFAULT_GENCOMPS_GNU_interlagos;PE_TPSL_DEFAULT_GENCOMPS_GNU_sandybridge=82 ;export PE_TPSL_DEFAULT_GENCOMPS_GNU_sandybridge;PE_TPSL_DEFAULT_GENCOMPS_GNU_x86_64=82 ;export PE_TPSL_DEFAULT_GENCOMPS_GNU_x86_64;PE_TPSL_DEFAULT_GENCOMPS_GNU_x86_skylake=82 ;export PE_TPSL_DEFAULT_GENCOMPS_GNU_x86_skylake;PE_TPSL_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH:PE_LIBSCI ;export PE_TPSL_DEFAULT_REQUIRED_PRODUCTS;PE_TPSL_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/tpsl/18.06.1/@PRGENV@/@PE_TPSL_DEFAULT_GENCOMPS@/@PE_TPSL_DEFAULT_TARGET@/lib/pkgconfig ;export PE_TPSL_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_TPSL_DEFAULT_VOLATILE_PRGENV=GNU\ GNU64 ;export PE_TPSL_DEFAULT_VOLATILE_PRGENV;PE_TRILINOS_DEFAULT_GENCOMPILERS_GNU_x86_64=8.2 ;export PE_TRILINOS_DEFAULT_GENCOMPILERS_GNU_x86_64;PE_TRILINOS_DEFAULT_GENCOMPS_GNU_x86_64=82 ;export PE_TRILINOS_DEFAULT_GENCOMPS_GNU_x86_64;PE_TRILINOS_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH:PE_HDF5_PARALLEL:PE_NETCDF_HDF5PARALLEL:PE_LIBSCI:PE_TPSL ;export PE_TRILINOS_DEFAULT_REQUIRED_PRODUCTS;PE_TRILINOS_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/trilinos/12.12.1.1/@PRGENV@/@PE_TRILINOS_DEFAULT_GENCOMPS@/@PE_TRILINOS_DEFAULT_TARGET@/lib/pkgconfig ;export PE_TRILINOS_DEFAULT_VOLATILE_PKGCONFIG_PATH;PE_TRILINOS_DEFAULT_VOLATILE_PRGENV=GNU ;export PE_TRILINOS_DEFAULT_VOLATILE_PRGENV;PKGCONFIG_ENABLED=1 ;export PKGCONFIG_ENABLED;PKG_CONFIG_PATH=/opt/gcc/6.3.0/pkg-config:/sw/EasyBuild/software/cURL/7.59.0/lib/pkgconfig:/sw/EasyBuild/software/OpenSSL/1.0.2m/lib/pkgconfig:/opt/cray/rca/1.0.0-2.0502.60530.1.63.gem/lib64/pkgconfig:/opt/cray/pmi/5.0.11/lib64/pkgconfig:/opt/cray/craype/2.5.16/pkg-config:/opt/cray/iobuf/2.0.8/lib/pkgconfig:/opt/cray/xpmem/0.1-2.0502.64982.7.26.gem/lib64/pkgconfig:/opt/cray/dmapp/7.0.1-1.0502.11080.8.74.gem/lib64/pkgconfig:/opt/cray/gni-headers/4.0-1.0502.10859.9.26.gem/lib64/pkgconfig:/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/lib64/pkgconfig:/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/lib64/pkgconfig:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/lib64/pkgconfig:/opt/cray/atp/2.1.0/lib/pkgconfig ;export PKG_CONFIG_PATH;PKG_CONFIG_PATH_DEFAULT=/opt/cray/papi/5.6.0.5/lib64/pkgconfig ;export PKG_CONFIG_PATH_DEFAULT;SHMEM_ABORT_ON_ERROR=1 ;export SHMEM_ABORT_ON_ERROR;XTOS_VERSION=5.2.82 ;export XTOS_VERSION;XTPE_NETWORK_TARGET=gemini ;export XTPE_NETWORK_TARGET;_LMFILES_=/opt/modulefiles/modules/3.2.10.5:/opt/cray/gem/modulefiles/nodestat/2.2-1.0502.60539.1.31.gem:/opt/cray/gem/modulefiles/sdb/1.1-1.0502.63652.4.27.gem:/opt/cray/gem/modulefiles/alps/5.2.4-2.0502.9774.31.12.gem:/opt/cray/modulefiles/lustre-cray_gem_s/2.5_3.0.101_0.46.1_1.0502.8871.44.1-1.0502.21728.75.3:/opt/cray/gem/modulefiles/udreg/2.3.2-1.0502.10518.2.17.gem:/opt/cray/gem/modulefiles/ugni/6.0-1.0502.10863.8.28.gem:/opt/cray/gem/modulefiles/gni-headers/4.0-1.0502.10859.9.26.gem:/opt/cray/gem/modulefiles/dmapp/7.0.1-1.0502.11080.8.74.gem:/opt/cray/gem/modulefiles/xpmem/0.1-2.0502.64982.7.26.gem:/opt/modulefiles/hss-llm/7.2.0:/opt/modulefiles/Base-opts/1.0.2-1.0502.60680.2.4.gem:/opt/modulefiles/gcc/6.3.0:/opt/cray/craype/2.5.16/modulefiles/craype-network-gemini:/opt/cray/craype/2.5.16/modulefiles/craype-interlagos:/opt/cray/modulefiles/craype/2.5.16:/opt/modulefiles/moab/9.1.2.h2-sles11:/opt/modulefiles/torque/6.1.2:/opt/cray/modulefiles/cray-mpich/7.7.4:/opt/modulefiles/java/jdk1.8.0_51:/usr/local/modulefiles/globus/6.0:/usr/local/modulefiles/gsissh/7.5p1b:/opt/modulefiles/xalt/0.7.6.local:/sw/xe/modulefiles/scripts:/sw/EasyBuild/modules/all/OpenSSL/1.0.2m:/sw/EasyBuild/modules/all/cURL/7.59.0:/sw/EasyBuild/modules/all/git/2.17.0:/sw/EasyBuild/modules/all/wget/1.19.4:/sw/xe/modulefiles/user-paths:/opt/cray/modulefiles/cray-libsci/18.12.1:/opt/cray/gem/modulefiles/pmi/5.0.11:/opt/cray/modulefiles/atp/2.1.0:/opt/cray/modulefiles/PrgEnv-gnu/5.2.82 ;export _LMFILES_;unset ASSEMBLER_AARCH64;unset ASSEMBLER_X86_64;unset CC_X86_64;unset CRAYLIBS_AARCH64;unset CRAYLIBS_X86_64;unset CRAYLMD_LICENSE_FILE;unset CRAY_BINUTILS_BIN;unset CRAY_BINUTILS_BIN_AARCH64;unset CRAY_BINUTILS_BIN_X86_64;unset CRAY_BINUTILS_ROOT;unset CRAY_BINUTILS_ROOT_AARCH64;unset CRAY_BINUTILS_ROOT_X86_64;unset CRAY_BINUTILS_VERSION;unset CRAY_CCE_SHARE;unset CRAY_CC_VERSION;unset CRAY_CXX_IPA_LIBS;unset CRAY_CXX_IPA_LIBS_AARCH64;unset CRAY_CXX_IPA_LIBS_X86_64;unset CRAY_FTN_VERSION;unset CRAY_PE_TARGET;unset CRAY_PRGENVCRAY;unset CRAY_RCA_INCLUDE_OPTS;unset CRAY_RCA_POST_LINK_OPTS;unset FORTRAN_SYSTEM_MODULE_NAMES;unset FTN_X86_64;unset GCC_AARCH64;unset GCC_X86_64;unset INCLUDE_PATH_AARCH64;unset INCLUDE_PATH_X86_64;unset LINKER_AARCH64;unset LINKER_X86_64;unset NLSPATH;unset PE_LEVEL; ${MODULESHOME}/bin/modulecmd bash $* CRAYPE_LINK_TYPE=dynamic ;export CRAYPE_LINK_TYPE;CRAY_CUDATOOLKIT_DIR=/opt/nvidia/cudatoolkit9.1/9.1.85_3.10-1.0502.df1cc54.3.1 ;export CRAY_CUDATOOLKIT_DIR;CRAY_CUDATOOLKIT_INCLUDE_OPTS=-I/opt/nvidia/cudatoolkit9.1/9.1.85_3.10-1.0502.df1cc54.3.1/include\ -I/opt/nvidia/cudatoolkit9.1/9.1.85_3.10-1.0502.df1cc54.3.1/extras/CUPTI/include\ -I/opt/nvidia/cudatoolkit9.1/9.1.85_3.10-1.0502.df1cc54.3.1/extras/Debugger/include ;export CRAY_CUDATOOLKIT_INCLUDE_OPTS;CRAY_CUDATOOLKIT_POST_LINK_OPTS=-L/opt/nvidia/cudatoolkit9.1/9.1.85_3.10-1.0502.df1cc54.3.1/lib64\ -L/opt/nvidia/cudatoolkit9.1/9.1.85_3.10-1.0502.df1cc54.3.1/extras/CUPTI/lib64\ -Wl,--as-needed\ -Wl,-lcupti\ -Wl,-lcudart\ -Wl,--no-as-needed\ -L/opt/cray/nvidia/default/lib64\ -lcuda ;export CRAY_CUDATOOLKIT_POST_LINK_OPTS;CRAY_CUDATOOLKIT_VERSION=9.1.85_3.10-1.0502.df1cc54.3.1 ;export CRAY_CUDATOOLKIT_VERSION;CRAY_LD_LIBRARY_PATH=/opt/nvidia/cudatoolkit9.1/9.1.85_3.10-1.0502.df1cc54.3.1/lib64:/opt/nvidia/cudatoolkit9.1/9.1.85_3.10-1.0502.df1cc54.3.1/extras/CUPTI/lib64:/opt/cray/pmi/5.0.11/lib64:/opt/cray/libsci/18.12.1/GNU/5.1/x86_64/lib:/opt/cray/mpt/7.7.4/gni/mpich-gnu/5.1/lib:/opt/cray/xpmem/0.1-2.0502.64982.7.26.gem/lib64:/opt/cray/dmapp/7.0.1-1.0502.11080.8.74.gem/lib64:/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/lib64:/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/lib64:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/lib64 ;export CRAY_LD_LIBRARY_PATH;CUDATOOLK#module unload gcc #module load gcc/4.9.3 ~/bin/myenv.sh # for topomapping run: # module load topomapping # cp ~/awp-odc-spp/Makefile.bluewaters.gnu.topomapping Makefile.bluewaters.gnu make -f Makefile.bluewaters.gnu clean make -f Makefile.bluewaters.gnu IT_HOME=/opt/nvidia/cudatoolkit9.1/9.1.85_3.10-1.0502.df1cc54.3.1 ;export CUDATOOLKIT_HOME;LOADEDMODULES=modules/3.2.10.5:nodestat/2.2-1.0502.60539.1.31.gem:sdb/1.1-1.0502.63652.4.27.gem:alps/5.2.4-2.0502.9774.31.12.gem:lustre-cray_gem_s/2.5_3.0.101_0.46.1_1.0502.8871.44.1-1.0502.21728.75.3:udreg/2.3.2-1.0502.10518.2.17.gem:ugni/6.0-1.0502.10863.8.28.gem:gni-headers/4.0-1.0502.10859.9.26.gem:dmapp/7.0.1-1.0502.11080.8.74.gem:xpmem/0.1-2.0502.64982.7.26.gem:hss-llm/7.2.0:Base-opts/1.0.2-1.0502.60680.2.4.gem:gcc/6.3.0:craype-network-gemini:craype-interlagos:craype/2.5.16:moab/9.1.2.h2-sles11:torque/6.1.2:cray-mpich/7.7.4:java/jdk1.8.0_51:globus/6.0:gsissh/7.5p1b:xalt/0.7.6.local:scripts:OpenSSL/1.0.2m:cURL/7.59.0:git/2.17.0:wget/1.19.4:user-paths:cray-libsci/18.12.1:pmi/5.0.11:atp/2.1.0:PrgEnv-gnu/5.2.82:cudatoolkit/9.1.85_3.10-1.0502.df1cc54.3.1 ;export LOADEDMODULES;MANPATH=/opt/nvidia/cudatoolkit9.1/9.1.85_3.10-1.0502.df1cc54.3.1/doc/man:/opt/cray/atp/2.1.0/man:/opt/cray/pmi/5.0.11/man:/opt/cray/libsci/18.12.1/man:/opt/cray/man/csmlversion:/opt/gcc/6.3.0/snos/share/man:/sw/EasyBuild/software/wget/1.19.4/share/man:/sw/EasyBuild/software/cURL/7.59.0/share/man:/opt/java/jdk1.8.0_51/man:/usr/local/globus-6.0/man:/opt/cray/mpt/7.7.4/gni/man/mpich:/opt/torque/6.1.2/share/man:/opt/moab/9.1.2.h2/share/man/:/opt/cray/craype/2.5.16/man:/opt/cray/llm/default/man:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.46.1_1.0502.8871.44.1-1.0502.21728.75.3/man:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/man:/opt/modules/3.2.10.5/share/man:/usr/local/man:/usr/share/man:/usr/man:/opt/cray/share/man:/opt/cray/man:/u/system/bwjenkins/man ;export MANPATH;PATH=/opt/nvidia/cudatoolkit9.1/9.1.85_3.10-1.0502.df1cc54.3.1/bin:/opt/nvidia/cudatoolkit9.1/9.1.85_3.10-1.0502.df1cc54.3.1/libnvvp:/opt/gcc/6.3.0/bin:/sw/EasyBuild/software/wget/1.19.4/bin:/sw/EasyBuild/software/git/2.17.0/bin:/sw/EasyBuild/software/cURL/7.59.0/bin:/sw/EasyBuild/software/OpenSSL/1.0.2m/bin:/sw/user/scripts:/sw/admin/scripts:/opt/xalt/0.7.6/sles11.3/libexec:/opt/xalt/0.7.6/sles11.3/bin:/usr/local/gsi-openssh-7.5p1b/bin:/usr/local/gsi-openssh-7.5p1b/libexec:/opt/java/jdk1.8.0_51/bin:/usr/local/globus-6.0/bin:/usr/local/globus-6.0/sbin:/opt/cray/mpt/7.7.4/gni/bin:/opt/torque/6.1.2/sbin:/opt/torque/6.1.2/bin:/opt/moab/9.1.2.h2/bin:/opt/moab/9.1.2.h2/sbin:/opt/cray/craype/2.5.16/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.64982.7.26.gem/bin:/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/bin:/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.46.1_1.0502.8871.44.1-1.0502.21728.75.3/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.46.1_1.0502.8871.44.1-1.0502.21728.75.3/bin:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/sbin:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/bin:/opt/cray/sdb/1.1-1.0502.63652.4.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.60539.1.31.gem/bin:/opt/modules/3.2.10.5/bin:/u/system/bwjenkins/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin ;export PATH;PE_PKGCONFIG_LIBS=cray-cudatoolkit:AtpSigHandler:libsci_mpi:libsci:mpich:gcc ;export PE_PKGCONFIG_LIBS;PKG_CONFIG_PATH=/opt/nvidia/cudatoolkit9.1/9.1.85_3.10-1.0502.df1cc54.3.1/lib64/pkgconfig:/opt/gcc/6.3.0/pkg-config:/sw/EasyBuild/software/cURL/7.59.0/lib/pkgconfig:/sw/EasyBuild/software/OpenSSL/1.0.2m/lib/pkgconfig:/opt/cray/rca/1.0.0-2.0502.60530.1.63.gem/lib64/pkgconfig:/opt/cray/pmi/5.0.11/lib64/pkgconfig:/opt/cray/craype/2.5.16/pkg-config:/opt/cray/iobuf/2.0.8/lib/pkgconfig:/opt/cray/xpmem/0.1-2.0502.64982.7.26.gem/lib64/pkgconfig:/opt/cray/dmapp/7.0.1-1.0502.11080.8.74.gem/lib64/pkgconfig:/opt/cray/gni-headers/4.0-1.0502.10859.9.26.gem/lib64/pkgconfig:/opt/cray/ugni/6.0-1.0502.10863.8.28.gem/lib64/pkgconfig:/opt/cray/udreg/2.3.2-1.0502.10518.2.17.gem/lib64/pkgconfig:/opt/cray/alps/5.2.4-2.0502.9774.31.12.gem/lib64/pkgconfig:/opt/cray/atp/2.1.0/lib/pkgconfig ;export PKG_CONFIG_PATH;XTPE_LINK_TYPE=dynamic ;export XTPE_LINK_TYPE;_LMFILES_=/opt/modulefiles/modules/3.2.10.5:/opt/cray/gem/modulefiles/nodestat/2.2-1.0502.60539.1.31.gem:/opt/cray/gem/modulefiles/sdb/1.1-1.0502.63652.4.27.gem:/opt/cray/gem/modulefiles/alps/5.2.4-2.0502.9774.31.12.gem:/opt/cray/modulefiles/lustre-cray_gem_s/2.5_3.0.101_0.46.1_1.0502.8871.44.1-1.0502.21728.75.3:/opt/cray/gem/modulefiles/udreg/2.3.2-1.0502.10518.2.17.gem:/opt/cray/gem/modulefiles/ugni/6.0-1.0502.10863.8.28.gem:/opt/cray/gem/modulefiles/gni-headers/4.0-1.0502.10859.9.26.gem:/opt/cray/gem/modulefiles/dmapp/7.0.1-1.0502.11080.8.74.gem:/opt/cray/gem/modulefiles/xpmem/0.1-2.0502.64982.7.26.gem:/opt/modulefiles/hss-llm/7.2.0:/opt/modulefiles/Base-opts/1.0.2-1.0502.60680.2.4.gem:/opt/modulefiles/gcc/6.3.0:/opt/cray/craype/2.5.16/modulefiles/craype-network-gemini:/opt/cray/craype/2.5.16/modulefiles/craype-interlagos:/opt/cray/modulefiles/craype/2.5.16:/opt/modulefiles/moab/9.1.2.h2-sles11:/opt/modulefiles/torque/6.1.2:/opt/cray/modulefiles/cray-mpich/7.7.4:/opt/modulefiles/java/jdk1.8.0_51:/usr/local/modulefiles/globus/6.0:/usr/local/modulefiles/gsissh/7.5p1b:/opt/modulefiles/xalt/0.7.6.local:/sw/xe/modulefiles/scripts:/sw/EasyBuild/modules/all/OpenSSL/1.0.2m:/sw/EasyBuild/modules/all/cURL/7.59.0:/sw/EasyBuild/modules/all/git/2.17.0:/sw/EasyBuild/modules/all/wget/1.19.4:/sw/xe/modulefiles/user-paths:/opt/cray/modulefiles/cray-libsci/18.12.1:/opt/cray/gem/modulefiles/pmi/5.0.11:/opt/cray/modulefiles/atp/2.1.0:/opt/cray/modulefiles/PrgEnv-gnu/5.2.82:/opt/cray/modulefiles/cudatoolkit/9.1.85_3.10-1.0502.df1cc54.3.1 ;export _LMFILES_; === RUNNING /u/system/bwjenkins/bin/myenv.sh === HOST=jyc1 HOME=/u/system/bwjenkins SCRATCH=/u/system/bwjenkins/scratch ls SCRATCH: mdtest setsm_benchmark SPP-PPM stripe8 sw_config sw_workdir test.log tests my id: uid=49294(bwjenkins) gid=16228(BW_System) groups=16228(BW_System),999(login_user),1000(ie_user),1001(hpss_user),1009(swtools),16237(jnd) Currently Loaded Modulefiles: 1) modules/3.2.10.5 2) nodestat/2.2-1.0502.60539.1.31.gem 3) sdb/1.1-1.0502.63652.4.27.gem 4) alps/5.2.4-2.0502.9774.31.12.gem 5) lustre-cray_gem_s/2.5_3.0.101_0.46.1_1.0502.8871.44.1-1.0502.21728.75.3 6) udreg/2.3.2-1.0502.10518.2.17.gem 7) ugni/6.0-1.0502.10863.8.28.gem 8) gni-headers/4.0-1.0502.10859.9.26.gem 9) dmapp/7.0.1-1.0502.11080.8.74.gem 10) xpmem/0.1-2.0502.64982.7.26.gem 11) hss-llm/7.2.0 12) Base-opts/1.0.2-1.0502.60680.2.4.gem 13) gcc/6.3.0 14) craype-network-gemini 15) craype-interlagos 16) craype/2.5.16 17) moab/9.1.2.h2-sles11 18) torque/6.1.2 19) cray-mpich/7.7.4 20) java/jdk1.8.0_51 21) globus/6.0 22) gsissh/7.5p1b 23) xalt/0.7.6.local 24) scripts 25) OpenSSL/1.0.2m 26) cURL/7.59.0 27) git/2.17.0 28) wget/1.19.4 29) user-paths 30) cray-libsci/18.12.1 31) pmi/5.0.11 32) atp/2.1.0 33) PrgEnv-gnu/5.2.82 34) cudatoolkit/9.1.85_3.10-1.0502.df1cc54.3.1 rm -f *.o pmcl3d cc -O3 -g -I/opt/nvidia/cudatoolkit9.1/9.1.85_3.10-1.0502.df1cc54.3.1/include -c -o command.o command.c cc -O3 -g -I/opt/nvidia/cudatoolkit9.1/9.1.85_3.10-1.0502.df1cc54.3.1/include -c -o pmcl3d.o pmcl3d.c cc -O3 -g -I/opt/nvidia/cudatoolkit9.1/9.1.85_3.10-1.0502.df1cc54.3.1/include -c -o grid.o grid.c cc -O3 -g -I/opt/nvidia/cudatoolkit9.1/9.1.85_3.10-1.0502.df1cc54.3.1/include -c -o source.o source.c cc -O3 -g -I/opt/nvidia/cudatoolkit9.1/9.1.85_3.10-1.0502.df1cc54.3.1/include -c -o mesh.o mesh.c cc -O3 -g -I/opt/nvidia/cudatoolkit9.1/9.1.85_3.10-1.0502.df1cc54.3.1/include -c -o cerjan.o cerjan.c cc -O3 -g -I/opt/nvidia/cudatoolkit9.1/9.1.85_3.10-1.0502.df1cc54.3.1/include -c -o swap.o swap.c /opt/nvidia/cudatoolkit9.1/9.1.85_3.10-1.0502.df1cc54.3.1/bin/nvcc -use_fast_math -arch=sm_35 -I/opt/nvidia/cudatoolkit9.1/9.1.85_3.10-1.0502.df1cc54.3.1/include -c -o kernel.o kernel.cu cc -O3 -g -I/opt/nvidia/cudatoolkit9.1/9.1.85_3.10-1.0502.df1cc54.3.1/include -c -o io.o io.c cc -O3 -g -I/opt/nvidia/cudatoolkit9.1/9.1.85_3.10-1.0502.df1cc54.3.1/include -o pmcl3d command.o pmcl3d.o grid.o source.o mesh.o cerjan.o swap.o kernel.o io.o -lm -ldl -L/opt/nvidia/cudatoolkit9.1/9.1.85_3.10-1.0502.df1cc54.3.1/lib64 -lcudart -lstdc++ cd ~/awp-odc-spp/awp-odc-os/run cat run_small.pbs | sed -e "s/aprun/time aprun/g" > runs.pbs cp runs.pbs run_small.pbs cat -n run_small.pbs rm awp-odc_gpu_small.* MYJOBID=`qsub run_small.pbs` qsub run_small.pbs 1 #!/bin/bash -login 2 #PBS -N awp-odc_gpu_small 3 #PBS -l walltime=00:30:00,nodes=4:ppn=1:xk 4 #PBS -j eo 5 6 module unload darshan 7 module swap $( module list | grep -o PrgEnv-.*$ ) PrgEnv-gnu 8 module load cudatoolkit 9 10 mkdir $PBS_O_WORKDIR/$PBS_JOBID 11 cd $PBS_O_WORKDIR/$PBS_JOBID 12 13 ln -fs ../../src/pmcl3d 14 mkdir input 15 mkdir input_rst 16 mkdir output_ckp 17 mkdir output_sfc 18 cp ../../input/source input/. 19 lfs setstripe -s 600m -c 4 input/mesh 20 cp ../../input/mesh input/. 21 lfs setstripe -s 3m -c 4 output_sfc 22 lfs setstripe -s 600m -c 1 input_rst 23 24 start_time=$(date +%s) 25 26 time aprun -n $PBS_NP -N1 ./pmcl3d \ 27 -X 320 -Y 320 -Z 2048 -x 2 -y 2 \ 28 --TMAX 4.0 --DH 40.0 --DT 0.002 \ 29 --NSRC 1 --NST 101 \ 30 --IFAULT 1 --MEDIASTART 2 --READ_STEP 101 --WRITE_STEP 100 \ 31 --NSKPX 2 --NSKPY 2 --NTISKP 20 -c output_ckp/ckp -o output_sfc --INSRC input/source --INVEL input/mesh 32 33 end_time=$(date +%s) 34 echo Completed in $(( end_time - start_time )) seconds 35 INFO: Job submitted to account: jnc echo MYJOBID=$MYJOBID ~/bin//wait_for.sh $MYJOBID MYJOBID=262805.jyc === RUNNING /u/system/bwjenkins/bin//wait_for.sh === Wed Mar 13 13:24:21 CDT 2019: waiting for 262805.jyc to finish Wed Mar 13 13:29:21 CDT 2019: waiting for 262805.jyc to finish Wed Mar 13 13:34:21 CDT 2019: waiting for 262805.jyc to finish qstat: Unknown Job Id Error 262805.nid00030 tail -50 awp-odc_gpu_small.e* > ~/awp-odc-spp/stderr.txt MYREALMIN=`grep real awp-odc_gpu_small.e* | cut -dl -f2 | cut -dm -f1` grep real awp-odc_gpu_small.e* | cut -dl -f2 | cut -dm -f1 echo "YVALUE=$MYREALMIN" > ~/awp-odc-spp/awp.realtime cat -n ~/awp-odc-spp/awp.realtime 262805.jyc finished 1 YVALUE= 5 [SSH] exit-status: 0 [awp-odc-spp-jyc] $ /bin/sh -xe /tmp/jenkins5331934862675223088.sh + rm -rf /var/lib/jenkins/workspace/awp-odc-spp-jyc/awp.realtime /var/lib/jenkins/workspace/awp-odc-spp-jyc/stderr.txt + scp 'JYC_REMOTE:~/awp-odc-spp/*.txt' /var/lib/jenkins/workspace/awp-odc-spp-jyc NCSA JYC System Access status: User Access UnRestricted. Access by OTP or Two Factor Certificate Authority only. Use myproxy-logon -s tfca.ncsa.illinois.edu -p 7512 for gsissh access. gsissh or ssh -o PreferredAuthentications=keyboard-interactive for otp access. Restricted access during weekly scheduled maintenance on Wednesdays 10AM - 6PM + scp 'JYC_REMOTE:~/awp-odc-spp/awp.realtime' /var/lib/jenkins/workspace/awp-odc-spp-jyc NCSA JYC System Access status: User Access UnRestricted. Access by OTP or Two Factor Certificate Authority only. Use myproxy-logon -s tfca.ncsa.illinois.edu -p 7512 for gsissh access. gsissh or ssh -o PreferredAuthentications=keyboard-interactive for otp access. Restricted access during weekly scheduled maintenance on Wednesdays 10AM - 6PM Recording plot data Saving plot series data from: /var/lib/jenkins/workspace/awp-odc-spp-jyc/awp.realtime Finished: SUCCESS