Started by user Greg Bauer
[EnvInject] - Loading node environment variables.
Building on master in workspace /var/lib/jenkins/workspace/SPP-PPM-1280-132XEs
executing script:

# Moving to the building folder
cd SPP-PPM/PPM-1280-132XEs
ls -al

# Updating modules
module unload darshan
module load craype-hugepages2M
module list

# Clear old auxiliary files
rm compile-JK-noPapi
rm generate-MPICH_RANK_ORDER
rm run-JK

# Building the executable
touch compile-JK-noPapi
chmod u+x compile-JK-noPapi
echo "#!/bin/csh" >compile-JK-noPapi 
echo "ln -sf iq\$1.h iq.h" >>compile-JK-noPapi 
echo "ftn -h nocaf -static -O2 -h fp3 -r m -h keepfiles PPM2F-tp3-5-1-12-ICF-short-loops.F cio.o -o PPM2F-\$1  "  >>compile-JK-noPapi 
cat -n compile-JK-noPapi
./compile-JK-noPapi 1280
ls -al

# Creating an auxiliary file for a job script
touch generate-MPICH_RANK_ORDER 
chmod u+x generate-MPICH_RANK_ORDER 
echo "#!/bin/csh -f" >generate-MPICH_RANK_ORDER 
echo "@ tasks = \$1" >>generate-MPICH_RANK_ORDER 
echo "@ iotasks = \$2" >>generate-MPICH_RANK_ORDER 

echo "# generate MPICH_RANK_ORDER file with " >>generate-MPICH_RANK_ORDER 
echo "# groups of \$k / 4 ranks, then 4 ioserver tasks." >>generate-MPICH_RANK_ORDER 

echo "@ n = 0" >>generate-MPICH_RANK_ORDER 
echo "@ io = \$tasks - \$iotasks" >>generate-MPICH_RANK_ORDER 
echo "@ step = 4 * \$io / \$iotasks" >>generate-MPICH_RANK_ORDER 

echo "rm -f MPICH_RANK_ORDER" >>generate-MPICH_RANK_ORDER 

echo "while ( \$n < \$tasks - \$step )" >>generate-MPICH_RANK_ORDER 
echo "@ nn = \$n + \$step - 1" >>generate-MPICH_RANK_ORDER 
echo "@ io2 = \$io + 3" >>generate-MPICH_RANK_ORDER 
echo "echo \$n'-'\$nn','\$io'-'\$io2 >>MPICH_RANK_ORDER" >>generate-MPICH_RANK_ORDER 
echo "@ n = \$n + \$step" >>generate-MPICH_RANK_ORDER 
echo "@ io = \$io + 4" >>generate-MPICH_RANK_ORDER 
echo "end" >>generate-MPICH_RANK_ORDER 

cat -n generate-MPICH_RANK_ORDER

# Creating a job script
touch run-JK
echo "#PBS -N PPM-xe-1280" >run-JK
echo "#PBS -l nodes=132:ppn=32:xe" >>run-JK
echo "#PBS -l walltime=00:30:00" >>run-JK
echo "#PBS -q debug" >>run-JK
echo "#PBS -j oe" >>run-JK
echo "#PBS -M jkwack2@illinois.edu" >>run-JK

echo ". /opt/modules/default/init/bash" >>run-JK
echo "module unload darshan" >>run-JK
echo "module load craype-hugepages2M" >>run-JK
echo "module list" >>run-JK

echo "cd \$PBS_O_WORKDIR" >>run-JK

echo "here='/u/system/bwjenkins/SPP-PPM/PPM-1280-132XEs' " >>run-JK

echo "data=DATA1280.\$PBS_JOBID" >>run-JK

echo "mkdir -p /scratch/system/bwjenkins/SPP-PPM/\$data" >>run-JK
echo "ln -sf /scratch/system/bwjenkins/SPP-PPM/\$data \$data" >>run-JK
echo "lfs setstripe --count=2 /scratch/system/bwjenkins/SPP-PPM/\$data" >>run-JK
echo "cd /scratch/system/bwjenkins/SPP-PPM/\$data" >>run-JK

echo "export MPICH_RANK_REORDER_METHOD=3" >>run-JK
echo "\$here/generate-MPICH_RANK_ORDER  2112 64" >>run-JK

echo "touch \$here/PPM4Fperf.ppm.\$PBS_JOBID" >>run-JK
echo "ln -sf \$here/PPM4Fperf.ppm.\$PBS_JOBID  PPM4Fperf.ppm" >>run-JK

echo "ulimit -s unlimited" >>run-JK
echo "export OMP_STACKSIZE=200M" >>run-JK
echo "export OMP_NUM_THREADS=2" >>run-JK

echo "date" >>run-JK
echo "start=\$(date +%s)" >>run-JK
echo "/usr/bin/time aprun -n 2112 -N 16 -S 4 -d 2 -ss \$here/PPM2F-1280 >& \$here/stderrout.\$PBS_JOBID" >>run-JK
echo "end=\$(date +%s)" >>run-JK
echo "date" >>run-JK
echo "WT=\$((end-start))" >>run-JK

echo "NNODE=\$(sort -u \$PBS_NODEFILE | wc -w)" >>run-JK
echo "python \$here/FLOPS-PPM-JK-v01.py \$here/PPM4Fperf.ppm.\$PBS_JOBID \$WT \$NNODE" >>run-JK
echo "cp GFLOPS_SEC_NODE.dat \$here/GFLOPS_SEC_NODE.dat" >>run-JK
echo "cp TFLOPS_SEC-AGG.dat \$here/TFLOPS_SEC-AGG.dat" >>run-JK

cat -n run-JK

# Submitting the job script
MYJOBID=`qsub run-JK`
echo MYJOBID=$MYJOBID

# Waiting for the job completion
cat -n wait_for.sh
./wait_for.sh $MYJOBID







[SSH] Exception:java.net.NoRouteToHostException: No route to host
com.jcraft.jsch.JSchException: java.net.NoRouteToHostException: No route to host
	at com.jcraft.jsch.Util.createSocket(Util.java:341)
	at com.jcraft.jsch.Session.connect(Session.java:182)
	at com.jcraft.jsch.Session.connect(Session.java:150)
	at org.jvnet.hudson.plugins.SSHSite.createSession(SSHSite.java:141)
	at org.jvnet.hudson.plugins.SSHSite.executeCommand(SSHSite.java:151)
	at org.jvnet.hudson.plugins.SSHBuilder.perform(SSHBuilder.java:60)
	at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
	at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:779)
	at hudson.model.Build$BuildExecution.build(Build.java:206)
	at hudson.model.Build$BuildExecution.doRun(Build.java:163)
	at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:534)
	at hudson.model.Run.execute(Run.java:1733)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:405)
Caused by: java.net.NoRouteToHostException: No route to host
	at java.net.PlainSocketImpl.socketConnect(Native Method)
	at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
	at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
	at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
	at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
	at java.net.Socket.connect(Socket.java:579)
	at java.net.Socket.connect(Socket.java:528)
	at java.net.Socket.<init>(Socket.java:425)
	at java.net.Socket.<init>(Socket.java:208)
	at com.jcraft.jsch.Util.createSocket(Util.java:335)
	... 14 more
Build step 'Execute shell script on remote host using ssh' marked build as failure
Recording plot data
Saving plot series data from: /var/lib/jenkins/workspace/SPP-PPM-1280-132XEs/GFLOPS_SEC_NODE.dat
Saving plot series data from: /var/lib/jenkins/workspace/SPP-PPM-1280-132XEs/TFLOPS_SEC-AGG.dat
Sending e-mails to: jkwack2@illinois.edu
Finished: FAILURE