|
1
|
1 #!/bin/bash
|
|
|
2
|
|
|
3 site=$1
|
|
|
4 n=$2
|
|
|
5 outdir=$3
|
|
|
6 logfile=$4
|
|
|
7 htmlfile=$5
|
|
|
8 mkdir -p $outdir
|
|
|
9 echo $@ >> /tmp/args.txt
|
|
|
10
|
|
|
11 swift=`\which swift`
|
|
|
12
|
|
|
13 # use "here" document for configs
|
|
|
14 cat << EOF > /tmp/hello.swift
|
|
|
15 type file;
|
|
|
16
|
|
|
17 app (file o) helloapp (string msg) {
|
|
|
18 echo msg stdout=@o;
|
|
|
19 }
|
|
|
20
|
|
|
21 file out[]<simple_mapper; location="$outdir", prefix="f.",suffix=".txt">;
|
|
|
22
|
|
|
23 foreach j in [1:@toInt(@arg("n","2"))] {
|
|
|
24 out[j] = helloapp ("Hello World!");
|
|
|
25 }
|
|
|
26 EOF
|
|
|
27
|
|
|
28 case "$site" in
|
|
|
29 cluster )
|
|
|
30 cat << 'EOF' > /tmp/sites.xml
|
|
|
31 <?xml version="1.0" encoding="UTF-8"?>
|
|
|
32 <config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
|
|
|
33 <pool handle="cluster">
|
|
|
34 <execution provider="coaster" jobmanager="local:pbs"/>
|
|
|
35 <profile namespace="globus" key="queue">default</profile>
|
|
|
36 <profile namespace="globus" key="jobsPerNode">8</profile>
|
|
|
37 <profile namespace="globus" key="maxWalltime">02:00:00</profile>
|
|
|
38 <profile namespace="globus" key="maxTime">7500</profile>
|
|
|
39 <profile namespace="globus" key="highOverAllocation">100</profile>
|
|
|
40 <profile namespace="globus" key="lowOverAllocation">100</profile>
|
|
|
41 <profile namespace="globus" key="slots">1</profile>
|
|
|
42 <profile namespace="globus" key="maxNodes">1</profile>
|
|
|
43 <profile namespace="globus" key="nodeGranularity">1</profile>
|
|
|
44 <profile namespace="karajan" key="jobThrottle">.31</profile>
|
|
|
45 <profile namespace="karajan" key="initialScore">10000</profile>
|
|
|
46 <workdirectory>/tmp/swift.workdir</workdirectory>
|
|
|
47 </pool>
|
|
|
48 </config>
|
|
|
49 EOF
|
|
|
50
|
|
|
51 cat << 'EOF' > /tmp/tc
|
|
|
52 cluster echo echo
|
|
|
53 cluster cat cat
|
|
|
54 EOF
|
|
|
55
|
|
|
56 cat << 'EOF' > /tmp/cf
|
|
|
57 use.provider.staging=true
|
|
|
58 wrapperlog.always.transfer=true
|
|
|
59 execution.retries=1
|
|
|
60 provider.staging.pin.swiftfiles=false
|
|
|
61 sitedir.keep=true
|
|
|
62 lazy.errors=false
|
|
|
63 EOF
|
|
|
64 ;;
|
|
|
65
|
|
|
66 localhost )
|
|
|
67 cat << 'EOF' > /tmp/sites.xml
|
|
|
68 <?xml version="1.0" encoding="UTF-8"?>
|
|
|
69 <config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
|
|
|
70 <pool handle="localhost">
|
|
|
71 <execution provider="local" />
|
|
|
72 <profile namespace="karajan" key="jobThrottle">0.01</profile>
|
|
|
73 <profile namespace="karajan" key="initialScore">10000</profile>
|
|
|
74 <filesystem provider="local"/>
|
|
|
75 <workdirectory>/tmp/swift.work</workdirectory>
|
|
|
76 <profile namespace="swift" key="stagingMethod">local</profile>
|
|
|
77 </pool>
|
|
|
78 </config>
|
|
|
79 EOF
|
|
|
80
|
|
|
81 cat << 'EOF' > /tmp/tc
|
|
|
82 localhost echo echo
|
|
|
83 localhost cat cat
|
|
|
84 EOF
|
|
|
85
|
|
|
86 cat << 'EOF' > /tmp/cf
|
|
|
87 use.provider.staging=false
|
|
|
88 wrapperlog.always.transfer=true
|
|
|
89 execution.retries=0
|
|
|
90 provider.staging.pin.swiftfiles=false
|
|
|
91 sitedir.keep=true
|
|
|
92 lazy.errors=false
|
|
|
93 EOF
|
|
|
94 ;;
|
|
|
95
|
|
|
96 * )
|
|
|
97 echo "defaulting to localhost"
|
|
|
98
|
|
|
99 cat << 'EOF' > /tmp/sites.xml
|
|
|
100 <?xml version="1.0" encoding="UTF-8"?>
|
|
|
101 <config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
|
|
|
102 <pool handle="localhost">
|
|
|
103 <execution provider="local" />
|
|
|
104 <profile namespace="karajan" key="jobThrottle">0.01</profile>
|
|
|
105 <profile namespace="karajan" key="initialScore">10000</profile>
|
|
|
106 <filesystem provider="local"/>
|
|
|
107 <workdirectory>/tmp/swift.work</workdirectory>
|
|
|
108 <profile namespace="swift" key="stagingMethod">local</profile>
|
|
|
109 </pool>
|
|
|
110 </config>
|
|
|
111 EOF
|
|
|
112
|
|
|
113 cat << 'EOF' > /tmp/tc
|
|
|
114 localhost echo echo
|
|
|
115 localhost cat cat
|
|
|
116 EOF
|
|
|
117
|
|
|
118 cat << 'EOF' > /tmp/cf
|
|
|
119 use.provider.staging=false
|
|
|
120 wrapperlog.always.transfer=true
|
|
|
121 execution.retries=0
|
|
|
122 provider.staging.pin.swiftfiles=false
|
|
|
123 sitedir.keep=true
|
|
|
124 lazy.errors=false
|
|
|
125 EOF
|
|
|
126 ;;
|
|
|
127 esac
|
|
|
128
|
|
|
129 $swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/hello.swift -n=$n -data=$data 2> std.err
|
|
|
130 EXITCODE=$?
|
|
|
131
|
|
|
132 swiftlog=`\ls -t *.log | head -1`
|
|
|
133
|
|
|
134 cat << 'EOF' > /tmp/plotswiftlogs
|
|
|
135 #!/bin/bash
|
|
|
136
|
|
|
137 #usage: ./extract4plots <swift.logfile>
|
|
|
138
|
|
|
139 SWIFTLOGFILE=$1
|
|
|
140 #TMPDIR=`mktemp -d plotlog.XXX`
|
|
|
141 grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out
|
|
|
142 SWIFTOUTFILE=swiftoutfile.out
|
|
|
143 #extract start time
|
|
|
144 TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '`
|
|
|
145 START_TIME=`date +%s -d "$TMPDATE"`
|
|
|
146
|
|
|
147 #extract end time
|
|
|
148 TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '`
|
|
|
149 END_TIME=`date +%s -d "$TMPDATE"`
|
|
|
150
|
|
|
151 #duration
|
|
|
152 DIFFTIME=$((END_TIME - START_TIME))
|
|
|
153 #echo 0 > active.txt
|
|
|
154 #echo 0 > cumulative.txt
|
|
|
155 #extract active runs in a file
|
|
|
156 (grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > active.txt)
|
|
|
157
|
|
|
158 #extract successful completions in a file
|
|
|
159 (grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > cumulative.txt)
|
|
|
160
|
|
|
161 #prepare tics
|
|
|
162 activelines=`wc -l active.txt | awk '{print $1}'`
|
|
|
163 cumulines=`wc -l cumulative.txt | awk '{print $1}'`
|
|
|
164
|
|
|
165 if [ $activelines -ne 0 ]
|
|
|
166 then
|
|
|
167 activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc`
|
|
|
168 fi
|
|
|
169 seq 0 $activelinespertic $DIFFTIME > activetics.txt
|
|
|
170
|
|
|
171 if [ $cumulines -ne 0 ]
|
|
|
172 then
|
|
|
173 cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc`
|
|
|
174 fi
|
|
|
175
|
|
|
176 seq 0 $cumulinespertic $DIFFTIME > cumultics.txt
|
|
|
177
|
|
|
178 #final plot data
|
|
|
179 paste activetics.txt active.txt > plot_active.txt
|
|
|
180 paste cumultics.txt cumulative.txt > plot_cumulative.txt
|
|
|
181
|
|
|
182 cat << EOF1 > plotit.gp
|
|
|
183 set terminal png enhanced
|
|
|
184 set output "activeplot.png"
|
|
|
185 set nokey
|
|
|
186 set xlabel "Time in sec"
|
|
|
187 set ylabel "number of active jobs"
|
|
|
188 set title "Active jobs"
|
|
|
189 plot "plot_active.txt" using 1:2 with line
|
|
|
190
|
|
|
191 set output "cumulativeplot.png"
|
|
|
192 set ylabel "number of completed jobs"
|
|
|
193 set title "Cumulative jobs"
|
|
|
194 plot "plot_cumulative.txt" using 1:2 with lines
|
|
|
195 EOF1
|
|
|
196
|
|
|
197 gnuplot plotit.gp 2>/dev/null
|
|
|
198
|
|
|
199 #cp *.png ../
|
|
|
200 echo `pwd`>/tmp/curdir
|
|
|
201 echo `ls`>/tmp/dirlist
|
|
|
202 #cd ..
|
|
|
203 EOF
|
|
|
204
|
|
|
205 cat << EOF > $htmlfile
|
|
|
206 <!DOCTYPE HTML>
|
|
|
207
|
|
|
208 <html xmlns="http://www.w3.org/1999/xhtml">
|
|
|
209 <head>
|
|
|
210 <title>Hello output</title>
|
|
|
211 </head>
|
|
|
212 <body>
|
|
|
213 <h2>Browse output of hello application</h2>
|
|
|
214 `for i in \`find $outdir -type f\`; do echo "<a href=\`basename $i\`>Output: \`basename $i\`</a><br/>"; done`
|
|
|
215 </body>
|
|
|
216 </html>
|
|
|
217 EOF
|
|
|
218
|
|
|
219 if [ "$EXITCODE" -ne "0" ]; then
|
|
|
220 cat std.err >&2
|
|
|
221 fi
|
|
|
222 cp std.err $logfile
|
|
|
223 rm std.err
|
|
|
224 exit $EXITCODE
|
