Mercurial > repos > fubar > tool_factory_2

comparison toolfactory/rgToolFactory2.py @ 36:ce2b1f8ea68d draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
passes flake8 tests finally :)
author fubar
date Mon, 10 Aug 2020 23:24:41 -0400
parents 5052ac89c036
children a30536c100bf
comparison
equal deleted inserted replaced
35:5d38cb3d9be8 36:ce2b1f8ea68d
17 # galaxyxml now generates the tool xml https://github.com/hexylena/galaxyxml 17 # galaxyxml now generates the tool xml https://github.com/hexylena/galaxyxml
18 # No support for automatic HTML file creation from arbitrary outputs 18 # No support for automatic HTML file creation from arbitrary outputs
19 # TODO: add option to run that code as a post execution hook 19 # TODO: add option to run that code as a post execution hook
20 # TODO: add additional history input parameters - currently only one 20 # TODO: add additional history input parameters - currently only one
21 21
22
23 import argparse
24 import logging
25 import os
26 import re
27 import shutil
28 import subprocess
22 import sys 29 import sys
23 import subprocess 30 import tarfile
24 import shutil 31 import tempfile
25 import os
26 import time 32 import time
27 import tempfile 33
28 import argparse
29 import tarfile
30 import re
31 import galaxyxml.tool as gxt 34 import galaxyxml.tool as gxt
32 import galaxyxml.tool.parameters as gxtp 35 import galaxyxml.tool.parameters as gxtp
33 import logging 36 import lxml
34 37
35 38 foo = lxml.__name__ # fug you, flake8. Say my name! Please accept the PR, Helena!
36 39
37 progname = os.path.split(sys.argv[0])[1] 40 progname = os.path.split(sys.argv[0])[1]
38 myversion = 'V2.1 July 2020' 41 myversion = "V2.1 July 2020"
39 verbose = True 42 verbose = True
40 debug = True 43 debug = True
41 toolFactoryURL = 'https://github.com/fubar2/toolfactory' 44 toolFactoryURL = "https://github.com/fubar2/toolfactory"
42 ourdelim = '~~~' 45 ourdelim = "~~~"
43 46
44 # --input_files="$input_files~~~$CL~~~$input_formats~~~$input_label~~~$input_help" 47 # --input_files="$input_files~~~$CL~~~$input_formats~~~$input_label~~~$input_help"
45 IPATHPOS = 0 48 IPATHPOS = 0
46 ICLPOS = 1 49 ICLPOS = 1
47 IFMTPOS = 2 50 IFMTPOS = 2
52 ONAMEPOS = 0 55 ONAMEPOS = 0
53 OFMTPOS = 1 56 OFMTPOS = 1
54 OCLPOS = 2 57 OCLPOS = 2
55 OOCLPOS = 3 58 OOCLPOS = 3
56 59
57 #--additional_parameters="$i.param_name~~~$i.param_value~~~$i.param_label~~~$i.param_help~~~$i.param_type~~~$i.CL" 60 # --additional_parameters="$i.param_name~~~$i.param_value~~~$i.param_label~~~$i.param_help~~~$i.param_type~~~$i.CL"
58 ANAMEPOS = 0 61 ANAMEPOS = 0
59 AVALPOS = 1 62 AVALPOS = 1
60 ALABPOS = 2 63 ALABPOS = 2
61 AHELPPOS = 3 64 AHELPPOS = 3
62 ATYPEPOS = 4 65 ATYPEPOS = 4
63 ACLPOS = 5 66 ACLPOS = 5
64 AOCLPOS = 6 67 AOCLPOS = 6
65 68
69
66 def timenow(): 70 def timenow():
67 """return current time as a string 71 """return current time as a string
68 """ 72 """
69 return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time())) 73 return time.strftime("%d/%m/%Y %H:%M:%S", time.localtime(time.time()))
70 74
71 75
72 def quote_non_numeric(s): 76 def quote_non_numeric(s):
73 """return a prequoted string for non-numerics 77 """return a prequoted string for non-numerics
74 useful for perl and Rscript parameter passing? 78 useful for perl and Rscript parameter passing?
78 return s 82 return s
79 except ValueError: 83 except ValueError:
80 return '"%s"' % s 84 return '"%s"' % s
81 85
82 86
83 html_escape_table = { 87 html_escape_table = {"&": "&amp;", ">": "&gt;", "<": "&lt;", "$": r"\$"}
84 "&": "&amp;",
85 ">": "&gt;",
86 "<": "&lt;",
87 "$": r"\$"
88 }
89 88
90 89
91 def html_escape(text): 90 def html_escape(text):
92 """Produce entities within text.""" 91 """Produce entities within text."""
93 return "".join(html_escape_table.get(c, c) for c in text) 92 return "".join(html_escape_table.get(c, c) for c in text)
94 93
95 94
96 def html_unescape(text): 95 def html_unescape(text):
97 """Revert entities within text. Multiple character targets so use replace""" 96 """Revert entities within text. Multiple character targets so use replace"""
98 t = text.replace('&amp;', '&') 97 t = text.replace("&amp;", "&")
99 t = t.replace('&gt;', '>') 98 t = t.replace("&gt;", ">")
100 t = t.replace('&lt;', '<') 99 t = t.replace("&lt;", "<")
101 t = t.replace('\\$', '$') 100 t = t.replace("\\$", "$")
102 return t 101 return t
103 102
104 103
105 def parse_citations(citations_text): 104 def parse_citations(citations_text):
106 """ 105 """
109 citation_tuples = [] 108 citation_tuples = []
110 for citation in citations: 109 for citation in citations:
111 if citation.startswith("doi"): 110 if citation.startswith("doi"):
112 citation_tuples.append(("doi", citation[len("doi"):].strip())) 111 citation_tuples.append(("doi", citation[len("doi"):].strip()))
113 else: 112 else:
114 citation_tuples.append( 113 citation_tuples.append(("bibtex", citation[len("bibtex"):].strip()))
115 ("bibtex", citation[len("bibtex"):].strip()))
116 return citation_tuples 114 return citation_tuples
117 115
118 116
119 class ScriptRunner: 117 class ScriptRunner:
120 """Wrapper for an arbitrary script 118 """Wrapper for an arbitrary script
121 uses galaxyxml 119 uses galaxyxml
122 120
123 """ 121 """
124 122
125
126 def __init__(self, args=None): 123 def __init__(self, args=None):
127 """ 124 """
128 prepare command line cl for running the tool here 125 prepare command line cl for running the tool here
129 and prepare elements needed for galaxyxml tool generation 126 and prepare elements needed for galaxyxml tool generation
130 """ 127 """
136 self.cleanuppar() 133 self.cleanuppar()
137 self.lastclredirect = None 134 self.lastclredirect = None
138 self.lastxclredirect = None 135 self.lastxclredirect = None
139 self.cl = [] 136 self.cl = []
140 self.xmlcl = [] 137 self.xmlcl = []
138 self.is_positional = self.args.parampass == "positional"
141 aCL = self.cl.append 139 aCL = self.cl.append
142 assert args.parampass in ['0','argparse','positional'],'Parameter passing in args.parampass must be "0","positional" or "argparse"' 140 assert args.parampass in [
143 self.tool_name = re.sub('[^a-zA-Z0-9_]+', '', args.tool_name) 141 "0",
142 "argparse",
143 "positional",
144 ], 'Parameter passing in args.parampass must be "0","positional" or "argparse"'
145 self.tool_name = re.sub("[^a-zA-Z0-9_]+", "", args.tool_name)
144 self.tool_id = self.tool_name 146 self.tool_id = self.tool_name
145 self.xmlfile = '%s.xml' % self.tool_name 147 self.xmlfile = "%s.xml" % self.tool_name
146 if self.args.runmode == "Executable" or self.args.runmode == "system": # binary - no need 148 if self.args.interpreter_name:
149 exe = "$runMe"
150 else:
151 exe = self.args.exe_package
152 assert (
153 exe is not None
154 ), "No interpeter or executable passed in - nothing to run so cannot build"
155 self.tool = gxt.Tool(
156 self.args.tool_name,
157 self.tool_id,
158 self.args.tool_version,
159 self.args.tool_desc,
160 exe,
161 )
162 self.tinputs = gxtp.Inputs()
163 self.toutputs = gxtp.Outputs()
164 self.testparam = []
165 if (
166 self.args.runmode == "Executable" or self.args.runmode == "system"
167 ): # binary - no need
147 aCL(self.args.exe_package) # this little CL will just run 168 aCL(self.args.exe_package) # this little CL will just run
148 else: 169 else:
149 rx = open(self.args.script_path, 'r').readlines() 170 self.prepScript()
150 rx = [x.rstrip() for x in rx ]
151 rxcheck = [x.strip() for x in rx if x.strip() > '']
152 assert len(rxcheck) > 0,"Supplied script is empty. Cannot run"
153 self.script = '\n'.join(rx)
154 fhandle, self.sfile = tempfile.mkstemp(
155 prefix=self.tool_name, suffix=".%s" % (args.interpreter_name))
156 tscript = open(self.sfile, 'w')
157 tscript.write(self.script)
158 tscript.close()
159 self.indentedScript = " %s" % '\n'.join(
160 [' %s' % html_escape(x) for x in rx])
161 self.escapedScript = "%s" % '\n'.join(
162 [' %s' % html_escape(x) for x in rx])
163 art = '%s.%s' % (self.tool_name, args.interpreter_name)
164 artifact = open(art, 'wb')
165 artifact.write(bytes(self.script, "utf8"))
166 artifact.close()
167 aCL(self.args.interpreter_name)
168 aCL(self.sfile)
169 self.elog = "%s_error_log.txt" % self.tool_name 171 self.elog = "%s_error_log.txt" % self.tool_name
170 self.tlog = "%s_runner_log.txt" % self.tool_name 172 self.tlog = "%s_runner_log.txt" % self.tool_name
171 173
172 if self.args.parampass == '0': 174 if self.args.parampass == "0":
173 self.clsimple() 175 self.clsimple()
174 else: 176 else:
175 clsuffix = [] 177 clsuffix = []
176 xclsuffix = [] 178 xclsuffix = []
177 for i, p in enumerate(self.infiles): 179 for i, p in enumerate(self.infiles):
178 appendme = [p[IOCLPOS], p[ICLPOS], p[IPATHPOS]] 180 appendme = [p[IOCLPOS], p[ICLPOS], p[IPATHPOS]]
179 clsuffix.append(appendme) 181 clsuffix.append(appendme)
180 xclsuffix.append([p[IOCLPOS],p[ICLPOS],'$%s' % p[ICLPOS]]) 182 xclsuffix.append([p[IOCLPOS], p[ICLPOS], "$%s" % p[ICLPOS]])
181 #print('##infile i=%d, appendme=%s' % (i,appendme)) 183 # print('##infile i=%d, appendme=%s' % (i,appendme))
182 for i, p in enumerate(self.outfiles): 184 for i, p in enumerate(self.outfiles):
183 if p[OOCLPOS] == "STDOUT": 185 if p[OOCLPOS] == "STDOUT":
184 self.lastclredirect = ['>',p[ONAMEPOS]] 186 self.lastclredirect = [">", p[ONAMEPOS]]
185 self.lastxclredirect = ['>','$%s' % p[OCLPOS]] 187 self.lastxclredirect = [">", "$%s" % p[OCLPOS]]
186 #print('##outfiles i=%d lastclredirect = %s' % (i,self.lastclredirect)) 188 # print('##outfiles i=%d lastclredirect = %s' % (i,self.lastclredirect))
187 else: 189 else:
188 appendme = [p[OOCLPOS], p[OCLPOS],p[ONAMEPOS]] 190 appendme = [p[OOCLPOS], p[OCLPOS], p[ONAMEPOS]]
189 clsuffix.append(appendme) 191 clsuffix.append(appendme)
190 xclsuffix.append([p[OOCLPOS], p[OCLPOS],'$%s' % p[ONAMEPOS]]) 192 xclsuffix.append([p[OOCLPOS], p[OCLPOS], "$%s" % p[ONAMEPOS]])
191 #print('##outfiles i=%d' % i,'appendme',appendme) 193 # print('##outfiles i=%d' % i,'appendme',appendme)
192 for p in self.addpar: 194 for p in self.addpar:
193 appendme = [p[AOCLPOS], p[ACLPOS], p[AVALPOS]] 195 appendme = [p[AOCLPOS], p[ACLPOS], p[AVALPOS]]
194 clsuffix.append(appendme) 196 clsuffix.append(appendme)
195 xclsuffix.append([p[AOCLPOS], p[ACLPOS], '"$%s"' % p[ANAMEPOS]]) 197 xclsuffix.append([p[AOCLPOS], p[ACLPOS], '"$%s"' % p[ANAMEPOS]])
196 #print('##adpar %d' % i,'appendme=',appendme) 198 # print('##adpar %d' % i,'appendme=',appendme)
197 clsuffix.sort() 199 clsuffix.sort()
198 xclsuffix.sort() 200 xclsuffix.sort()
199 self.xclsuffix = xclsuffix 201 self.xclsuffix = xclsuffix
200 self.clsuffix = clsuffix 202 self.clsuffix = clsuffix
201 if self.args.parampass == 'positional': 203 if self.args.parampass == "positional":
202 self.clpositional() 204 self.clpositional()
203 else: 205 else:
204 self.clargparse() 206 self.clargparse()
205 207
208 def prepScript(self):
209 aCL = self.cl.append
210 rx = open(self.args.script_path, "r").readlines()
211 rx = [x.rstrip() for x in rx]
212 rxcheck = [x.strip() for x in rx if x.strip() > ""]
213 assert len(rxcheck) > 0, "Supplied script is empty. Cannot run"
214 self.script = "\n".join(rx)
215 fhandle, self.sfile = tempfile.mkstemp(
216 prefix=self.tool_name, suffix=".%s" % (self.args.interpreter_name)
217 )
218 tscript = open(self.sfile, "w")
219 tscript.write(self.script)
220 tscript.close()
221 self.indentedScript = " %s" % "\n".join([" %s" % html_escape(x) for x in rx])
222 self.escapedScript = "%s" % "\n".join([" %s" % html_escape(x) for x in rx])
223 art = "%s.%s" % (self.tool_name, self.args.interpreter_name)
224 artifact = open(art, "wb")
225 artifact.write(bytes(self.script, "utf8"))
226 artifact.close()
227 aCL(self.args.interpreter_name)
228 aCL(self.sfile)
229
206 def cleanuppar(self): 230 def cleanuppar(self):
207 """ positional parameters are complicated by their numeric ordinal""" 231 """ positional parameters are complicated by their numeric ordinal"""
208 for i,p in enumerate(self.infiles): 232 for i, p in enumerate(self.infiles):
209 if self.args.parampass == 'positional': 233 if self.args.parampass == "positional":
210 assert p[ICLPOS].isdigit(), "Positional parameters must be ordinal integers - got %s for %s" % (p[ICLPOS],p[ILABPOS]) 234 assert p[ICLPOS].isdigit(), (
235 "Positional parameters must be ordinal integers - got %s for %s"
236 % (p[ICLPOS], p[ILABPOS])
237 )
211 p.append(p[ICLPOS]) 238 p.append(p[ICLPOS])
212 if p[ICLPOS].isdigit() or self.args.parampass == "0": 239 if p[ICLPOS].isdigit() or self.args.parampass == "0":
213 scl = 'input%d' % (i+1) 240 scl = "input%d" % (i + 1)
214 p[ICLPOS] = scl 241 p[ICLPOS] = scl
215 self.infiles[i] = p 242 self.infiles[i] = p
216 for i,p in enumerate(self.outfiles): # trying to automagically gather using extensions 243 for i, p in enumerate(
217 if self.args.parampass == 'positional' and p[OCLPOS] != "STDOUT": 244 self.outfiles
218 assert p[OCLPOS].isdigit(), "Positional parameters must be ordinal integers - got %s for %s" % (p[OCLPOS],p[ONAMEPOS]) 245 ): # trying to automagically gather using extensions
246 if self.args.parampass == "positional" and p[OCLPOS] != "STDOUT":
247 assert p[OCLPOS].isdigit(), (
248 "Positional parameters must be ordinal integers - got %s for %s"
249 % (p[OCLPOS], p[ONAMEPOS])
250 )
219 p.append(p[OCLPOS]) 251 p.append(p[OCLPOS])
220 if p[OCLPOS].isdigit() or p[OCLPOS] == "STDOUT": 252 if p[OCLPOS].isdigit() or p[OCLPOS] == "STDOUT":
221 scl = p[ONAMEPOS] 253 scl = p[ONAMEPOS]
222 p[OCLPOS] = scl 254 p[OCLPOS] = scl
223 self.outfiles[i] = p 255 self.outfiles[i] = p
224 for i,p in enumerate(self.addpar): 256 for i, p in enumerate(self.addpar):
225 if self.args.parampass == 'positional': 257 if self.args.parampass == "positional":
226 assert p[ACLPOS].isdigit(), "Positional parameters must be ordinal integers - got %s for %s" % (p[ACLPOS],p[ANAMEPOS]) 258 assert p[ACLPOS].isdigit(), (
259 "Positional parameters must be ordinal integers - got %s for %s"
260 % (p[ACLPOS], p[ANAMEPOS])
261 )
227 p.append(p[ACLPOS]) 262 p.append(p[ACLPOS])
228 if p[ACLPOS].isdigit(): 263 if p[ACLPOS].isdigit():
229 scl = 'input%s' % p[ACLPOS] 264 scl = "input%s" % p[ACLPOS]
230 p[ACLPOS] = scl 265 p[ACLPOS] = scl
231 self.addpar[i] = p 266 self.addpar[i] = p
232 267
233
234
235 def clsimple(self): 268 def clsimple(self):
236 """ no parameters - uses < and > for i/o 269 """ no parameters - uses < and > for i/o
237 """ 270 """
238 aCL = self.cl.append 271 aCL = self.cl.append
239 aCL('<') 272 aCL("<")
240 aCL(self.infiles[0][IPATHPOS]) 273 aCL(self.infiles[0][IPATHPOS])
241 aCL('>') 274 aCL(">")
242 aCL(self.outfiles[0][OCLPOS]) 275 aCL(self.outfiles[0][OCLPOS])
243 aXCL = self.xmlcl.append 276 aXCL = self.xmlcl.append
244 aXCL('<') 277 aXCL("<")
245 aXCL('$%s' % self.infiles[0][ICLPOS]) 278 aXCL("$%s" % self.infiles[0][ICLPOS])
246 aXCL('>') 279 aXCL(">")
247 aXCL('$%s' % self.outfiles[0][ONAMEPOS]) 280 aXCL("$%s" % self.outfiles[0][ONAMEPOS])
248
249 281
250 def clpositional(self): 282 def clpositional(self):
251 # inputs in order then params 283 # inputs in order then params
252 aCL = self.cl.append 284 aCL = self.cl.append
253 for (o_v,k, v) in self.clsuffix: 285 for (o_v, k, v) in self.clsuffix:
254 if " " in v: 286 if " " in v:
255 aCL("%s" % v) 287 aCL("%s" % v)
256 else: 288 else:
257 aCL(v) 289 aCL(v)
258 aXCL = self.xmlcl.append 290 aXCL = self.xmlcl.append
259 for (o_v,k, v) in self.xclsuffix: 291 for (o_v, k, v) in self.xclsuffix:
260 aXCL(v) 292 aXCL(v)
261 if self.lastxclredirect: 293 if self.lastxclredirect:
262 aXCL(self.lastxclredirect[0]) 294 aXCL(self.lastxclredirect[0])
263 aXCL(self.lastxclredirect[1]) 295 aXCL(self.lastxclredirect[1])
264
265
266 296
267 def clargparse(self): 297 def clargparse(self):
268 """ argparse style 298 """ argparse style
269 """ 299 """
270 aCL = self.cl.append 300 aCL = self.cl.append
271 aXCL = self.xmlcl.append 301 aXCL = self.xmlcl.append
272 # inputs then params in argparse named form 302 # inputs then params in argparse named form
273 for (o_v,k, v) in self.xclsuffix: 303 for (o_v, k, v) in self.xclsuffix:
274 if len(k.strip()) == 1: 304 if len(k.strip()) == 1:
275 k = '-%s' % k 305 k = "-%s" % k
276 else: 306 else:
277 k = '--%s' % k 307 k = "--%s" % k
278 aXCL(k) 308 aXCL(k)
279 aXCL(v) 309 aXCL(v)
280 for (o_v,k, v) in self.clsuffix: 310 for (o_v, k, v) in self.clsuffix:
281 if len(k.strip()) == 1: 311 if len(k.strip()) == 1:
282 k = '-%s' % k 312 k = "-%s" % k
283 else: 313 else:
284 k = '--%s' % k 314 k = "--%s" % k
285 aCL(k) 315 aCL(k)
286 aCL(v) 316 aCL(v)
287 317
288 318 def getNdash(self, newname):
289 319 if self.is_positional:
320 ndash = 0
321 else:
322 ndash = 2
323 if len(newname) < 2:
324 ndash = 1
325 return ndash
326
327 def doXMLparam(self):
328 """flake8 made me do this..."""
329 for p in self.outfiles:
330 newname, newfmt, newcl, oldcl = p
331 ndash = self.getNdash(newcl)
332 aparm = gxtp.OutputData(newcl, format=newfmt, num_dashes=ndash)
333 aparm.positional = self.is_positional
334 if self.is_positional:
335 if oldcl == "STDOUT":
336 aparm.positional = 9999999
337 aparm.command_line_override = "> $%s" % newcl
338 else:
339 aparm.positional = int(oldcl)
340 aparm.command_line_override = "$%s" % newcl
341 self.toutputs.append(aparm)
342 tp = gxtp.TestOutput(name=newcl, value="%s_sample" % newcl, format=newfmt)
343 self.testparam.append(tp)
344 for p in self.infiles:
345 newname = p[ICLPOS]
346 newfmt = p[IFMTPOS]
347 ndash = self.getNdash(newname)
348 if not len(p[ILABPOS]) > 0:
349 alab = p[ICLPOS]
350 else:
351 alab = p[ILABPOS]
352 aninput = gxtp.DataParam(
353 newname,
354 optional=False,
355 label=alab,
356 help=p[IHELPOS],
357 format=newfmt,
358 multiple=False,
359 num_dashes=ndash,
360 )
361 aninput.positional = self.is_positional
362 self.tinputs.append(aninput)
363 tparm = gxtp.TestParam(name=newname, value="%s_sample" % newname)
364 self.testparam.append(tparm)
365 for p in self.addpar:
366 newname, newval, newlabel, newhelp, newtype, newcl, oldcl = p
367 if not len(newlabel) > 0:
368 newlabel = newname
369 ndash = self.getNdash(newname)
370 if newtype == "text":
371 aparm = gxtp.TextParam(
372 newname,
373 label=newlabel,
374 help=newhelp,
375 value=newval,
376 num_dashes=ndash,
377 )
378 elif newtype == "integer":
379 aparm = gxtp.IntegerParam(
380 newname,
381 label=newname,
382 help=newhelp,
383 value=newval,
384 num_dashes=ndash,
385 )
386 elif newtype == "float":
387 aparm = gxtp.FloatParam(
388 newname,
389 label=newname,
390 help=newhelp,
391 value=newval,
392 num_dashes=ndash,
393 )
394 else:
395 raise ValueError(
396 'Unrecognised parameter type "%s" for\
397 additional parameter %s in makeXML'
398 % (newtype, newname)
399 )
400 aparm.positional = self.is_positional
401 if self.is_positional:
402 aninput.positional = int(oldcl)
403 self.tinputs.append(aparm)
404 self.tparm = gxtp.TestParam(newname, value=newval)
405 self.testparam.append(tparm)
406
407 def doNoXMLparam(self):
408 alab = self.infiles[0][ILABPOS]
409 if len(alab) == 0:
410 alab = self.infiles[0][ICLPOS]
411 max1s = (
412 "Maximum one input if parampass is 0 - more than one input files supplied - %s"
413 % str(self.infiles)
414 )
415 assert len(self.infiles) == 1, max1s
416 newname = self.infiles[0][ICLPOS]
417 aninput = gxtp.DataParam(
418 newname,
419 optional=False,
420 label=alab,
421 help=self.infiles[0][IHELPOS],
422 format=self.infiles[0][IFMTPOS],
423 multiple=False,
424 num_dashes=0,
425 )
426 aninput.command_line_override = "< $%s" % newname
427 aninput.positional = self.is_positional
428 self.tinputs.append(aninput)
429 tp = gxtp.TestParam(name=newname, value="%s_sample" % newname)
430 self.testparam.append(tp)
431 newname = self.outfiles[0][OCLPOS]
432 newfmt = self.outfiles[0][OFMTPOS]
433 anout = gxtp.OutputData(newname, format=newfmt, num_dashes=0)
434 anout.command_line_override = "> $%s" % newname
435 anout.positional = self.is_positional
436 self.toutputs.append(anout)
437 tp = gxtp.TestOutput(name=newname, value="%s_sample" % newname, format=newfmt)
438 self.testparam.append(tp)
290 439
291 def makeXML(self): 440 def makeXML(self):
292 """ 441 """
293 Create a Galaxy xml tool wrapper for the new script 442 Create a Galaxy xml tool wrapper for the new script
294 Uses galaxyhtml 443 Uses galaxyhtml
295 Hmmm. How to get the command line into correct order... 444 Hmmm. How to get the command line into correct order...
296 """ 445 """
297 446 self.tool.command_line_override = self.xmlcl
298 if self.args.interpreter_name: 447 if self.args.interpreter_name:
299 exe = "$runMe" 448 self.tool.interpreter = self.interp
300 interp = self.args.interpreter_name
301 else:
302 interp = None
303 exe = self.args.exe_package
304 assert exe is not None, 'No interpeter or executable passed in to makeXML'
305 tool = gxt.Tool(self.args.tool_name, self.tool_id,
306 self.args.tool_version, self.args.tool_desc, exe)
307 tool.command_line_override = self.xmlcl
308 if interp:
309 tool.interpreter = interp
310 if self.args.help_text: 449 if self.args.help_text:
311 helptext = open(self.args.help_text, 'r').readlines() 450 helptext = open(self.args.help_text, "r").readlines()
312 helptext = [html_escape(x) for x in helptext] 451 helptext = [html_escape(x) for x in helptext]
313 tool.help = ''.join([x for x in helptext]) 452 self.tool.help = "".join([x for x in helptext])
314 else: 453 else:
315 tool.help = 'Please ask the tool author (%s) for help \ 454 self.tool.help = (
316 as none was supplied at tool generation\n' % (self.args.user_email) 455 "Please ask the tool author (%s) for help \
317 tool.version_command = None # do not want 456 as none was supplied at tool generation\n"
318 tinputs = gxtp.Inputs() 457 % (self.args.user_email)
319 toutputs = gxtp.Outputs() 458 )
459 self.tool.version_command = None # do not want
320 requirements = gxtp.Requirements() 460 requirements = gxtp.Requirements()
321 testparam = [] 461
322 is_positional = (self.args.parampass == 'positional')
323 if self.args.interpreter_name: 462 if self.args.interpreter_name:
324 if self.args.interpreter_name == 'python': 463 if self.args.interpreter_name == "python":
325 requirements.append(gxtp.Requirement( 464 requirements.append(
326 'package', 'python', self.args.interpreter_version)) 465 gxtp.Requirement("package", "python", self.args.interpreter_version)
327 elif self.args.interpreter_name not in ['bash', 'sh']: 466 )
328 requirements.append(gxtp.Requirement( 467 elif self.args.interpreter_name not in ["bash", "sh"]:
329 'package', self.args.interpreter_name, self.args.interpreter_version)) 468 requirements.append(
469 gxtp.Requirement(
470 "package",
471 self.args.interpreter_name,
472 self.args.interpreter_version,
473 )
474 )
330 else: 475 else:
331 if self.args.exe_package and self.args.parampass != "system": 476 if self.args.exe_package and self.args.parampass != "system":
332 requirements.append(gxtp.Requirement( 477 requirements.append(
333 'package', self.args.exe_package, self.args.exe_package_version)) 478 gxtp.Requirement(
334 tool.requirements = requirements 479 "package", self.args.exe_package, self.args.exe_package_version
335 if self.args.parampass == '0': 480 )
336 alab = self.infiles[0][ILABPOS] 481 )
337 if len(alab) == 0: 482 self.tool.requirements = requirements
338 alab = self.infiles[0][ICLPOS] 483 if self.args.parampass == "0":
339 max1s = 'Maximum one input if parampass is 0 - more than one input files supplied - %s' % str(self.infiles) 484 self.doXMLNoparam()
340 assert len(self.infiles) == 1,max1s 485 else:
341 newname = self.infiles[0][ICLPOS] 486 self.doXMLParam()
342 aninput = gxtp.DataParam(newname, optional=False, label=alab, help=self.infiles[0][IHELPOS], 487 self.tool.outputs = self.toutputs
343 format=self.infiles[0][IFMTPOS], multiple=False, num_dashes=0) 488 self.tool.inputs = self.tinputs
344 aninput.command_line_override = '< $%s' % newname 489 if self.args.runmode not in ["Executable", "system"]:
345 aninput.positional = is_positional
346 tinputs.append(aninput)
347 tp = gxtp.TestParam(name=newname, value='%s_sample' % newname)
348 testparam.append(tp)
349 newname = self.outfiles[0][OCLPOS]
350 newfmt = self.outfiles[0][OFMTPOS]
351 anout = gxtp.OutputData(newname, format=newfmt, num_dashes=0)
352 anout.command_line_override = '> $%s' % newname
353 anout.positional = is_positional
354 toutputs.append(anout)
355 tp = gxtp.TestOutput(name=newname, value='%s_sample' % newname,format=newfmt)
356 testparam.append(tp)
357 else:
358 for p in self.outfiles:
359 newname,newfmt,newcl,oldcl = p
360 if is_positional:
361 ndash = 0
362 else:
363 ndash = 2
364 if len(newcl) < 2:
365 ndash = 1
366 aparm = gxtp.OutputData(newcl, format=newfmt, num_dashes=ndash)
367 aparm.positional = is_positional
368 if is_positional:
369 if oldcl == "STDOUT":
370 aparm.positional = 9999999
371 aparm.command_line_override = "> $%s" % newcl
372 else:
373 aparm.positional = int(oldcl)
374 aparm.command_line_override = '$%s' % newcl
375 toutputs.append(aparm)
376 tp = gxtp.TestOutput(name=newcl, value='%s_sample' % newcl ,format=newfmt)
377 testparam.append(tp)
378 for p in self.infiles:
379 newname = p[ICLPOS]
380 newfmt = p[IFMTPOS]
381 if is_positional:
382 ndash = 0
383 else:
384 if len(newname) > 1:
385 ndash = 2
386 else:
387 ndash = 1
388 if not len(p[ILABPOS]) > 0:
389 alab = p[ICLPOS]
390 else:
391 alab = p[ILABPOS]
392 aninput = gxtp.DataParam(newname, optional=False, label=alab, help=p[IHELPOS],
393 format=newfmt, multiple=False, num_dashes=ndash)
394 aninput.positional = is_positional
395 if is_positional:
396 aninput.positional = is_positional
397 tinputs.append(aninput)
398 tparm = gxtp.TestParam(name=newname, value='%s_sample' % newname )
399 testparam.append(tparm)
400 for p in self.addpar:
401 newname, newval, newlabel, newhelp, newtype, newcl, oldcl = p
402 if not len(newlabel) > 0:
403 newlabel = newname
404 if is_positional:
405 ndash = 0
406 else:
407 if len(newname) > 1:
408 ndash = 2
409 else:
410 ndash = 1
411 if newtype == "text":
412 aparm = gxtp.TextParam(
413 newname, label=newlabel, help=newhelp, value=newval, num_dashes=ndash)
414 elif newtype == "integer":
415 aparm = gxtp.IntegerParam(
416 newname, label=newname, help=newhelp, value=newval, num_dashes=ndash)
417 elif newtype == "float":
418 aparm = gxtp.FloatParam(
419 newname, label=newname, help=newhelp, value=newval, num_dashes=ndash)
420 else:
421 raise ValueError('Unrecognised parameter type "%s" for\
422 additional parameter %s in makeXML' % (newtype, newname))
423 aparm.positional = is_positional
424 if is_positional:
425 aninput.positional = int(oldcl)
426 tinputs.append(aparm)
427 tparm = gxtp.TestParam(newname, value=newval)
428 testparam.append(tparm)
429 tool.outputs = toutputs
430 tool.inputs = tinputs
431 if not self.args.runmode in ['Executable','system']:
432 configfiles = gxtp.Configfiles() 490 configfiles = gxtp.Configfiles()
433 configfiles.append(gxtp.Configfile(name="runMe", text=self.script)) 491 configfiles.append(gxtp.Configfile(name="runMe", text=self.script))
434 tool.configfiles = configfiles 492 self.tool.configfiles = configfiles
435 tests = gxtp.Tests() 493 tests = gxtp.Tests()
436 test_a = gxtp.Test() 494 test_a = gxtp.Test()
437 for tp in testparam: 495 for tp in self.testparam:
438 test_a.append(tp) 496 test_a.append(tp)
439 tests.append(test_a) 497 tests.append(test_a)
440 tool.tests = tests 498 self.tool.tests = tests
441 tool.add_comment('Created by %s at %s using the Galaxy Tool Factory.' % ( 499 self.tool.add_comment(
442 self.args.user_email, timenow())) 500 "Created by %s at %s using the Galaxy Tool Factory."
443 tool.add_comment('Source in git at: %s' % (toolFactoryURL)) 501 % (self.args.user_email, timenow())
444 tool.add_comment( 502 )
445 'Cite: Creating re-usable tools from scripts doi: 10.1093/bioinformatics/bts573') 503 self.tool.add_comment("Source in git at: %s" % (toolFactoryURL))
446 exml = tool.export() 504 self.tool.add_comment(
447 xf = open(self.xmlfile, 'w') 505 "Cite: Creating re-usable tools from scripts doi: 10.1093/bioinformatics/bts573"
506 )
507 exml = self.tool.export()
508 xf = open(self.xmlfile, "w")
448 xf.write(exml) 509 xf.write(exml)
449 xf.write('\n') 510 xf.write("\n")
450 xf.close() 511 xf.close()
451 # ready for the tarball 512 # ready for the tarball
452 513
453 def makeTooltar(self): 514 def makeTooltar(self):
454 """ 515 """
457 NOTE names for test inputs and outputs are munged here so must 518 NOTE names for test inputs and outputs are munged here so must
458 correspond to actual input and output names used on the generated cl 519 correspond to actual input and output names used on the generated cl
459 """ 520 """
460 retval = self.run() 521 retval = self.run()
461 if retval: 522 if retval:
462 sys.stderr.write( 523 sys.stderr.write("## Run failed. Cannot build yet. Please fix and retry")
463 '## Run failed. Cannot build yet. Please fix and retry')
464 sys.exit(1) 524 sys.exit(1)
465 tdir = 'tfout' 525 tdir = "tfout"
466 if not os.path.exists(tdir): 526 if not os.path.exists(tdir):
467 os.mkdir(tdir) 527 os.mkdir(tdir)
468 self.makeXML() 528 self.makeXML()
469 testdir = os.path.join(tdir,'test-data') 529 testdir = os.path.join(tdir, "test-data")
470 if not os.path.exists(testdir): 530 if not os.path.exists(testdir):
471 os.mkdir(testdir) # make tests directory 531 os.mkdir(testdir) # make tests directory
472 for p in self.infiles: 532 for p in self.infiles:
473 pth = p[IPATHPOS] 533 pth = p[IPATHPOS]
474 dest = os.path.join(testdir, '%s_sample' % 534 dest = os.path.join(testdir, "%s_sample" % p[ICLPOS])
475 p[ICLPOS])
476 shutil.copyfile(pth, dest) 535 shutil.copyfile(pth, dest)
477 for p in self.outfiles: 536 for p in self.outfiles:
478 pth = p[OCLPOS] 537 pth = p[OCLPOS]
479 if p[OOCLPOS] == 'STDOUT' or self.args.parampass == "0": 538 if p[OOCLPOS] == "STDOUT" or self.args.parampass == "0":
480 pth = p[ONAMEPOS] 539 pth = p[ONAMEPOS]
481 dest = os.path.join(testdir,'%s_sample' % p[ONAMEPOS]) 540 dest = os.path.join(testdir, "%s_sample" % p[ONAMEPOS])
482 shutil.copyfile(pth, dest) 541 shutil.copyfile(pth, dest)
483 dest = os.path.join(tdir, p[ONAMEPOS]) 542 dest = os.path.join(tdir, p[ONAMEPOS])
484 shutil.copyfile(pth, dest) 543 shutil.copyfile(pth, dest)
485 else: 544 else:
486 pth = p[OCLPOS] 545 pth = p[OCLPOS]
487 dest = os.path.join(testdir,'%s_sample' % p[OCLPOS]) 546 dest = os.path.join(testdir, "%s_sample" % p[OCLPOS])
488 shutil.copyfile(pth, dest) 547 shutil.copyfile(pth, dest)
489 dest = os.path.join(tdir, p[OCLPOS]) 548 dest = os.path.join(tdir, p[OCLPOS])
490 shutil.copyfile(pth, dest) 549 shutil.copyfile(pth, dest)
491 550
492 if os.path.exists(self.tlog) and os.stat(self.tlog).st_size > 0: 551 if os.path.exists(self.tlog) and os.stat(self.tlog).st_size > 0:
493 shutil.copyfile(self.tlog, os.path.join( 552 shutil.copyfile(self.tlog, os.path.join(testdir, "test1_log.txt"))
494 testdir, 'test1_log.txt')) 553 if self.args.runmode not in ["Executable", "system"]:
495 if not self.args.runmode in ['Executable','system']: 554 stname = os.path.join(tdir, "%s" % (self.sfile))
496 stname = os.path.join(tdir, '%s' % (self.sfile))
497 if not os.path.exists(stname): 555 if not os.path.exists(stname):
498 shutil.copyfile(self.sfile, stname) 556 shutil.copyfile(self.sfile, stname)
499 xtname = os.path.join(tdir,self.xmlfile) 557 xtname = os.path.join(tdir, self.xmlfile)
500 if not os.path.exists(xtname): 558 if not os.path.exists(xtname):
501 shutil.copyfile(self.xmlfile, xtname) 559 shutil.copyfile(self.xmlfile, xtname)
502 tarpath = 'toolfactory_%s.tgz' % self.tool_name 560 tarpath = "toolfactory_%s.tgz" % self.tool_name
503 tf = tarfile.open(tarpath,"w:gz") 561 tf = tarfile.open(tarpath, "w:gz")
504 tf.add(name=tdir,arcname=self.tool_name) 562 tf.add(name=tdir, arcname=self.tool_name)
505 tf.close() 563 tf.close()
506 shutil.copyfile(tarpath, self.args.new_tool) 564 shutil.copyfile(tarpath, self.args.new_tool)
507 return retval 565 return retval
508 566
509 def run(self): 567 def run(self):
510 """ 568 """
511 Some devteam tools have this defensive stderr read so I'm keeping with the faith 569 Some devteam tools have this defensive stderr read so I'm keeping with the faith
512 Feel free to update. 570 Feel free to update.
513 """ 571 """
514 s = 'run cl=%s' % str(self.cl) 572 s = "run cl=%s" % str(self.cl)
515 573
516 logging.debug(s) 574 logging.debug(s)
517 scl = ' '.join(self.cl) 575 scl = " ".join(self.cl)
518 err = None 576 err = None
519 if self.args.parampass != '0': 577 if self.args.parampass != "0":
520 ste = open(self.elog, 'wb') 578 ste = open(self.elog, "wb")
521 if self.lastclredirect: 579 if self.lastclredirect:
522 sto = open(self.lastclredirect[1],'wb') # is name of an output file 580 sto = open(self.lastclredirect[1], "wb") # is name of an output file
523 else: 581 else:
524 sto = open(self.tlog, 'wb') 582 sto = open(self.tlog, "wb")
525 sto.write( 583 sto.write(
526 bytes('## Executing Toolfactory generated command line = %s\n' % scl, "utf8")) 584 bytes(
585 "## Executing Toolfactory generated command line = %s\n" % scl,
586 "utf8",
587 )
588 )
527 sto.flush() 589 sto.flush()
528 p = subprocess.run(self.cl, shell=False, stdout=sto, 590 p = subprocess.run(self.cl, shell=False, stdout=sto, stderr=ste)
529 stderr=ste)
530 sto.close() 591 sto.close()
531 ste.close() 592 ste.close()
532 tmp_stderr = open(self.elog, 'rb') 593 tmp_stderr = open(self.elog, "rb")
533 err = '' 594 err = ""
534 buffsize = 1048576 595 buffsize = 1048576
535 try: 596 try:
536 while True: 597 while True:
537 err += str(tmp_stderr.read(buffsize)) 598 err += str(tmp_stderr.read(buffsize))
538 if not err or len(err) % buffsize != 0: 599 if not err or len(err) % buffsize != 0:
540 except OverflowError: 601 except OverflowError:
541 pass 602 pass
542 tmp_stderr.close() 603 tmp_stderr.close()
543 retval = p.returncode 604 retval = p.returncode
544 else: # work around special case of simple scripts that take stdin and write to stdout 605 else: # work around special case of simple scripts that take stdin and write to stdout
545 sti = open(self.infiles[0][IPATHPOS], 'rb') 606 sti = open(self.infiles[0][IPATHPOS], "rb")
546 sto = open(self.outfiles[0][ONAMEPOS], 'wb') 607 sto = open(self.outfiles[0][ONAMEPOS], "wb")
547 # must use shell to redirect 608 # must use shell to redirect
548 p = subprocess.run(self.cl, shell=False, stdout=sto, stdin=sti) 609 p = subprocess.run(self.cl, shell=False, stdout=sto, stdin=sti)
549 retval = p.returncode 610 retval = p.returncode
550 sto.close() 611 sto.close()
551 sti.close() 612 sti.close()
553 os.unlink(self.tlog) 614 os.unlink(self.tlog)
554 if os.path.isfile(self.elog) and os.stat(self.elog).st_size == 0: 615 if os.path.isfile(self.elog) and os.stat(self.elog).st_size == 0:
555 os.unlink(self.elog) 616 os.unlink(self.elog)
556 if p.returncode != 0 and err: # problem 617 if p.returncode != 0 and err: # problem
557 sys.stderr.write(err) 618 sys.stderr.write(err)
558 logging.debug('run done') 619 logging.debug("run done")
559 return retval 620 return retval
560 621
561 622
562 def main(): 623 def main():
563 """ 624 """
565 <command interpreter="python">rgBaseScriptWrapper.py --script_path "$scriptPath" --tool_name "foo" --interpreter "Rscript" 626 <command interpreter="python">rgBaseScriptWrapper.py --script_path "$scriptPath" --tool_name "foo" --interpreter "Rscript"
566 </command> 627 </command>
567 """ 628 """
568 parser = argparse.ArgumentParser() 629 parser = argparse.ArgumentParser()
569 a = parser.add_argument 630 a = parser.add_argument
570 a('--script_path', default='') 631 a("--script_path", default="")
571 a('--tool_name', default=None) 632 a("--tool_name", default=None)
572 a('--interpreter_name', default=None) 633 a("--interpreter_name", default=None)
573 a('--interpreter_version', default=None) 634 a("--interpreter_version", default=None)
574 a('--exe_package', default=None) 635 a("--exe_package", default=None)
575 a('--exe_package_version', default=None) 636 a("--exe_package_version", default=None)
576 a('--input_files', default=[], action="append") 637 a("--input_files", default=[], action="append")
577 a('--output_files', default=[], action="append") 638 a("--output_files", default=[], action="append")
578 a('--user_email', default='Unknown') 639 a("--user_email", default="Unknown")
579 a('--bad_user', default=None) 640 a("--bad_user", default=None)
580 a('--make_Tool', default=None) 641 a("--make_Tool", default=None)
581 a('--help_text', default=None) 642 a("--help_text", default=None)
582 a('--tool_desc', default=None) 643 a("--tool_desc", default=None)
583 a('--tool_version', default=None) 644 a("--tool_version", default=None)
584 a('--citations', default=None) 645 a("--citations", default=None)
585 a('--additional_parameters', action='append', default=[]) 646 a("--additional_parameters", action="append", default=[])
586 a('--edit_additional_parameters', action="store_true", default=False) 647 a("--edit_additional_parameters", action="store_true", default=False)
587 a('--parampass', default="positional") 648 a("--parampass", default="positional")
588 a('--tfout', default="./tfout") 649 a("--tfout", default="./tfout")
589 a('--new_tool',default="new_tool") 650 a("--new_tool", default="new_tool")
590 a('--runmode',default=None) 651 a("--runmode", default=None)
591 args = parser.parse_args() 652 args = parser.parse_args()
592 assert not args.bad_user, 'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy admin adds %s to "admin_users" in the Galaxy configuration file' % ( 653 assert not args.bad_user, (
593 args.bad_user, args.bad_user) 654 'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy admin adds %s to "admin_users" in the Galaxy configuration file'
594 assert args.tool_name, '## Tool Factory expects a tool name - eg --tool_name=DESeq' 655 % (args.bad_user, args.bad_user)
595 assert (args.interpreter_name or args.exe_package), '## Tool Factory wrapper expects an interpreter - eg --interpreter_name=Rscript or an executable package findable by the dependency management package' 656 )
596 assert args.exe_package or (len(args.script_path) > 0 and os.path.isfile( 657 assert args.tool_name, "## Tool Factory expects a tool name - eg --tool_name=DESeq"
597 args.script_path)), '## Tool Factory wrapper expects a script path - eg --script_path=foo.R if no executable' 658 assert (
598 args.input_files = [x.replace('"', '').replace("'", '') 659 args.interpreter_name or args.exe_package
599 for x in args.input_files] 660 ), "## Tool Factory wrapper expects an interpreter or an executable package"
661 assert args.exe_package or (
662 len(args.script_path) > 0 and os.path.isfile(args.script_path)
663 ), "## Tool Factory wrapper expects a script path - eg --script_path=foo.R if no executable"
664 args.input_files = [x.replace('"', "").replace("'", "") for x in args.input_files]
600 # remove quotes we need to deal with spaces in CL params 665 # remove quotes we need to deal with spaces in CL params
601 for i, x in enumerate(args.additional_parameters): 666 for i, x in enumerate(args.additional_parameters):
602 args.additional_parameters[i] = args.additional_parameters[i].replace( 667 args.additional_parameters[i] = args.additional_parameters[i].replace('"', "")
603 '"', '')
604 r = ScriptRunner(args) 668 r = ScriptRunner(args)
605 if args.make_Tool: 669 if args.make_Tool:
606 retcode = r.makeTooltar() 670 retcode = r.makeTooltar()
607 else: 671 else:
608 retcode = r.run() 672 retcode = r.run()