Mercurial > repos > fubar > tool_factory_2

comparison toolfactory/rgToolFactory2.py @ 49:35a912ce0c83 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Can now make the bwa example from planemo :)
author fubar
date Thu, 27 Aug 2020 23:11:01 -0400
parents 5a7a5b06bce0
children bf432f4486c7
comparison
equal deleted inserted replaced
48:5a7a5b06bce0 49:35a912ce0c83
4 # 4 #
5 # copyright ross lazarus (ross stop lazarus at gmail stop com) May 2012 5 # copyright ross lazarus (ross stop lazarus at gmail stop com) May 2012
6 # 6 #
7 # all rights reserved 7 # all rights reserved
8 # Licensed under the LGPL 8 # Licensed under the LGPL
9 # suggestions for improvement and bug fixes welcome at https://github.com/fubar2/toolfactory 9 # suggestions for improvement and bug fixes welcome at
10 # https://github.com/fubar2/toolfactory
10 # 11 #
11 # July 2020: BCC was fun and I feel like rip van winkle after 5 years. 12 # July 2020: BCC was fun and I feel like rip van winkle after 5 years.
12 # Decided to 13 # Decided to
13 # 1. Fix the toolfactory so it works - done for simplest case 14 # 1. Fix the toolfactory so it works - done for simplest case
14 # 2. Fix planemo so the toolfactory function works 15 # 2. Fix planemo so the toolfactory function works
44 myversion = "V2.1 July 2020" 45 myversion = "V2.1 July 2020"
45 verbose = True 46 verbose = True
46 debug = True 47 debug = True
47 toolFactoryURL = "https://github.com/fubar2/toolfactory" 48 toolFactoryURL = "https://github.com/fubar2/toolfactory"
48 ourdelim = "~~~" 49 ourdelim = "~~~"
50 ALOT = 10000000 # srsly. command or test overrides use read() so just in case
51 STDIOXML = """<stdio>
52 <exit_code range="100:" level="debug" description="shite happens" />
53 </stdio>"""
49 54
50 # --input_files="$input_files~~~$CL~~~$input_formats~~~$input_label 55 # --input_files="$input_files~~~$CL~~~$input_formats~~~$input_label
51 # ~~~$input_help" 56 # ~~~$input_help"
52 IPATHPOS = 0 57 IPATHPOS = 0
53 ICLPOS = 1 58 ICLPOS = 1
54 IFMTPOS = 2 59 IFMTPOS = 2
55 ILABPOS = 3 60 ILABPOS = 3
56 IHELPOS = 4 61 IHELPOS = 4
57 IOCLPOS = 5 62 IOCLPOS = 5
58 63
59 # --output_files "$otab.history_name~~~$otab.history_format~~~$otab.CL 64 # --output_files "$otab.history_name~~~$otab.history_format~~~$otab.CL~~~otab.history_test
60 ONAMEPOS = 0 65 ONAMEPOS = 0
61 OFMTPOS = 1 66 OFMTPOS = 1
62 OCLPOS = 2 67 OCLPOS = 2
63 OOCLPOS = 3 68 OTESTPOS = 3
69 OOCLPOS = 4
70
64 71
65 # --additional_parameters="$i.param_name~~~$i.param_value~~~ 72 # --additional_parameters="$i.param_name~~~$i.param_value~~~
66 # $i.param_label~~~$i.param_help~~~$i.param_type~~~$i.CL~~~i$.param_CLoverride" 73 # $i.param_label~~~$i.param_help~~~$i.param_type~~~$i.CL~~~i$.param_CLoverride"
67 ANAMEPOS = 0 74 ANAMEPOS = 0
68 AVALPOS = 1 75 AVALPOS = 1
74 AOCLPOS = 7 81 AOCLPOS = 7
75 82
76 83
77 foo = len(lxml.__version__) 84 foo = len(lxml.__version__)
78 # fug you, flake8. Say my name! 85 # fug you, flake8. Say my name!
86 FAKEEXE = "~~~REMOVE~~~ME~~~"
87 # need this until a PR/version bump to fix galaxyxml prepending the exe even
88 # with override.
89
79 90
80 def timenow(): 91 def timenow():
81 """return current time as a string 92 """return current time as a string
82 """ 93 """
83 return time.strftime("%d/%m/%Y %H:%M:%S", time.localtime(time.time())) 94 return time.strftime("%d/%m/%Y %H:%M:%S", time.localtime(time.time()))
118 citation_tuples = [] 129 citation_tuples = []
119 for citation in citations: 130 for citation in citations:
120 if citation.startswith("doi"): 131 if citation.startswith("doi"):
121 citation_tuples.append(("doi", citation[len("doi") :].strip())) 132 citation_tuples.append(("doi", citation[len("doi") :].strip()))
122 else: 133 else:
123 citation_tuples.append( 134 citation_tuples.append(("bibtex", citation[len("bibtex") :].strip()))
124 ("bibtex", citation[len("bibtex") :].strip())
125 )
126 return citation_tuples 135 return citation_tuples
127 136
128 137
129 class ScriptRunner: 138 class ScriptRunner:
130 """Wrapper for an arbitrary script 139 """Wrapper for an arbitrary script
135 def __init__(self, args=None): 144 def __init__(self, args=None):
136 """ 145 """
137 prepare command line cl for running the tool here 146 prepare command line cl for running the tool here
138 and prepare elements needed for galaxyxml tool generation 147 and prepare elements needed for galaxyxml tool generation
139 """ 148 """
140
141 self.infiles = [x.split(ourdelim) for x in args.input_files] 149 self.infiles = [x.split(ourdelim) for x in args.input_files]
142 self.outfiles = [x.split(ourdelim) for x in args.output_files] 150 self.outfiles = [x.split(ourdelim) for x in args.output_files]
143 self.addpar = [x.split(ourdelim) for x in args.additional_parameters] 151 self.addpar = [x.split(ourdelim) for x in args.additional_parameters]
144 self.args = args 152 self.args = args
145 self.cleanuppar() 153 self.cleanuppar()
146 self.lastclredirect = None 154 self.lastclredirect = None
147 self.lastxclredirect = None 155 self.lastxclredirect = None
148 self.cl = [] 156 self.cl = []
149 self.xmlcl = [] 157 self.xmlcl = []
150 self.is_positional = self.args.parampass == "positional" 158 self.is_positional = self.args.parampass == "positional"
159 if self.args.packages:
160 self.executeme = self.args.packages.split(",")[0].split(":")[0]
161 else:
162 self.executeme = self.args.sysexe
163 assert (
164 self.executeme is not None
165 ), "No system or managed executable passed in. Cannot build"
151 aCL = self.cl.append 166 aCL = self.cl.append
167 aXCL = self.xmlcl.append
152 assert args.parampass in [ 168 assert args.parampass in [
153 "0", 169 "0",
154 "argparse", 170 "argparse",
155 "positional", 171 "positional",
156 ], 'Parameter passing in args.parampass must be "0","positional" or "argparse"' 172 ], 'args.parampass must be "0","positional" or "argparse"'
157 self.tool_name = re.sub("[^a-zA-Z0-9_]+", "", args.tool_name) 173 self.tool_name = re.sub("[^a-zA-Z0-9_]+", "", args.tool_name)
158 self.tool_id = self.tool_name 174 self.tool_id = self.tool_name
159 if self.args.interpreter_name:
160 exe = "$runMe"
161 else:
162 exe = self.args.exe_package
163 assert (
164 exe is not None
165 ), "No interpeter or executable passed in - nothing to run so cannot build"
166 self.tool = gxt.Tool( 175 self.tool = gxt.Tool(
167 self.args.tool_name, 176 self.args.tool_name,
168 self.tool_id, 177 self.tool_id,
169 self.args.tool_version, 178 self.args.tool_version,
170 self.args.tool_desc, 179 self.args.tool_desc,
171 exe, 180 FAKEEXE,
172 ) 181 )
182 if self.args.script_path:
183 self.tool.interpreter = self.executeme
173 self.tooloutdir = "tfout" 184 self.tooloutdir = "tfout"
174 self.repdir = "TF_run_report_tempdir" 185 self.repdir = "TF_run_report_tempdir"
175 self.testdir = os.path.join(self.tooloutdir, "test-data") 186 self.testdir = os.path.join(self.tooloutdir, "test-data")
176 if not os.path.exists(self.tooloutdir): 187 if not os.path.exists(self.tooloutdir):
177 os.mkdir(self.tooloutdir) 188 os.mkdir(self.tooloutdir)
180 if not os.path.exists(self.repdir): 191 if not os.path.exists(self.repdir):
181 os.mkdir(self.repdir) 192 os.mkdir(self.repdir)
182 self.tinputs = gxtp.Inputs() 193 self.tinputs = gxtp.Inputs()
183 self.toutputs = gxtp.Outputs() 194 self.toutputs = gxtp.Outputs()
184 self.testparam = [] 195 self.testparam = []
185 if ( 196 if self.args.script_path:
186 self.args.runmode == "Executable" or self.args.runmode == "system" 197 self.prepScript()
187 ): 198 if self.args.command_override:
188 if len(self.args.cl_override) > 0: 199 scos = open(self.args.command_override, "r").readlines()
189 for x in self.args.cl_override.split(' '): 200 self.command_override = [x.rstrip() for x in scos]
190 aCL(x) 201 else:
202 self.command_override = None
203 if self.args.test_override:
204 stos = open(self.args.test_override, "r").readlines()
205 self.test_override = [x.rstrip() for x in stos]
206 else:
207 self.test_override = None
208 if self.args.cl_prefix: # DIY CL start
209 clp = self.args.cl_prefix.split(" ")
210 for c in clp:
211 aCL(c)
212 aXCL(c)
213 else:
214 if self.args.runmode == "Executable":
215 if self.args.script_path:
216 aCL(self.executeme)
217 aCL(self.sfile)
218 aXCL("$runme")
219 else:
220 aCL(self.executeme) # this little CL will just run
221 aXCL(self.executeme)
191 else: 222 else:
192 aCL(self.args.exe_package) # this little CL will just run 223 if self.args.script_path:
193 else: 224 aCL(self.executeme)
194 self.prepScript() 225 aCL(self.sfile)
195 aCL(self.args.interpreter_name) 226 aXCL("$runme")
196 aCL(self.sfile) 227 else:
197 228 aCL(self.executeme) # this little CL will just run
198 self.elog = "%s_error_log.txt" % self.tool_name 229 aXCL(self.executeme)
199 self.tlog = "%s_runner_log.txt" % self.tool_name 230 self.elog = os.path.join(self.repdir,"%s_error_log.txt" % self.tool_name)
231 self.tlog = os.path.join(self.repdir,"%s_runner_log.txt" % self.tool_name)
200 232
201 if self.args.parampass == "0": 233 if self.args.parampass == "0":
202 self.clsimple() 234 self.clsimple()
203 else: 235 else:
204 clsuffix = [] 236 clsuffix = []
220 else: 252 else:
221 appendme = [p[IOCLPOS], p[ICLPOS], p[IPATHPOS], ""] 253 appendme = [p[IOCLPOS], p[ICLPOS], p[IPATHPOS], ""]
222 xappendme = [p[IOCLPOS], p[ICLPOS], "$%s" % p[ICLPOS], ""] 254 xappendme = [p[IOCLPOS], p[ICLPOS], "$%s" % p[ICLPOS], ""]
223 clsuffix.append(appendme) 255 clsuffix.append(appendme)
224 xclsuffix.append(xappendme) 256 xclsuffix.append(xappendme)
225 # print('##infile i=%d, appendme=%s' % (i,appendme))
226 for i, p in enumerate(self.outfiles): 257 for i, p in enumerate(self.outfiles):
227 if p[OOCLPOS] == "STDOUT": 258 if p[OOCLPOS] == "STDOUT":
228 self.lastclredirect = [">", p[ONAMEPOS]] 259 self.lastclredirect = [">", p[ONAMEPOS]]
229 self.lastxclredirect = [">", "$%s" % p[OCLPOS]] 260 self.lastxclredirect = [">", "$%s" % p[OCLPOS]]
230 else: 261 else:
231 clsuffix.append([p[OOCLPOS], p[OCLPOS], p[ONAMEPOS], ""]) 262 clsuffix.append([p[OOCLPOS], p[OCLPOS], p[ONAMEPOS], ""])
232 xclsuffix.append( 263 xclsuffix.append([p[OOCLPOS], p[OCLPOS], "$%s" % p[ONAMEPOS], ""])
233 [p[OOCLPOS], p[OCLPOS], "$%s" % p[ONAMEPOS], ""]
234 )
235 for p in self.addpar: 264 for p in self.addpar:
236 clsuffix.append( 265 clsuffix.append([p[AOCLPOS], p[ACLPOS], p[AVALPOS], p[AOVERPOS]])
237 [p[AOCLPOS], p[ACLPOS], p[AVALPOS], p[AOVERPOS]]
238 )
239 xclsuffix.append( 266 xclsuffix.append(
240 [p[AOCLPOS], p[ACLPOS], '"$%s"' % p[ANAMEPOS], p[AOVERPOS]] 267 [p[AOCLPOS], p[ACLPOS], '"$%s"' % p[ANAMEPOS], p[AOVERPOS]]
241 ) 268 )
242 clsuffix.sort() 269 clsuffix.sort()
243 xclsuffix.sort() 270 xclsuffix.sort()
253 rx = [x.rstrip() for x in rx] 280 rx = [x.rstrip() for x in rx]
254 rxcheck = [x.strip() for x in rx if x.strip() > ""] 281 rxcheck = [x.strip() for x in rx if x.strip() > ""]
255 assert len(rxcheck) > 0, "Supplied script is empty. Cannot run" 282 assert len(rxcheck) > 0, "Supplied script is empty. Cannot run"
256 self.script = "\n".join(rx) 283 self.script = "\n".join(rx)
257 fhandle, self.sfile = tempfile.mkstemp( 284 fhandle, self.sfile = tempfile.mkstemp(
258 prefix=self.tool_name, suffix="_%s" % (self.args.interpreter_name) 285 prefix=self.tool_name, suffix="_%s" % (self.executeme)
259 ) 286 )
260 tscript = open(self.sfile, "w") 287 tscript = open(self.sfile, "w")
261 tscript.write(self.script) 288 tscript.write(self.script)
262 tscript.close() 289 tscript.close()
263 self.indentedScript = " %s" % "\n".join( 290 self.indentedScript = " %s" % "\n".join([" %s" % html_escape(x) for x in rx])
264 [" %s" % html_escape(x) for x in rx] 291 self.escapedScript = "%s" % "\n".join([" %s" % html_escape(x) for x in rx])
265 ) 292 art = "%s.%s" % (self.tool_name, self.executeme)
266 self.escapedScript = "%s" % "\n".join(
267 [" %s" % html_escape(x) for x in rx]
268 )
269 art = "%s.%s" % (self.tool_name, self.args.interpreter_name)
270 artifact = open(art, "wb") 293 artifact = open(art, "wb")
271 if self.args.interpreter_name == "python":
272 artifact.write(bytes("#!/usr/bin/env python\n", "utf8"))
273 artifact.write(bytes(self.script, "utf8")) 294 artifact.write(bytes(self.script, "utf8"))
274 artifact.close() 295 artifact.close()
275 296
276 def cleanuppar(self): 297 def cleanuppar(self):
277 """ positional parameters are complicated by their numeric ordinal""" 298 """ positional parameters are complicated by their numeric ordinal"""
375 return ndash 396 return ndash
376 397
377 def doXMLparam(self): 398 def doXMLparam(self):
378 """flake8 made me do this...""" 399 """flake8 made me do this..."""
379 for p in self.outfiles: 400 for p in self.outfiles:
380 newname, newfmt, newcl, oldcl = p 401 newname, newfmt, newcl, test, oldcl = p
381 ndash = self.getNdash(newcl) 402 ndash = self.getNdash(newcl)
382 aparm = gxtp.OutputData(newcl, format=newfmt, num_dashes=ndash) 403 aparm = gxtp.OutputData(newcl, format=newfmt, num_dashes=ndash)
383 aparm.positional = self.is_positional 404 aparm.positional = self.is_positional
384 if self.is_positional: 405 if self.is_positional:
385 if oldcl == "STDOUT": 406 if oldcl == "STDOUT":
387 aparm.command_line_override = "> $%s" % newcl 408 aparm.command_line_override = "> $%s" % newcl
388 else: 409 else:
389 aparm.positional = int(oldcl) 410 aparm.positional = int(oldcl)
390 aparm.command_line_override = "$%s" % newcl 411 aparm.command_line_override = "$%s" % newcl
391 self.toutputs.append(aparm) 412 self.toutputs.append(aparm)
392 tp = gxtp.TestOutput( 413 usetest = None
393 name=newcl, value="%s_sample" % newcl, format=newfmt 414 ld = None
394 ) 415 if test > '':
416 if test.startswith('diff'):
417 usetest = 'diff'
418 if test.split(':')[1].isdigit:
419 ld = int(test.split(':')[1])
420 else:
421 usetest = test
422 tp = gxtp.TestOutput(name=newcl, value="%s_sample" % newcl, format=newfmt,
423 compare=usetest, lines_diff=ld, delta=None,)
395 self.testparam.append(tp) 424 self.testparam.append(tp)
396 for p in self.infiles: 425 for p in self.infiles:
397 newname = p[ICLPOS] 426 newname = p[ICLPOS]
398 newfmt = p[IFMTPOS] 427 newfmt = p[IFMTPOS]
399 ndash = self.getNdash(newname) 428 ndash = self.getNdash(newname)
455 self.tinputs.append(aparm) 484 self.tinputs.append(aparm)
456 self.tparm = gxtp.TestParam(newname, value=newval) 485 self.tparm = gxtp.TestParam(newname, value=newval)
457 self.testparam.append(tparm) 486 self.testparam.append(tparm)
458 487
459 def doNoXMLparam(self): 488 def doNoXMLparam(self):
489 """filter style package - stdin to stdout"""
460 alab = self.infiles[0][ILABPOS] 490 alab = self.infiles[0][ILABPOS]
461 if len(alab) == 0: 491 if len(alab) == 0:
462 alab = self.infiles[0][ICLPOS] 492 alab = self.infiles[0][ICLPOS]
463 max1s = ( 493 max1s = (
464 "Maximum one input if parampass is 0 - more than one input files supplied - %s" 494 "Maximum one input if parampass is 0 but multiple input files supplied - %s"
465 % str(self.infiles) 495 % str(self.infiles)
466 ) 496 )
467 assert len(self.infiles) == 1, max1s 497 assert len(self.infiles) == 1, max1s
468 newname = self.infiles[0][ICLPOS] 498 newname = self.infiles[0][ICLPOS]
469 aninput = gxtp.DataParam( 499 aninput = gxtp.DataParam(
484 newfmt = self.outfiles[0][OFMTPOS] 514 newfmt = self.outfiles[0][OFMTPOS]
485 anout = gxtp.OutputData(newname, format=newfmt, num_dashes=0) 515 anout = gxtp.OutputData(newname, format=newfmt, num_dashes=0)
486 anout.command_line_override = "> $%s" % newname 516 anout.command_line_override = "> $%s" % newname
487 anout.positional = self.is_positional 517 anout.positional = self.is_positional
488 self.toutputs.append(anout) 518 self.toutputs.append(anout)
489 tp = gxtp.TestOutput( 519 tp = gxtp.TestOutput(name=newname, value="%s_sample" % newname, format=newfmt)
490 name=newname, value="%s_sample" % newname, format=newfmt
491 )
492 self.testparam.append(tp) 520 self.testparam.append(tp)
493 521
494 def makeXML(self): 522 def makeXML(self):
495 """ 523 """
496 Create a Galaxy xml tool wrapper for the new script 524 Create a Galaxy xml tool wrapper for the new script
497 Uses galaxyhtml 525 Uses galaxyhtml
498 Hmmm. How to get the command line into correct order... 526 Hmmm. How to get the command line into correct order...
499 """ 527 """
500 if self.args.cl_override: 528 if self.command_override:
501 self.tool.command_line_override = self.args.cl_override.split(' ') + self.xmlcl 529 self.tool.command_line_override = self.command_override # config file
502 else: 530 else:
503 self.tool.command_line_override = self.xmlcl 531 self.tool.command_line_override = self.xmlcl
504 if self.args.interpreter_name: 532 # if self.args.interpreter_name:
505 self.tool.interpreter = self.args.interpreter_name 533 # self.tool.interpreter = self.args.interpreter_name
506 if self.args.help_text: 534 if self.args.help_text:
507 helptext = open(self.args.help_text, "r").readlines() 535 helptext = open(self.args.help_text, "r").readlines()
508 helptext = [html_escape(x) for x in helptext] 536 helptext = [html_escape(x) for x in helptext]
509 self.tool.help = "".join([x for x in helptext]) 537 self.tool.help = "".join([x for x in helptext])
510 else: 538 else:
513 as none was supplied at tool generation\n" 541 as none was supplied at tool generation\n"
514 % (self.args.user_email) 542 % (self.args.user_email)
515 ) 543 )
516 self.tool.version_command = None # do not want 544 self.tool.version_command = None # do not want
517 requirements = gxtp.Requirements() 545 requirements = gxtp.Requirements()
518 546 if self.args.packages:
519 if self.args.interpreter_name: 547 for d in self.args.packages.split(","):
520 if self.args.dependencies: 548 if ":" in d:
521 for d in self.args.dependencies.split(','): 549 packg, ver = d.split(":")
522 requirements.append( 550 else:
523 gxtp.Requirement( 551 packg = d
524 "package", d, "" 552 ver = ""
525 ) 553 requirements.append(gxtp.Requirement("package", packg.strip(), ver.strip()))
526 )
527 if self.args.interpreter_name == "python":
528 requirements.append(
529 gxtp.Requirement(
530 "package", "python", self.args.interpreter_version
531 )
532 )
533 elif self.args.interpreter_name not in ["bash", "sh"]:
534 requirements.append(
535 gxtp.Requirement(
536 "package",
537 self.args.interpreter_name,
538 self.args.interpreter_version,
539 )
540 )
541 else:
542 if self.args.exe_package and self.args.parampass != "system":
543 requirements.append(
544 gxtp.Requirement(
545 "package",
546 self.args.exe_package,
547 self.args.exe_package_version,
548 )
549 )
550 self.tool.requirements = requirements 554 self.tool.requirements = requirements
551 if self.args.parampass == "0": 555 if self.args.parampass == "0":
552 self.doNoXMLparam() 556 self.doNoXMLparam()
553 else: 557 else:
554 self.doXMLparam() 558 self.doXMLparam()
555 self.tool.outputs = self.toutputs 559 self.tool.outputs = self.toutputs
556 self.tool.inputs = self.tinputs 560 self.tool.inputs = self.tinputs
557 if self.args.runmode not in ["Executable", "system"]: 561 if (
562 self.args.script_path
563 ):
558 configfiles = gxtp.Configfiles() 564 configfiles = gxtp.Configfiles()
559 configfiles.append(gxtp.Configfile(name="runMe", text=self.script)) 565 configfiles.append(gxtp.Configfile(name="runme", text=self.script))
560 self.tool.configfiles = configfiles 566 self.tool.configfiles = configfiles
561 tests = gxtp.Tests() 567 tests = gxtp.Tests()
562 test_a = gxtp.Test() 568 test_a = gxtp.Test()
563 for tp in self.testparam: 569 for tp in self.testparam:
564 test_a.append(tp) 570 test_a.append(tp)
571 self.tool.add_comment("Source in git at: %s" % (toolFactoryURL)) 577 self.tool.add_comment("Source in git at: %s" % (toolFactoryURL))
572 self.tool.add_comment( 578 self.tool.add_comment(
573 "Cite: Creating re-usable tools from scripts doi: \ 579 "Cite: Creating re-usable tools from scripts doi: \
574 10.1093/bioinformatics/bts573" 580 10.1093/bioinformatics/bts573"
575 ) 581 )
576 exml = self.tool.export() 582 exml0 = self.tool.export()
577 xf = open('%s.xml' % self.tool_name, "w") 583 exml = exml0.replace(FAKEEXE, "") # temporary work around until PR accepted
584 if self.test_override: # cannot do this inside galaxyxml as it expects lxml objects for tests
585 part1 = exml.split('<tests>')[0]
586 part2 = exml.split('</tests>')[1]
587 fixed = '%s\n%s\n%s' % (part1,self.test_override,part2)
588 exml = fixed
589 xf = open("%s.xml" % self.tool_name, "w")
578 xf.write(exml) 590 xf.write(exml)
579 xf.write("\n") 591 xf.write("\n")
580 xf.close() 592 xf.close()
581 # ready for the tarball 593 # ready for the tarball
582 594
583 def makeTooltar(self): 595 def makeTool(self):
584 """ 596 """write xmls and samples into place
585 a tool is a gz tarball with eg 597 """
586 /toolname/tool.xml /toolname/tool.py /toolname/test-data/test1_in.foo ...
587 NOTE names for test inputs and outputs are munged here so must
588 correspond to actual input and output names used on the generated cl
589 """
590 retval = self.run()
591 if retval:
592 sys.stderr.write(
593 "## Run failed. Cannot build yet. Please fix and retry"
594 )
595 sys.exit(1)
596 self.makeXML() 598 self.makeXML()
599 if self.args.script_path:
600 stname = os.path.join(self.tooloutdir, "%s" % (self.sfile))
601 if not os.path.exists(stname):
602 shutil.copyfile(self.sfile, stname)
603 xreal = "%s.xml" % self.tool_name
604 xout = os.path.join(self.tooloutdir, xreal)
605 shutil.copyfile(xreal, xout)
597 for p in self.infiles: 606 for p in self.infiles:
598 pth = p[IPATHPOS] 607 pth = p[IPATHPOS]
599 dest = os.path.join(self.testdir, "%s_sample" % p[ICLPOS]) 608 dest = os.path.join(self.testdir, "%s_sample" % p[ICLPOS])
600 shutil.copyfile(pth, dest) 609 shutil.copyfile(pth, dest)
601 for p in self.outfiles: 610
602 pth = p[OCLPOS] 611 def makeToolTar(self):
603 if p[OOCLPOS] == "STDOUT" or self.args.parampass == "0":
604 pth = p[ONAMEPOS]
605 dest = os.path.join(self.testdir, "%s_sample" % p[ONAMEPOS])
606 shutil.copyfile(pth, dest)
607 dest = os.path.join(self.tooloutdir, p[ONAMEPOS])
608 shutil.copyfile(pth, dest)
609 else:
610 pth = p[OCLPOS]
611 dest = os.path.join(self.testdir, "%s_sample" % p[OCLPOS])
612 shutil.copyfile(pth, dest)
613 dest = os.path.join(self.tooloutdir, p[OCLPOS])
614 shutil.copyfile(pth, dest)
615 if os.path.exists(self.tlog) and os.stat(self.tlog).st_size > 0:
616 shutil.copyfile(self.tlog, os.path.join(self.tooloutdir, "test1_log_outfiletxt"))
617 if self.args.runmode not in ["Executable", "system"]:
618 stname = os.path.join(self.tooloutdir, "%s" % (self.sfile))
619 if not os.path.exists(stname):
620 shutil.copyfile(self.sfile, stname)
621 xreal = '%s.xml' % self.tool_name
622 xout = os.path.join(self.tooloutdir,xreal)
623 shutil.copyfile(xreal, xout)
624 self.newtarpath = "toolfactory_%s.tgz" % self.tool_name 612 self.newtarpath = "toolfactory_%s.tgz" % self.tool_name
625 tf = tarfile.open(self.newtarpath, "w:gz") 613 tf = tarfile.open(self.newtarpath, "w:gz")
626 tf.add(name=self.tooloutdir, arcname=self.tool_name) 614 tf.add(name=self.tooloutdir, arcname=self.tool_name)
627 tf.close() 615 tf.close()
628 shutil.copyfile(self.newtarpath, self.args.new_tool) 616 shutil.copyfile(self.newtarpath, self.args.new_tool)
629 shutil.copyfile(xreal,"tool_xml.txt") 617 if os.path.exists(self.tlog) and os.stat(self.tlog).st_size > 0:
630 repoutnames = [x[OCLPOS] for x in self.outfiles] 618 shutil.copyfile(
631 with os.scandir('.') as outs: 619 self.tlog, os.path.join(self.tooloutdir, "test1_log_outfiletxt")
620 )
621
622 def moveRunOutputs(self):
623 """need to move files into toolfactory collection after any run - planemo or not
624 """
625 for p in self.outfiles:
626 naym = p[ONAMEPOS]
627 src = os.path.join(self.tooloutdir,naym)
628 if os.path.isfile(src):
629 dest = os.path.join(self.testdir, "%s_sample" % naym)
630 shutil.copyfile(naym, dest)
631 else:
632 print('### problem - output file %s not found in tooloutdir %s' % (src,self.tooloutdir))
633 with os.scandir(self.tooloutdir) as outs:
632 for entry in outs: 634 for entry in outs:
633 if entry.name.endswith('.tgz') or not entry.is_file(): 635 if not entry.is_file() or entry.name.startswith('.'):
634 continue 636 continue
635 if entry.name in repoutnames: 637 if "." in entry.name:
636 if '.' in entry.name: 638 nayme,ext = os.path.splitext(entry.name)
637 ofne = os.path.splitext(entry.name)[1] 639 else:
638 else: 640 ext = ".txt"
639 ofne = '.txt' 641 ofn = "%s%s" % (entry.name.replace(".", "_"), ext)
640 ofn = '%s%s' % (entry.name.replace('.','_'),ofne) 642 dest = os.path.join(self.repdir, ofn)
641 shutil.copyfile(entry.name,os.path.join(self.repdir,ofn)) 643 src = os.path.join(self.tooloutdir,entry.name)
642 elif entry.name == "%s.xml" % self.tool_name: 644 shutil.copyfile(src, dest)
643 shutil.copyfile(entry.name,os.path.join(self.repdir,"new_tool_xml.xml"))
644 return retval
645 645
646 def run(self): 646 def run(self):
647 """ 647 """
648 648
649 """ 649 """
653 scl = " ".join(self.cl) 653 scl = " ".join(self.cl)
654 err = None 654 err = None
655 if self.args.parampass != "0": 655 if self.args.parampass != "0":
656 ste = open(self.elog, "wb") 656 ste = open(self.elog, "wb")
657 if self.lastclredirect: 657 if self.lastclredirect:
658 sto = open( 658 sto = open(self.lastclredirect[1], "wb") # is name of an output file
659 self.lastclredirect[1], "wb"
660 ) # is name of an output file
661 else: 659 else:
662 sto = open(self.tlog, "wb") 660 sto = open(self.tlog, "wb")
663 sto.write( 661 sto.write(
664 bytes( 662 bytes(
665 "## Executing Toolfactory generated command line = %s\n" 663 "## Executing Toolfactory generated command line = %s\n" % scl,
666 % scl,
667 "utf8", 664 "utf8",
668 ) 665 )
669 ) 666 )
670 sto.flush() 667 sto.flush()
671 p = subprocess.run(self.cl, shell=False, stdout=sto, stderr=ste) 668 p = subprocess.run(self.cl, shell=False, stdout=sto, stderr=ste)
681 break 678 break
682 except OverflowError: 679 except OverflowError:
683 pass 680 pass
684 tmp_stderr.close() 681 tmp_stderr.close()
685 retval = p.returncode 682 retval = p.returncode
686 else: # work around special case of simple scripts that take stdin and write to stdout 683 else: # work around special case - stdin and write to stdout
687 sti = open(self.infiles[0][IPATHPOS], "rb") 684 sti = open(self.infiles[0][IPATHPOS], "rb")
688 sto = open(self.outfiles[0][ONAMEPOS], "wb") 685 sto = open(self.outfiles[0][ONAMEPOS], "wb")
689 # must use shell to redirect 686 # must use shell to redirect
690 p = subprocess.run(self.cl, shell=False, stdout=sto, stdin=sti) 687 p = subprocess.run(self.cl, shell=False, stdout=sto, stdin=sti)
691 retval = p.returncode 688 retval = p.returncode
698 if retval != 0 and err: # problem 695 if retval != 0 and err: # problem
699 sys.stderr.write(err) 696 sys.stderr.write(err)
700 logging.debug("run done") 697 logging.debug("run done")
701 return retval 698 return retval
702 699
703 def install_load(self):
704 testres = self.planemo_test()
705 if testres == 0:
706 self.planemo_shedload()
707 self,eph_galaxy_load()
708 else:
709 stderr.write('Planemo test failed - tool %s was not installed' % self.args.tool_name)
710
711
712 def planemo_shedload(self): 700 def planemo_shedload(self):
713 """ 701 """
714 planemo shed_create --shed_target testtoolshed 702 planemo shed_create --shed_target testtoolshed
715 planemo shed_update --check_diff --shed_target testtoolshed 703 planemo shed_update --check_diff --shed_target testtoolshed
716 """ 704 """
717 cll = ['planemo', 'shed_create', '--shed_target','local'] 705 if os.path.exists(self.tlog):
718 p = subprocess.run(cll, shell=False,cwd=self.tooloutdir ) 706 tout = open(self.tlog,'a')
707 else:
708 tout = open(self.tlog,'w')
709 cll = ["planemo", "shed_create", "--shed_target", "local"]
710 try:
711 p = subprocess.run(cll, shell=False, cwd=self.tooloutdir, stdout=tout, stderr = tout)
712 except:
713 pass
719 if p.returncode != 0: 714 if p.returncode != 0:
720 print('Repository %s exists' % self.args.tool_name) 715 print("Repository %s exists" % self.args.tool_name)
721 else: 716 else:
722 print('initiated %s' % self.args.tool_name) 717 print("initiated %s" % self.args.tool_name)
723 cll = ['planemo', 'shed_upload', '--shed_target','local','-r','--owner','fubar', 718 cll = [
724 '--name',self.args.tool_name,'--shed_key',self.args.toolshed_api_key,'--tar',self.newtarpath] 719 "planemo",
725 p = subprocess.run(cll, shell=False,cwd=self.tooloutdir) 720 "shed_upload",
726 print('Ran',' '.join(cll),'got',p.returncode) 721 "--shed_target",
722 "local",
723 "--owner",
724 "fubar",
725 "--name",
726 self.args.tool_name,
727 "--shed_key",
728 self.args.toolshed_api_key,
729 "--tar",
730 self.newtarpath,
731 ]
732 print("Run", " ".join(cll))
733 p = subprocess.run(cll, shell=False)
734 print("Ran", " ".join(cll), "got", p.returncode)
735 tout.close()
727 return p.returncode 736 return p.returncode
728 737
729 738 def planemo_test(self, genoutputs=True):
730 def planemo_test(self):
731 """planemo is a requirement so is available 739 """planemo is a requirement so is available
732 """ 740 """
733 cll = ['planemo', 'test', '--galaxy_root', self.args.galaxy_root, 741 xreal = "%s.xml" % self.tool_name
734 self.args.tool_name] 742 if os.path.exists(self.tlog):
735 p = subprocess.run(cll, shell=False) 743 tout = open(self.tlog,'a')
736 ols = os.listdir('.') 744 else:
737 for fn in ols: 745 tout = open(self.tlog,'w')
738 if '.' in fn: 746 if genoutputs:
739 ofne = os.path.splitext(fn)[1] 747 cll = [
740 else: 748 "planemo",
741 ofne = '.txt' 749 "test",
742 ofn = '%s%s' % (fn.replace('.','_'),ofne) 750 "--galaxy_root",
743 shutil.copyfile(fn,os.path.join(self.repdir,ofn)) 751 self.args.galaxy_root,
752 "--update_test_data",
753 xreal,
754 ]
755 else:
756 cll = ["planemo", "test", "--galaxy_root", self.args.galaxy_root, xreal]
757 try:
758 p = subprocess.run(cll, shell=False, cwd=self.tooloutdir, stderr=tout, stdout=tout)
759 except:
760 pass
761 tout.close()
744 return p.returncode 762 return p.returncode
745 763
746
747 def eph_galaxy_load(self): 764 def eph_galaxy_load(self):
748 """ 765 """
749 """ 766 """
750 cll = ['shed-tools', 'install', '-g', self.args.galaxy_url, '--latest', 767 if os.path.exists(self.tlog):
751 '-a', self.args.galaxy_api_key , '--name', self.args.tool_name, '--owner','fubar', 768 tout = open(self.tlog,'a')
752 '--toolshed', self.args.toolshed_url, 769 else:
753 '--section_label','Generated Tools','--install_tool_dependencies',] 770 tout = open(self.tlog,'w')
754 print('running\n',' '.join(cll)) 771 cll = [
755 p = subprocess.run(cll, shell=False) 772 "shed-tools",
773 "install",
774 "-g",
775 self.args.galaxy_url,
776 "--latest",
777 "-a",
778 self.args.galaxy_api_key,
779 "--name",
780 self.args.tool_name,
781 "--owner",
782 "fubar",
783 "--toolshed",
784 self.args.toolshed_url,
785 "--section_label",
786 "Generated Tools",
787 "--install_tool_dependencies",
788 ]
789 print("running\n", " ".join(cll))
790 p = subprocess.run(cll, shell=False, stderr=tout, stdout=tout)
756 if p.returncode != 0: 791 if p.returncode != 0:
757 print('Repository %s installation returned %d' % (self.args.tool_name,p.retcode)) 792 print(
758 else: 793 "Repository %s installation returned %d"
759 print('installed %s' % self.args.tool_name) 794 % (self.args.tool_name, p.returncode)
795 )
796 else:
797 print("installed %s" % self.args.tool_name)
798 tout.close()
760 return p.returncode 799 return p.returncode
761 800
762 def writeShedyml(self): 801 def writeShedyml(self):
763 yuser = self.args.user_email.split('@')[0] 802 yuser = self.args.user_email.split("@")[0]
764 yfname = os.path.join(self.tooloutdir,'.shed.yml') 803 yfname = os.path.join(self.tooloutdir, ".shed.yml")
765 yamlf = open(yfname, 'w') 804 yamlf = open(yfname, "w")
766 odict = {'name':self.tool_name,'owner':yuser,'type':'unrestricted','description':self.args.tool_desc} 805 odict = {
806 "name": self.tool_name,
807 "owner": yuser,
808 "type": "unrestricted",
809 "description": self.args.tool_desc,
810 }
767 yaml.dump(odict, yamlf, allow_unicode=True) 811 yaml.dump(odict, yamlf, allow_unicode=True)
768 yamlf.close() 812 yamlf.close()
813
814
815 def install_load(self):
816 _ = self.planemo_test(genoutputs=True)
817 testres = self.planemo_test(genoutputs=False)
818 if testres == 0:
819 if self.args.make_Tool == "install":
820 self.planemo_shedload()
821 self.eph_galaxy_load()
822 else:
823 os.stderr.write(
824 "Planemo test failed - tool %s was not installed" % self.args.tool_name
825 )
769 826
770 def main(): 827 def main():
771 """ 828 """
772 This is a Galaxy wrapper. It expects to be called by a special purpose tool.xml as: 829 This is a Galaxy wrapper. It expects to be called by a special purpose tool.xml as:
773 <command interpreter="python">rgBaseScriptWrapper.py --script_path "$scriptPath" --tool_name "foo" --interpreter "Rscript" 830 <command interpreter="python">rgBaseScriptWrapper.py --script_path "$scriptPath"
831 --tool_name "foo" --interpreter "Rscript"
774 </command> 832 </command>
775 """ 833 """
776 parser = argparse.ArgumentParser() 834 parser = argparse.ArgumentParser()
777 a = parser.add_argument 835 a = parser.add_argument
778 a("--script_path", default="") 836 a("--script_path", default=None)
779 a("--dependencies", default="") 837 a("--history_test", default=None)
780 a("--cl_override", default="") 838 a("--cl_prefix", default=None)
839 a("--sysexe", default=None)
840 a("--packages", default=None)
781 a("--tool_name", default=None) 841 a("--tool_name", default=None)
782 a("--interpreter_name", default=None)
783 a("--interpreter_version", default=None)
784 a("--exe_package", default=None)
785 a("--exe_package_version", default=None)
786 a("--input_files", default=[], action="append") 842 a("--input_files", default=[], action="append")
787 a("--output_files", default=[], action="append") 843 a("--output_files", default=[], action="append")
788 a("--user_email", default="Unknown") 844 a("--user_email", default="Unknown")
789 a("--bad_user", default=None) 845 a("--bad_user", default=None)
790 a("--make_Tool", default=None) 846 a("--make_Tool", default="runonly")
791 a("--help_text", default=None) 847 a("--help_text", default=None)
792 a("--tool_desc", default=None) 848 a("--tool_desc", default=None)
793 a("--tool_version", default=None) 849 a("--tool_version", default=None)
794 a("--citations", default=None) 850 a("--citations", default=None)
851 a("--command_override", default=None)
852 a("--test_override", default=None)
795 a("--additional_parameters", action="append", default=[]) 853 a("--additional_parameters", action="append", default=[])
796 a("--edit_additional_parameters", action="store_true", default=False) 854 a("--edit_additional_parameters", action="store_true", default=False)
797 a("--parampass", default="positional") 855 a("--parampass", default="positional")
798 a("--tfout", default="./tfout") 856 a("--tfout", default="./tfout")
799 a("--new_tool", default="new_tool") 857 a("--new_tool", default="new_tool")
800 a("--runmode", default=None) 858 a("--runmode", default=None)
801 a("--galaxy_url", default="http://localhost") 859 a("--galaxy_url", default="http://localhost:8080")
802 a("--galaxy_api_key", default='fakekey') 860 a("--galaxy_api_key", default="fbdd3c2eecd191e88939fffc02eeeaf8")
803 a("--toolshed_url", default="http://localhost:9009") 861 a("--toolshed_url", default="http://localhost:9009")
804 a("--toolshed_api_key", default=None) 862 a("--toolshed_api_key", default="d46e5ed0e242ed52c6e1f506b5d7f9f7")
805 a("--planemo_test", default="yes") 863 a("--galaxy_root", default="/home/ross/galaxy")
806 864
807 args = parser.parse_args() 865 args = parser.parse_args()
808 assert not args.bad_user, ( 866 assert not args.bad_user, (
809 'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy admin adds %s to "admin_users" in the Galaxy configuration file' 867 'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy admin adds %s to "admin_users" in the Galaxy configuration file'
810 % (args.bad_user, args.bad_user) 868 % (args.bad_user, args.bad_user)
811 ) 869 )
870 assert args.tool_name, "## Tool Factory expects a tool name - eg --tool_name=DESeq"
812 assert ( 871 assert (
813 args.tool_name 872 args.sysexe or args.packages
814 ), "## Tool Factory expects a tool name - eg --tool_name=DESeq"
815 assert (
816 args.interpreter_name or args.exe_package
817 ), "## Tool Factory wrapper expects an interpreter or an executable package" 873 ), "## Tool Factory wrapper expects an interpreter or an executable package"
818 assert args.exe_package or ( 874 args.input_files = [x.replace('"', "").replace("'", "") for x in args.input_files]
819 len(args.script_path) > 0 and os.path.isfile(args.script_path)
820 ), "## Tool Factory wrapper expects a script path - eg --script_path=foo.R if no executable"
821 args.input_files = [
822 x.replace('"', "").replace("'", "") for x in args.input_files
823 ]
824 # remove quotes we need to deal with spaces in CL params 875 # remove quotes we need to deal with spaces in CL params
825 for i, x in enumerate(args.additional_parameters): 876 for i, x in enumerate(args.additional_parameters):
826 args.additional_parameters[i] = args.additional_parameters[i].replace( 877 args.additional_parameters[i] = args.additional_parameters[i].replace('"', "")
827 '"', ""
828 )
829 r = ScriptRunner(args) 878 r = ScriptRunner(args)
830 if args.make_Tool: 879 r.writeShedyml()
831 r.writeShedyml() 880 r.makeTool()
832 retcode = r.makeTooltar() 881 if args.make_Tool == "runonly":
833 if retcode == 0: 882 retcode = r.run()
834 if args.planemo_test == "yes": 883 if retcode:
835 r.install_load() 884 sys.stderr.write(
885 "## Run failed with return code %d. Cannot build yet. Please fix and retry"
886 % retcode
887 )
888 sys.exit(1)
889 else:
890 r.moveRunOutputs()
891 elif args.make_Tool in ["gentestinstall", "generate", "gentest"]:
892 retcode = r.run()
893 if retcode:
894 sys.stderr.write(
895 "## Run failed with return code %d. Cannot build yet. Please fix and retry"
896 % retcode
897 )
898 sys.exit(1)
899 r.moveRunOutputs()
900 r.makeToolTar()
901 if args.make_Tool in ["gentestinstall","gentest"]:
902 r.planemo_test(genoutputs=False)
903 r.moveRunOutputs()
904 r.planemo_shedload()
905 r.eph_galaxy_load()
836 else: 906 else:
837 retcode = r.run() 907 retcode = r.planemo_test(genoutputs=True) # this fails :(
838 # if retcode: 908 r.moveRunOutputs()
909 r.makeToolTar()
910 retcode = r.planemo_test(genoutputs=False)
911 r.moveRunOutputs()
912 if args.make_Tool == "planemotestinstall":
913 r.planemo_shedload()
914 r.eph_galaxy_load()
915 # if retcode:
839 # sys.exit(retcode) # indicate failure to job runner 916 # sys.exit(retcode) # indicate failure to job runner
840 917
841 918
842 if __name__ == "__main__": 919 if __name__ == "__main__":
843 main() 920 main()