Mercurial > repos > fubar > tool_factory_2
comparison toolfactory/rgToolFactory2.py @ 43:e7e9732ebed6 draft
Uploaded
| author | fubar |
|---|---|
| date | Sun, 16 Aug 2020 08:51:14 -0400 |
| parents | |
| children | f8700bed1e17 |
comparison
equal
deleted
inserted
replaced
| 42:b938475235e3 | 43:e7e9732ebed6 |
|---|---|
| 1 #!/usr/bin/env python | |
| 2 # rgToolFactory.py | |
| 3 # see https://github.com/fubar2/toolfactory | |
| 4 # | |
| 5 # copyright ross lazarus (ross stop lazarus at gmail stop com) May 2012 | |
| 6 # | |
| 7 # all rights reserved | |
| 8 # Licensed under the LGPL | |
| 9 # suggestions for improvement and bug fixes welcome at https://github.com/fubar2/toolfactory | |
| 10 # | |
| 11 # July 2020: BCC was fun and I feel like rip van winkle after 5 years. | |
| 12 # Decided to | |
| 13 # 1. Fix the toolfactory so it works - done for simplest case | |
| 14 # 2. Fix planemo so the toolfactory function works | |
| 15 # 3. Rewrite bits using galaxyxml functions where that makes sense - done | |
| 16 # | |
| 17 # removed all the old complications including making the new tool use this same script | |
| 18 # galaxyxml now generates the tool xml https://github.com/hexylena/galaxyxml | |
| 19 # No support for automatic HTML file creation from arbitrary outputs | |
| 20 # essential problem is to create two command lines - one for the tool xml and a different | |
| 21 # one to run the executable with the supplied test data and settings | |
| 22 # Be simpler to write the tool, then run it with planemo and soak up the test outputs. | |
| 23 | |
| 24 | |
| 25 | |
| 26 import argparse | |
| 27 import logging | |
| 28 import os | |
| 29 import re | |
| 30 import shutil | |
| 31 import subprocess | |
| 32 import sys | |
| 33 import tarfile | |
| 34 import tempfile | |
| 35 import time | |
| 36 | |
| 37 import galaxyxml.tool as gxt | |
| 38 import galaxyxml.tool.parameters as gxtp | |
| 39 | |
| 40 import lxml | |
| 41 | |
| 42 myversion = "V2.1 July 2020" | |
| 43 verbose = True | |
| 44 debug = True | |
| 45 toolFactoryURL = "https://github.com/fubar2/toolfactory" | |
| 46 ourdelim = "~~~" | |
| 47 | |
| 48 # --input_files="$input_files~~~$CL~~~$input_formats~~~$input_label | |
| 49 # ~~~$input_help" | |
| 50 IPATHPOS = 0 | |
| 51 ICLPOS = 1 | |
| 52 IFMTPOS = 2 | |
| 53 ILABPOS = 3 | |
| 54 IHELPOS = 4 | |
| 55 IOCLPOS = 5 | |
| 56 | |
| 57 # --output_files "$otab.history_name~~~$otab.history_format~~~$otab.CL | |
| 58 ONAMEPOS = 0 | |
| 59 OFMTPOS = 1 | |
| 60 OCLPOS = 2 | |
| 61 OOCLPOS = 3 | |
| 62 | |
| 63 # --additional_parameters="$i.param_name~~~$i.param_value~~~ | |
| 64 # $i.param_label~~~$i.param_help~~~$i.param_type~~~$i.CL~~~i$.param_CLoverride" | |
| 65 ANAMEPOS = 0 | |
| 66 AVALPOS = 1 | |
| 67 ALABPOS = 2 | |
| 68 AHELPPOS = 3 | |
| 69 ATYPEPOS = 4 | |
| 70 ACLPOS = 5 | |
| 71 AOVERPOS = 6 | |
| 72 AOCLPOS = 7 | |
| 73 | |
| 74 | |
| 75 foo = len(lxml.__version__) | |
| 76 # fug you, flake8. Say my name! | |
| 77 | |
| 78 def timenow(): | |
| 79 """return current time as a string | |
| 80 """ | |
| 81 return time.strftime("%d/%m/%Y %H:%M:%S", time.localtime(time.time())) | |
| 82 | |
| 83 | |
| 84 def quote_non_numeric(s): | |
| 85 """return a prequoted string for non-numerics | |
| 86 useful for perl and Rscript parameter passing? | |
| 87 """ | |
| 88 try: | |
| 89 _ = float(s) | |
| 90 return s | |
| 91 except ValueError: | |
| 92 return '"%s"' % s | |
| 93 | |
| 94 | |
| 95 html_escape_table = {"&": "&", ">": ">", "<": "<", "$": r"\$"} | |
| 96 | |
| 97 | |
| 98 def html_escape(text): | |
| 99 """Produce entities within text.""" | |
| 100 return "".join(html_escape_table.get(c, c) for c in text) | |
| 101 | |
| 102 | |
| 103 def html_unescape(text): | |
| 104 """Revert entities within text. Multiple character targets so use replace""" | |
| 105 t = text.replace("&", "&") | |
| 106 t = t.replace(">", ">") | |
| 107 t = t.replace("<", "<") | |
| 108 t = t.replace("\\$", "$") | |
| 109 return t | |
| 110 | |
| 111 | |
| 112 def parse_citations(citations_text): | |
| 113 """ | |
| 114 """ | |
| 115 citations = [c for c in citations_text.split("**ENTRY**") if c.strip()] | |
| 116 citation_tuples = [] | |
| 117 for citation in citations: | |
| 118 if citation.startswith("doi"): | |
| 119 citation_tuples.append(("doi", citation[len("doi") :].strip())) | |
| 120 else: | |
| 121 citation_tuples.append( | |
| 122 ("bibtex", citation[len("bibtex") :].strip()) | |
| 123 ) | |
| 124 return citation_tuples | |
| 125 | |
| 126 | |
| 127 class ScriptRunner: | |
| 128 """Wrapper for an arbitrary script | |
| 129 uses galaxyxml | |
| 130 | |
| 131 """ | |
| 132 | |
| 133 def __init__(self, args=None): | |
| 134 """ | |
| 135 prepare command line cl for running the tool here | |
| 136 and prepare elements needed for galaxyxml tool generation | |
| 137 """ | |
| 138 | |
| 139 self.infiles = [x.split(ourdelim) for x in args.input_files] | |
| 140 self.outfiles = [x.split(ourdelim) for x in args.output_files] | |
| 141 self.addpar = [x.split(ourdelim) for x in args.additional_parameters] | |
| 142 self.args = args | |
| 143 self.cleanuppar() | |
| 144 self.lastclredirect = None | |
| 145 self.lastxclredirect = None | |
| 146 self.cl = [] | |
| 147 self.xmlcl = [] | |
| 148 self.is_positional = self.args.parampass == "positional" | |
| 149 aCL = self.cl.append | |
| 150 assert args.parampass in [ | |
| 151 "0", | |
| 152 "argparse", | |
| 153 "positional", | |
| 154 ], 'Parameter passing in args.parampass must be "0","positional" or "argparse"' | |
| 155 self.tool_name = re.sub("[^a-zA-Z0-9_]+", "", args.tool_name) | |
| 156 self.tool_id = self.tool_name | |
| 157 if self.args.interpreter_name: | |
| 158 exe = "$runMe" | |
| 159 else: | |
| 160 exe = self.args.exe_package | |
| 161 assert ( | |
| 162 exe is not None | |
| 163 ), "No interpeter or executable passed in - nothing to run so cannot build" | |
| 164 self.tool = gxt.Tool( | |
| 165 self.args.tool_name, | |
| 166 self.tool_id, | |
| 167 self.args.tool_version, | |
| 168 self.args.tool_desc, | |
| 169 exe, | |
| 170 ) | |
| 171 self.tinputs = gxtp.Inputs() | |
| 172 self.toutputs = gxtp.Outputs() | |
| 173 self.testparam = [] | |
| 174 if ( | |
| 175 self.args.runmode == "Executable" or self.args.runmode == "system" | |
| 176 ): # binary - no need | |
| 177 aCL(self.args.exe_package) # this little CL will just run | |
| 178 else: | |
| 179 self.prepScript() | |
| 180 self.elog = "%s_error_log.txt" % self.tool_name | |
| 181 self.tlog = "%s_runner_log.txt" % self.tool_name | |
| 182 | |
| 183 if self.args.parampass == "0": | |
| 184 self.clsimple() | |
| 185 else: | |
| 186 clsuffix = [] | |
| 187 xclsuffix = [] | |
| 188 for i, p in enumerate(self.infiles): | |
| 189 if p[IOCLPOS] == "STDIN": | |
| 190 appendme = [ | |
| 191 p[IOCLPOS], | |
| 192 p[ICLPOS], | |
| 193 p[IPATHPOS], | |
| 194 "< %s" % p[IPATHPOS], | |
| 195 ] | |
| 196 xappendme = [ | |
| 197 p[IOCLPOS], | |
| 198 p[ICLPOS], | |
| 199 p[IPATHPOS], | |
| 200 "< $%s" % p[ICLPOS], | |
| 201 ] | |
| 202 else: | |
| 203 appendme = [p[IOCLPOS], p[ICLPOS], p[IPATHPOS], ""] | |
| 204 xappendme = [p[IOCLPOS], p[ICLPOS], "$%s" % p[ICLPOS], ""] | |
| 205 clsuffix.append(appendme) | |
| 206 xclsuffix.append(xappendme) | |
| 207 # print('##infile i=%d, appendme=%s' % (i,appendme)) | |
| 208 for i, p in enumerate(self.outfiles): | |
| 209 if p[OOCLPOS] == "STDOUT": | |
| 210 self.lastclredirect = [">", p[ONAMEPOS]] | |
| 211 self.lastxclredirect = [">", "$%s" % p[OCLPOS]] | |
| 212 else: | |
| 213 clsuffix.append([p[OOCLPOS], p[OCLPOS], p[ONAMEPOS], ""]) | |
| 214 xclsuffix.append( | |
| 215 [p[OOCLPOS], p[OCLPOS], "$%s" % p[ONAMEPOS], ""] | |
| 216 ) | |
| 217 for p in self.addpar: | |
| 218 clsuffix.append( | |
| 219 [p[AOCLPOS], p[ACLPOS], p[AVALPOS], p[AOVERPOS]] | |
| 220 ) | |
| 221 xclsuffix.append( | |
| 222 [p[AOCLPOS], p[ACLPOS], '"$%s"' % p[ANAMEPOS], p[AOVERPOS]] | |
| 223 ) | |
| 224 clsuffix.sort() | |
| 225 xclsuffix.sort() | |
| 226 self.xclsuffix = xclsuffix | |
| 227 self.clsuffix = clsuffix | |
| 228 if self.args.parampass == "positional": | |
| 229 self.clpositional() | |
| 230 else: | |
| 231 self.clargparse() | |
| 232 | |
| 233 def prepScript(self): | |
| 234 aCL = self.cl.append | |
| 235 rx = open(self.args.script_path, "r").readlines() | |
| 236 rx = [x.rstrip() for x in rx] | |
| 237 rxcheck = [x.strip() for x in rx if x.strip() > ""] | |
| 238 assert len(rxcheck) > 0, "Supplied script is empty. Cannot run" | |
| 239 self.script = "\n".join(rx) | |
| 240 fhandle, self.sfile = tempfile.mkstemp( | |
| 241 prefix=self.tool_name, suffix="_%s" % (self.args.interpreter_name) | |
| 242 ) | |
| 243 tscript = open(self.sfile, "w") | |
| 244 tscript.write(self.script) | |
| 245 tscript.close() | |
| 246 self.indentedScript = " %s" % "\n".join( | |
| 247 [" %s" % html_escape(x) for x in rx] | |
| 248 ) | |
| 249 self.escapedScript = "%s" % "\n".join( | |
| 250 [" %s" % html_escape(x) for x in rx] | |
| 251 ) | |
| 252 art = "%s.%s" % (self.tool_name, self.args.interpreter_name) | |
| 253 artifact = open(art, "wb") | |
| 254 if self.args.interpreter_name == "python": | |
| 255 artifact.write(bytes("#!/usr/bin/env python\n", "utf8")) | |
| 256 artifact.write(bytes(self.script, "utf8")) | |
| 257 artifact.close() | |
| 258 aCL(self.args.interpreter_name) | |
| 259 aCL(self.sfile) | |
| 260 | |
| 261 def cleanuppar(self): | |
| 262 """ positional parameters are complicated by their numeric ordinal""" | |
| 263 for i, p in enumerate(self.infiles): | |
| 264 if self.args.parampass == "positional": | |
| 265 assert p[ICLPOS].isdigit(), ( | |
| 266 "Positional parameters must be ordinal integers - got %s for %s" | |
| 267 % (p[ICLPOS], p[ILABPOS]) | |
| 268 ) | |
| 269 p.append(p[ICLPOS]) | |
| 270 if p[ICLPOS].isdigit() or self.args.parampass == "0": | |
| 271 scl = "input%d" % (i + 1) | |
| 272 p[ICLPOS] = scl | |
| 273 self.infiles[i] = p | |
| 274 for i, p in enumerate( | |
| 275 self.outfiles | |
| 276 ): # trying to automagically gather using extensions | |
| 277 if self.args.parampass == "positional" and p[OCLPOS] != "STDOUT": | |
| 278 assert p[OCLPOS].isdigit(), ( | |
| 279 "Positional parameters must be ordinal integers - got %s for %s" | |
| 280 % (p[OCLPOS], p[ONAMEPOS]) | |
| 281 ) | |
| 282 p.append(p[OCLPOS]) | |
| 283 if p[OCLPOS].isdigit() or p[OCLPOS] == "STDOUT": | |
| 284 scl = p[ONAMEPOS] | |
| 285 p[OCLPOS] = scl | |
| 286 self.outfiles[i] = p | |
| 287 for i, p in enumerate(self.addpar): | |
| 288 if self.args.parampass == "positional": | |
| 289 assert p[ACLPOS].isdigit(), ( | |
| 290 "Positional parameters must be ordinal integers - got %s for %s" | |
| 291 % (p[ACLPOS], p[ANAMEPOS]) | |
| 292 ) | |
| 293 p.append(p[ACLPOS]) | |
| 294 if p[ACLPOS].isdigit(): | |
| 295 scl = "input%s" % p[ACLPOS] | |
| 296 p[ACLPOS] = scl | |
| 297 self.addpar[i] = p | |
| 298 | |
| 299 def clsimple(self): | |
| 300 """ no parameters - uses < and > for i/o | |
| 301 """ | |
| 302 aCL = self.cl.append | |
| 303 aCL("<") | |
| 304 aCL(self.infiles[0][IPATHPOS]) | |
| 305 aCL(">") | |
| 306 aCL(self.outfiles[0][OCLPOS]) | |
| 307 aXCL = self.xmlcl.append | |
| 308 aXCL("<") | |
| 309 aXCL("$%s" % self.infiles[0][ICLPOS]) | |
| 310 aXCL(">") | |
| 311 aXCL("$%s" % self.outfiles[0][ONAMEPOS]) | |
| 312 | |
| 313 def clpositional(self): | |
| 314 # inputs in order then params | |
| 315 aCL = self.cl.append | |
| 316 for (o_v, k, v, koverride) in self.clsuffix: | |
| 317 if " " in v: | |
| 318 aCL("%s" % v) | |
| 319 else: | |
| 320 aCL(v) | |
| 321 aXCL = self.xmlcl.append | |
| 322 for (o_v, k, v, koverride) in self.xclsuffix: | |
| 323 aXCL(v) | |
| 324 if self.lastxclredirect: | |
| 325 aXCL(self.lastxclredirect[0]) | |
| 326 aXCL(self.lastxclredirect[1]) | |
| 327 | |
| 328 def clargparse(self): | |
| 329 """ argparse style | |
| 330 """ | |
| 331 aCL = self.cl.append | |
| 332 aXCL = self.xmlcl.append | |
| 333 # inputs then params in argparse named form | |
| 334 for (o_v, k, v, koverride) in self.xclsuffix: | |
| 335 if koverride > "": | |
| 336 k = koverride | |
| 337 elif len(k.strip()) == 1: | |
| 338 k = "-%s" % k | |
| 339 else: | |
| 340 k = "--%s" % k | |
| 341 aXCL(k) | |
| 342 aXCL(v) | |
| 343 for (o_v, k, v, koverride) in self.clsuffix: | |
| 344 if koverride > "": | |
| 345 k = koverride | |
| 346 elif len(k.strip()) == 1: | |
| 347 k = "-%s" % k | |
| 348 else: | |
| 349 k = "--%s" % k | |
| 350 aCL(k) | |
| 351 aCL(v) | |
| 352 | |
| 353 def getNdash(self, newname): | |
| 354 if self.is_positional: | |
| 355 ndash = 0 | |
| 356 else: | |
| 357 ndash = 2 | |
| 358 if len(newname) < 2: | |
| 359 ndash = 1 | |
| 360 return ndash | |
| 361 | |
| 362 def doXMLparam(self): | |
| 363 """flake8 made me do this...""" | |
| 364 for p in self.outfiles: | |
| 365 newname, newfmt, newcl, oldcl = p | |
| 366 ndash = self.getNdash(newcl) | |
| 367 aparm = gxtp.OutputData(newcl, format=newfmt, num_dashes=ndash) | |
| 368 aparm.positional = self.is_positional | |
| 369 if self.is_positional: | |
| 370 if oldcl == "STDOUT": | |
| 371 aparm.positional = 9999999 | |
| 372 aparm.command_line_override = "> $%s" % newcl | |
| 373 else: | |
| 374 aparm.positional = int(oldcl) | |
| 375 aparm.command_line_override = "$%s" % newcl | |
| 376 self.toutputs.append(aparm) | |
| 377 tp = gxtp.TestOutput( | |
| 378 name=newcl, value="%s_sample" % newcl, format=newfmt | |
| 379 ) | |
| 380 self.testparam.append(tp) | |
| 381 for p in self.infiles: | |
| 382 newname = p[ICLPOS] | |
| 383 newfmt = p[IFMTPOS] | |
| 384 ndash = self.getNdash(newname) | |
| 385 if not len(p[ILABPOS]) > 0: | |
| 386 alab = p[ICLPOS] | |
| 387 else: | |
| 388 alab = p[ILABPOS] | |
| 389 aninput = gxtp.DataParam( | |
| 390 newname, | |
| 391 optional=False, | |
| 392 label=alab, | |
| 393 help=p[IHELPOS], | |
| 394 format=newfmt, | |
| 395 multiple=False, | |
| 396 num_dashes=ndash, | |
| 397 ) | |
| 398 aninput.positional = self.is_positional | |
| 399 self.tinputs.append(aninput) | |
| 400 tparm = gxtp.TestParam(name=newname, value="%s_sample" % newname) | |
| 401 self.testparam.append(tparm) | |
| 402 for p in self.addpar: | |
| 403 newname, newval, newlabel, newhelp, newtype, newcl, override, oldcl = p | |
| 404 if not len(newlabel) > 0: | |
| 405 newlabel = newname | |
| 406 ndash = self.getNdash(newname) | |
| 407 if newtype == "text": | |
| 408 aparm = gxtp.TextParam( | |
| 409 newname, | |
| 410 label=newlabel, | |
| 411 help=newhelp, | |
| 412 value=newval, | |
| 413 num_dashes=ndash, | |
| 414 ) | |
| 415 elif newtype == "integer": | |
| 416 aparm = gxtp.IntegerParam( | |
| 417 newname, | |
| 418 label=newname, | |
| 419 help=newhelp, | |
| 420 value=newval, | |
| 421 num_dashes=ndash, | |
| 422 ) | |
| 423 elif newtype == "float": | |
| 424 aparm = gxtp.FloatParam( | |
| 425 newname, | |
| 426 label=newname, | |
| 427 help=newhelp, | |
| 428 value=newval, | |
| 429 num_dashes=ndash, | |
| 430 ) | |
| 431 else: | |
| 432 raise ValueError( | |
| 433 'Unrecognised parameter type "%s" for\ | |
| 434 additional parameter %s in makeXML' | |
| 435 % (newtype, newname) | |
| 436 ) | |
| 437 aparm.positional = self.is_positional | |
| 438 if self.is_positional: | |
| 439 aninput.positional = int(oldcl) | |
| 440 self.tinputs.append(aparm) | |
| 441 self.tparm = gxtp.TestParam(newname, value=newval) | |
| 442 self.testparam.append(tparm) | |
| 443 | |
| 444 def doNoXMLparam(self): | |
| 445 alab = self.infiles[0][ILABPOS] | |
| 446 if len(alab) == 0: | |
| 447 alab = self.infiles[0][ICLPOS] | |
| 448 max1s = ( | |
| 449 "Maximum one input if parampass is 0 - more than one input files supplied - %s" | |
| 450 % str(self.infiles) | |
| 451 ) | |
| 452 assert len(self.infiles) == 1, max1s | |
| 453 newname = self.infiles[0][ICLPOS] | |
| 454 aninput = gxtp.DataParam( | |
| 455 newname, | |
| 456 optional=False, | |
| 457 label=alab, | |
| 458 help=self.infiles[0][IHELPOS], | |
| 459 format=self.infiles[0][IFMTPOS], | |
| 460 multiple=False, | |
| 461 num_dashes=0, | |
| 462 ) | |
| 463 aninput.command_line_override = "< $%s" % newname | |
| 464 aninput.positional = self.is_positional | |
| 465 self.tinputs.append(aninput) | |
| 466 tp = gxtp.TestParam(name=newname, value="%s_sample" % newname) | |
| 467 self.testparam.append(tp) | |
| 468 newname = self.outfiles[0][OCLPOS] | |
| 469 newfmt = self.outfiles[0][OFMTPOS] | |
| 470 anout = gxtp.OutputData(newname, format=newfmt, num_dashes=0) | |
| 471 anout.command_line_override = "> $%s" % newname | |
| 472 anout.positional = self.is_positional | |
| 473 self.toutputs.append(anout) | |
| 474 tp = gxtp.TestOutput( | |
| 475 name=newname, value="%s_sample" % newname, format=newfmt | |
| 476 ) | |
| 477 self.testparam.append(tp) | |
| 478 | |
| 479 def makeXML(self): | |
| 480 """ | |
| 481 Create a Galaxy xml tool wrapper for the new script | |
| 482 Uses galaxyhtml | |
| 483 Hmmm. How to get the command line into correct order... | |
| 484 """ | |
| 485 self.tool.command_line_override = self.xmlcl | |
| 486 if self.args.interpreter_name: | |
| 487 self.tool.interpreter = self.args.interpreter_name | |
| 488 if self.args.help_text: | |
| 489 helptext = open(self.args.help_text, "r").readlines() | |
| 490 helptext = [html_escape(x) for x in helptext] | |
| 491 self.tool.help = "".join([x for x in helptext]) | |
| 492 else: | |
| 493 self.tool.help = ( | |
| 494 "Please ask the tool author (%s) for help \ | |
| 495 as none was supplied at tool generation\n" | |
| 496 % (self.args.user_email) | |
| 497 ) | |
| 498 self.tool.version_command = None # do not want | |
| 499 requirements = gxtp.Requirements() | |
| 500 | |
| 501 if self.args.interpreter_name: | |
| 502 if self.args.interpreter_name == "python": | |
| 503 requirements.append( | |
| 504 gxtp.Requirement( | |
| 505 "package", "python", self.args.interpreter_version | |
| 506 ) | |
| 507 ) | |
| 508 elif self.args.interpreter_name not in ["bash", "sh"]: | |
| 509 requirements.append( | |
| 510 gxtp.Requirement( | |
| 511 "package", | |
| 512 self.args.interpreter_name, | |
| 513 self.args.interpreter_version, | |
| 514 ) | |
| 515 ) | |
| 516 else: | |
| 517 if self.args.exe_package and self.args.parampass != "system": | |
| 518 requirements.append( | |
| 519 gxtp.Requirement( | |
| 520 "package", | |
| 521 self.args.exe_package, | |
| 522 self.args.exe_package_version, | |
| 523 ) | |
| 524 ) | |
| 525 self.tool.requirements = requirements | |
| 526 if self.args.parampass == "0": | |
| 527 self.doNoXMLparam() | |
| 528 else: | |
| 529 self.doXMLparam() | |
| 530 self.tool.outputs = self.toutputs | |
| 531 self.tool.inputs = self.tinputs | |
| 532 if self.args.runmode not in ["Executable", "system"]: | |
| 533 configfiles = gxtp.Configfiles() | |
| 534 configfiles.append(gxtp.Configfile(name="runMe", text=self.script)) | |
| 535 self.tool.configfiles = configfiles | |
| 536 tests = gxtp.Tests() | |
| 537 test_a = gxtp.Test() | |
| 538 for tp in self.testparam: | |
| 539 test_a.append(tp) | |
| 540 tests.append(test_a) | |
| 541 self.tool.tests = tests | |
| 542 self.tool.add_comment( | |
| 543 "Created by %s at %s using the Galaxy Tool Factory." | |
| 544 % (self.args.user_email, timenow()) | |
| 545 ) | |
| 546 self.tool.add_comment("Source in git at: %s" % (toolFactoryURL)) | |
| 547 self.tool.add_comment( | |
| 548 "Cite: Creating re-usable tools from scripts doi: \ | |
| 549 10.1093/bioinformatics/bts573" | |
| 550 ) | |
| 551 exml = self.tool.export() | |
| 552 xf = open('%s.xml' % self.tool_name, "w") | |
| 553 xf.write(exml) | |
| 554 xf.write("\n") | |
| 555 xf.close() | |
| 556 # ready for the tarball | |
| 557 | |
| 558 def makeTooltar(self): | |
| 559 """ | |
| 560 a tool is a gz tarball with eg | |
| 561 /toolname/tool.xml /toolname/tool.py /toolname/test-data/test1_in.foo ... | |
| 562 NOTE names for test inputs and outputs are munged here so must | |
| 563 correspond to actual input and output names used on the generated cl | |
| 564 """ | |
| 565 retval = self.run() | |
| 566 if retval: | |
| 567 sys.stderr.write( | |
| 568 "## Run failed. Cannot build yet. Please fix and retry" | |
| 569 ) | |
| 570 sys.exit(1) | |
| 571 tdir = "tfout" | |
| 572 if not os.path.exists(tdir): | |
| 573 os.mkdir(tdir) | |
| 574 self.makeXML() | |
| 575 testdir = os.path.join(tdir, "test-data") | |
| 576 if not os.path.exists(testdir): | |
| 577 os.mkdir(testdir) # make tests directory | |
| 578 for p in self.infiles: | |
| 579 pth = p[IPATHPOS] | |
| 580 dest = os.path.join(testdir, "%s_sample" % p[ICLPOS]) | |
| 581 shutil.copyfile(pth, dest) | |
| 582 for p in self.outfiles: | |
| 583 pth = p[OCLPOS] | |
| 584 if p[OOCLPOS] == "STDOUT" or self.args.parampass == "0": | |
| 585 pth = p[ONAMEPOS] | |
| 586 dest = os.path.join(testdir, "%s_sample" % p[ONAMEPOS]) | |
| 587 shutil.copyfile(pth, dest) | |
| 588 dest = os.path.join(tdir, p[ONAMEPOS]) | |
| 589 shutil.copyfile(pth, dest) | |
| 590 else: | |
| 591 pth = p[OCLPOS] | |
| 592 dest = os.path.join(testdir, "%s_sample" % p[OCLPOS]) | |
| 593 shutil.copyfile(pth, dest) | |
| 594 dest = os.path.join(tdir, p[OCLPOS]) | |
| 595 shutil.copyfile(pth, dest) | |
| 596 | |
| 597 if os.path.exists(self.tlog) and os.stat(self.tlog).st_size > 0: | |
| 598 shutil.copyfile(self.tlog, os.path.join(testdir, "test1_log_outfiletxt")) | |
| 599 if self.args.runmode not in ["Executable", "system"]: | |
| 600 stname = os.path.join(tdir, "%s" % (self.sfile)) | |
| 601 if not os.path.exists(stname): | |
| 602 shutil.copyfile(self.sfile, stname) | |
| 603 xreal = '%s.xml' % self.tool_name | |
| 604 xout = os.path.join(tdir,xreal) | |
| 605 shutil.copyfile(xreal, xout) | |
| 606 tarpath = "toolfactory_%s.tgz" % self.tool_name | |
| 607 tf = tarfile.open(tarpath, "w:gz") | |
| 608 tf.add(name=tdir, arcname=self.tool_name) | |
| 609 tf.close() | |
| 610 shutil.copyfile(tarpath, self.args.new_tool) | |
| 611 shutil.copyfile(xreal,"tool_xml.txt") | |
| 612 repdir = "TF_run_report_tempdir" | |
| 613 if not os.path.exists(repdir): | |
| 614 os.mkdir(repdir) | |
| 615 repoutnames = [x[OCLPOS] for x in self.outfiles] | |
| 616 with os.scandir('.') as outs: | |
| 617 for entry in outs: | |
| 618 if entry.name.endswith('.tgz') or not entry.is_file(): | |
| 619 continue | |
| 620 if entry.name in repoutnames: | |
| 621 shutil.copyfile(entry.name,os.path.join(repdir,entry.name)) | |
| 622 elif entry.name == "%s.xml" % self.tool_name: | |
| 623 shutil.copyfile(entry.name,os.path.join(repdir,"new_tool_xml")) | |
| 624 return retval | |
| 625 | |
| 626 def run(self): | |
| 627 """ | |
| 628 Some devteam tools have this defensive stderr read so I'm keeping with the faith | |
| 629 Feel free to update. | |
| 630 """ | |
| 631 s = "run cl=%s" % str(self.cl) | |
| 632 | |
| 633 logging.debug(s) | |
| 634 scl = " ".join(self.cl) | |
| 635 err = None | |
| 636 if self.args.parampass != "0": | |
| 637 ste = open(self.elog, "wb") | |
| 638 if self.lastclredirect: | |
| 639 sto = open( | |
| 640 self.lastclredirect[1], "wb" | |
| 641 ) # is name of an output file | |
| 642 else: | |
| 643 sto = open(self.tlog, "wb") | |
| 644 sto.write( | |
| 645 bytes( | |
| 646 "## Executing Toolfactory generated command line = %s\n" | |
| 647 % scl, | |
| 648 "utf8", | |
| 649 ) | |
| 650 ) | |
| 651 sto.flush() | |
| 652 p = subprocess.run(self.cl, shell=False, stdout=sto, stderr=ste) | |
| 653 sto.close() | |
| 654 ste.close() | |
| 655 tmp_stderr = open(self.elog, "rb") | |
| 656 err = "" | |
| 657 buffsize = 1048576 | |
| 658 try: | |
| 659 while True: | |
| 660 err += str(tmp_stderr.read(buffsize)) | |
| 661 if not err or len(err) % buffsize != 0: | |
| 662 break | |
| 663 except OverflowError: | |
| 664 pass | |
| 665 tmp_stderr.close() | |
| 666 retval = p.returncode | |
| 667 else: # work around special case of simple scripts that take stdin and write to stdout | |
| 668 sti = open(self.infiles[0][IPATHPOS], "rb") | |
| 669 sto = open(self.outfiles[0][ONAMEPOS], "wb") | |
| 670 # must use shell to redirect | |
| 671 p = subprocess.run(self.cl, shell=False, stdout=sto, stdin=sti) | |
| 672 retval = p.returncode | |
| 673 sto.close() | |
| 674 sti.close() | |
| 675 if os.path.isfile(self.tlog) and os.stat(self.tlog).st_size == 0: | |
| 676 os.unlink(self.tlog) | |
| 677 if os.path.isfile(self.elog) and os.stat(self.elog).st_size == 0: | |
| 678 os.unlink(self.elog) | |
| 679 if p.returncode != 0 and err: # problem | |
| 680 sys.stderr.write(err) | |
| 681 logging.debug("run done") | |
| 682 return retval | |
| 683 | |
| 684 | |
| 685 def main(): | |
| 686 """ | |
| 687 This is a Galaxy wrapper. It expects to be called by a special purpose tool.xml as: | |
| 688 <command interpreter="python">rgBaseScriptWrapper.py --script_path "$scriptPath" --tool_name "foo" --interpreter "Rscript" | |
| 689 </command> | |
| 690 """ | |
| 691 parser = argparse.ArgumentParser() | |
| 692 a = parser.add_argument | |
| 693 a("--script_path", default="") | |
| 694 a("--tool_name", default=None) | |
| 695 a("--interpreter_name", default=None) | |
| 696 a("--interpreter_version", default=None) | |
| 697 a("--exe_package", default=None) | |
| 698 a("--exe_package_version", default=None) | |
| 699 a("--input_files", default=[], action="append") | |
| 700 a("--output_files", default=[], action="append") | |
| 701 a("--user_email", default="Unknown") | |
| 702 a("--bad_user", default=None) | |
| 703 a("--make_Tool", default=None) | |
| 704 a("--help_text", default=None) | |
| 705 a("--tool_desc", default=None) | |
| 706 a("--tool_version", default=None) | |
| 707 a("--citations", default=None) | |
| 708 a("--additional_parameters", action="append", default=[]) | |
| 709 a("--edit_additional_parameters", action="store_true", default=False) | |
| 710 a("--parampass", default="positional") | |
| 711 a("--tfout", default="./tfout") | |
| 712 a("--new_tool", default="new_tool") | |
| 713 a("--runmode", default=None) | |
| 714 args = parser.parse_args() | |
| 715 assert not args.bad_user, ( | |
| 716 'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy admin adds %s to "admin_users" in the Galaxy configuration file' | |
| 717 % (args.bad_user, args.bad_user) | |
| 718 ) | |
| 719 assert ( | |
| 720 args.tool_name | |
| 721 ), "## Tool Factory expects a tool name - eg --tool_name=DESeq" | |
| 722 assert ( | |
| 723 args.interpreter_name or args.exe_package | |
| 724 ), "## Tool Factory wrapper expects an interpreter or an executable package" | |
| 725 assert args.exe_package or ( | |
| 726 len(args.script_path) > 0 and os.path.isfile(args.script_path) | |
| 727 ), "## Tool Factory wrapper expects a script path - eg --script_path=foo.R if no executable" | |
| 728 args.input_files = [ | |
| 729 x.replace('"', "").replace("'", "") for x in args.input_files | |
| 730 ] | |
| 731 # remove quotes we need to deal with spaces in CL params | |
| 732 for i, x in enumerate(args.additional_parameters): | |
| 733 args.additional_parameters[i] = args.additional_parameters[i].replace( | |
| 734 '"', "" | |
| 735 ) | |
| 736 r = ScriptRunner(args) | |
| 737 if args.make_Tool: | |
| 738 retcode = r.makeTooltar() | |
| 739 else: | |
| 740 retcode = r.run() | |
| 741 if retcode: | |
| 742 sys.exit(retcode) # indicate failure to job runner | |
| 743 | |
| 744 | |
| 745 if __name__ == "__main__": | |
| 746 main() |
