Mercurial > repos > fubar > tool_factory_2

comparison toolfactory/rgToolFactory2.py @ 121:2050b2475ae5 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Uploaded
author fubar
date Thu, 07 Jan 2021 09:24:17 +0000
parents
children c763ae9237ca
comparison
equal deleted inserted replaced
120:0c6c3e10a8f4 121:2050b2475ae5
1 # replace with shebang for biocontainer
2 # see https://github.com/fubar2/toolfactory
3 #
4 # copyright ross lazarus (ross stop lazarus at gmail stop com) May 2012
5 #
6 # all rights reserved
7 # Licensed under the LGPL
8 # suggestions for improvement and bug fixes welcome at
9 # https://github.com/fubar2/toolfactory
10 #
11 # July 2020: BCC was fun and I feel like rip van winkle after 5 years.
12 # Decided to
13 # 1. Fix the toolfactory so it works - done for simplest case
14 # 2. Fix planemo so the toolfactory function works
15 # 3. Rewrite bits using galaxyxml functions where that makes sense - done
16
17 import argparse
18 import copy
19 import logging
20 import os
21 import re
22 import shutil
23 import subprocess
24 import sys
25 import tarfile
26 import tempfile
27 import time
28
29 from bioblend import ConnectionError
30 from bioblend import toolshed
31
32 import galaxyxml.tool as gxt
33 import galaxyxml.tool.parameters as gxtp
34
35 import lxml
36
37 import yaml
38
39 myversion = "V2.1 July 2020"
40 verbose = True
41 debug = True
42 toolFactoryURL = "https://github.com/fubar2/toolfactory"
43 ourdelim = "~~~"
44
45 # --input_files="$intab.input_files~~~$intab.input_CL~~~
46 # $intab.input_formats# ~~~$intab.input_label
47 # ~~~$intab.input_help"
48 IPATHPOS = 0
49 ICLPOS = 1
50 IFMTPOS = 2
51 ILABPOS = 3
52 IHELPOS = 4
53 IOCLPOS = 5
54
55 # --output_files "$otab.history_name~~~$otab.history_format~~~
56 # $otab.history_CL~~~$otab.history_test"
57 ONAMEPOS = 0
58 OFMTPOS = 1
59 OCLPOS = 2
60 OTESTPOS = 3
61 OOCLPOS = 4
62
63
64 # --additional_parameters="$i.param_name~~~$i.param_value~~~
65 # $i.param_label~~~$i.param_help~~~$i.param_type
66 # ~~~$i.CL~~~i$.param_CLoverride"
67 ANAMEPOS = 0
68 AVALPOS = 1
69 ALABPOS = 2
70 AHELPPOS = 3
71 ATYPEPOS = 4
72 ACLPOS = 5
73 AOVERPOS = 6
74 AOCLPOS = 7
75
76
77 foo = len(lxml.__version__)
78 # fug you, flake8. Say my name!
79 FAKEEXE = "~~~REMOVE~~~ME~~~"
80 # need this until a PR/version bump to fix galaxyxml prepending the exe even
81 # with override.
82
83
84 def timenow():
85 """return current time as a string"""
86 return time.strftime("%d/%m/%Y %H:%M:%S", time.localtime(time.time()))
87
88
89 def quote_non_numeric(s):
90 """return a prequoted string for non-numerics
91 useful for perl and Rscript parameter passing?
92 """
93 try:
94 _ = float(s)
95 return s
96 except ValueError:
97 return '"%s"' % s
98
99
100 html_escape_table = {
101 "&": "&",
102 ">": ">",
103 "<": "&lt;",
104 "#": "&#35;",
105 "$": "&#36;",
106 }
107 cheetah_escape_table = {"$": "\\$", "#": "\\#"}
108
109
110 def html_escape(text):
111 """Produce entities within text."""
112 return "".join([html_escape_table.get(c, c) for c in text])
113
114
115 def cheetah_escape(text):
116 """Produce entities within text."""
117 return "".join([cheetah_escape_table.get(c, c) for c in text])
118
119
120 def parse_citations(citations_text):
121 """"""
122 citations = [c for c in citations_text.split("**ENTRY**") if c.strip()]
123 citation_tuples = []
124 for citation in citations:
125 if citation.startswith("doi"):
126 citation_tuples.append(("doi", citation[len("doi") :].strip()))
127 else:
128 citation_tuples.append(("bibtex", citation[len("bibtex") :].strip()))
129 return citation_tuples
130
131
132 class ScriptRunner:
133 """Wrapper for an arbitrary script
134 uses galaxyxml
135
136 """
137
138 def __init__(self, args=None):
139 """
140 prepare command line cl for running the tool here
141 and prepare elements needed for galaxyxml tool generation
142 """
143 self.ourcwd = os.getcwd()
144 self.ourenv = copy.deepcopy(os.environ)
145 self.infiles = [x.split(ourdelim) for x in args.input_files]
146 self.outfiles = [x.split(ourdelim) for x in args.output_files]
147 self.addpar = [x.split(ourdelim) for x in args.additional_parameters]
148 self.args = args
149 self.cleanuppar()
150 self.lastclredirect = None
151 self.lastxclredirect = None
152 self.cl = []
153 self.xmlcl = []
154 self.is_positional = self.args.parampass == "positional"
155 if self.args.sysexe:
156 self.executeme = self.args.sysexe
157 else:
158 if self.args.packages:
159 self.executeme = self.args.packages.split(",")[0].split(":")[0].strip()
160 else:
161 self.executeme = None
162 aCL = self.cl.append
163 aXCL = self.xmlcl.append
164 assert args.parampass in [
165 "0",
166 "argparse",
167 "positional",
168 ], 'args.parampass must be "0","positional" or "argparse"'
169 self.tool_name = re.sub("[^a-zA-Z0-9_]+", "", args.tool_name)
170 self.tool_id = self.tool_name
171 self.newtool = gxt.Tool(
172 self.tool_name,
173 self.tool_id,
174 self.args.tool_version,
175 self.args.tool_desc,
176 FAKEEXE,
177 )
178 self.newtarpath = "toolfactory_%s.tgz" % self.tool_name
179 self.tooloutdir = "./tfout"
180 self.repdir = "./TF_run_report_tempdir"
181 self.testdir = os.path.join(self.tooloutdir, "test-data")
182 if not os.path.exists(self.tooloutdir):
183 os.mkdir(self.tooloutdir)
184 if not os.path.exists(self.testdir):
185 os.mkdir(self.testdir)
186 if not os.path.exists(self.repdir):
187 os.mkdir(self.repdir)
188 self.tinputs = gxtp.Inputs()
189 self.toutputs = gxtp.Outputs()
190 self.testparam = []
191 if self.args.script_path:
192 self.prepScript()
193 if self.args.command_override:
194 scos = open(self.args.command_override, "r").readlines()
195 self.command_override = [x.rstrip() for x in scos]
196 else:
197 self.command_override = None
198 if self.args.test_override:
199 stos = open(self.args.test_override, "r").readlines()
200 self.test_override = [x.rstrip() for x in stos]
201 else:
202 self.test_override = None
203 if self.args.cl_prefix: # DIY CL start
204 clp = self.args.cl_prefix.split(" ")
205 for c in clp:
206 aCL(c)
207 aXCL(c)
208 else:
209 if self.args.script_path:
210 aCL(self.executeme)
211 aCL(self.sfile)
212 aXCL(self.executeme)
213 aXCL("$runme")
214 else:
215 aCL(self.executeme)
216 aXCL(self.executeme)
217 self.elog = os.path.join(self.repdir, "%s_error_log.txt" % self.tool_name)
218 self.tlog = os.path.join(self.repdir, "%s_runner_log.txt" % self.tool_name)
219
220 if self.args.parampass == "0":
221 self.clsimple()
222 else:
223 clsuffix = []
224 xclsuffix = []
225 for i, p in enumerate(self.infiles):
226 if p[IOCLPOS].upper() == "STDIN":
227 appendme = [
228 p[ICLPOS],
229 p[ICLPOS],
230 p[IPATHPOS],
231 "< %s" % p[IPATHPOS],
232 ]
233 xappendme = [
234 p[ICLPOS],
235 p[ICLPOS],
236 p[IPATHPOS],
237 "< $%s" % p[ICLPOS],
238 ]
239 else:
240 appendme = [p[IOCLPOS], p[ICLPOS], p[IPATHPOS], ""]
241 xappendme = [p[IOCLPOS], p[ICLPOS], "$%s" % p[ICLPOS], ""]
242 clsuffix.append(appendme)
243 xclsuffix.append(xappendme)
244 for i, p in enumerate(self.outfiles):
245 if p[OOCLPOS] == "STDOUT":
246 self.lastclredirect = [">", p[ONAMEPOS]]
247 self.lastxclredirect = [">", "$%s" % p[OCLPOS]]
248 else:
249 clsuffix.append([p[OCLPOS], p[ONAMEPOS], p[ONAMEPOS], ""])
250 xclsuffix.append([p[OCLPOS], p[ONAMEPOS], "$%s" % p[ONAMEPOS], ""])
251 for p in self.addpar:
252 clsuffix.append([p[AOCLPOS], p[ACLPOS], p[AVALPOS], p[AOVERPOS]])
253 xclsuffix.append(
254 [p[AOCLPOS], p[ACLPOS], '"$%s"' % p[ANAMEPOS], p[AOVERPOS]]
255 )
256 clsuffix.sort()
257 xclsuffix.sort()
258 self.xclsuffix = xclsuffix
259 self.clsuffix = clsuffix
260 if self.args.parampass == "positional":
261 self.clpositional()
262 else:
263 self.clargparse()
264
265 def prepScript(self):
266 rx = open(self.args.script_path, "r").readlines()
267 rx = [x.rstrip() for x in rx]
268 rxcheck = [x.strip() for x in rx if x.strip() > ""]
269 assert len(rxcheck) > 0, "Supplied script is empty. Cannot run"
270 self.script = "\n".join(rx)
271 fhandle, self.sfile = tempfile.mkstemp(
272 prefix=self.tool_name, suffix="_%s" % (self.executeme)
273 )
274 tscript = open(self.sfile, "w")
275 tscript.write(self.script)
276 tscript.close()
277 self.escapedScript = [cheetah_escape(x) for x in rx]
278 self.spacedScript = [f" {x}" for x in rx if x.strip() > ""]
279 art = "%s.%s" % (self.tool_name, self.executeme)
280 artifact = open(art, "wb")
281 artifact.write(bytes("\n".join(self.escapedScript), "utf8"))
282 artifact.close()
283
284 def cleanuppar(self):
285 """ positional parameters are complicated by their numeric ordinal"""
286 if self.args.parampass == "positional":
287 for i, p in enumerate(self.infiles):
288 assert (
289 p[ICLPOS].isdigit() or p[ICLPOS].strip().upper() == "STDIN"
290 ), "Positional parameters must be ordinal integers - got %s for %s" % (
291 p[ICLPOS],
292 p[ILABPOS],
293 )
294 for i, p in enumerate(self.outfiles):
295 assert (
296 p[OCLPOS].isdigit() or p[OCLPOS].strip().upper() == "STDOUT"
297 ), "Positional parameters must be ordinal integers - got %s for %s" % (
298 p[OCLPOS],
299 p[ONAMEPOS],
300 )
301 for i, p in enumerate(self.addpar):
302 assert p[
303 ACLPOS
304 ].isdigit(), "Positional parameters must be ordinal integers - got %s for %s" % (
305 p[ACLPOS],
306 p[ANAMEPOS],
307 )
308 for i, p in enumerate(self.infiles):
309 infp = copy.copy(p)
310 icl = infp[ICLPOS]
311 infp.append(icl)
312 if (
313 infp[ICLPOS].isdigit() \
314 or self.args.parampass == "0" \
315 or infp[ICLPOS].strip().upper() == "STDOUT"
316 ):
317 scl = "input%d" % (i + 1)
318 infp[ICLPOS] = scl
319 self.infiles[i] = infp
320 for i, p in enumerate(self.outfiles):
321 p.append(p[OCLPOS]) # keep copy
322 if (p[OOCLPOS].isdigit() and self.args.parampass != "positional") or p[
323 OOCLPOS
324 ].strip().upper() == "STDOUT":
325 scl = p[ONAMEPOS]
326 p[OCLPOS] = scl
327 self.outfiles[i] = p
328 for i, p in enumerate(self.addpar):
329 p.append(p[ACLPOS])
330 if p[ACLPOS].isdigit():
331 scl = "param%s" % p[ACLPOS]
332 p[ACLPOS] = scl
333 self.addpar[i] = p
334
335 def clsimple(self):
336 """no parameters - uses < and > for i/o"""
337 aCL = self.cl.append
338 aXCL = self.xmlcl.append
339
340 if len(self.infiles) > 0:
341 aCL("<")
342 aCL(self.infiles[0][IPATHPOS])
343 aXCL("<")
344 aXCL("$%s" % self.infiles[0][ICLPOS])
345 if len(self.outfiles) > 0:
346 aCL(">")
347 aCL(self.outfiles[0][OCLPOS])
348 aXCL(">")
349 aXCL("$%s" % self.outfiles[0][ONAMEPOS])
350
351 def clpositional(self):
352 # inputs in order then params
353 aCL = self.cl.append
354 for (o_v, k, v, koverride) in self.clsuffix:
355 if " " in v:
356 aCL("%s" % v)
357 else:
358 aCL(v)
359 aXCL = self.xmlcl.append
360 for (o_v, k, v, koverride) in self.xclsuffix:
361 aXCL(v)
362 if self.lastxclredirect:
363 aXCL(self.lastxclredirect[0])
364 aXCL(self.lastxclredirect[1])
365
366 def clargparse(self):
367 """argparse style"""
368 aCL = self.cl.append
369 aXCL = self.xmlcl.append
370 # inputs then params in argparse named form
371
372 for (o_v, k, v, koverride) in self.xclsuffix:
373 if koverride > "":
374 k = koverride
375 elif len(k.strip()) == 1:
376 k = "-%s" % k
377 else:
378 k = "--%s" % k
379 aXCL(k)
380 aXCL(v)
381 for (o_v, k, v, koverride) in self.clsuffix:
382 if koverride > "":
383 k = koverride
384 elif len(k.strip()) == 1:
385 k = "-%s" % k
386 else:
387 k = "--%s" % k
388 aCL(k)
389 aCL(v)
390
391 def getNdash(self, newname):
392 if self.is_positional:
393 ndash = 0
394 else:
395 ndash = 2
396 if len(newname) < 2:
397 ndash = 1
398 return ndash
399
400 def doXMLparam(self):
401 """flake8 made me do this..."""
402 for p in self.outfiles:
403 # --output_files "$otab.history_name~~~$otab.history_format~~~$otab.history_CL~~~$otab.history_test"
404 newname, newfmt, newcl, test, oldcl = p
405 test = test.strip()
406 ndash = self.getNdash(newcl)
407 aparm = gxtp.OutputData(
408 name=newname, format=newfmt, num_dashes=ndash, label=newcl
409 )
410 aparm.positional = self.is_positional
411 if self.is_positional:
412 if oldcl.upper() == "STDOUT":
413 aparm.positional = 9999999
414 aparm.command_line_override = "> $%s" % newname
415 else:
416 aparm.positional = int(oldcl)
417 aparm.command_line_override = "$%s" % newname
418 self.toutputs.append(aparm)
419 ld = None
420 if test.strip() > "":
421 if test.startswith("diff"):
422 c = "diff"
423 ld = 0
424 if test.split(":")[1].isdigit:
425 ld = int(test.split(":")[1])
426 tp = gxtp.TestOutput(
427 name=newname,
428 value="%s_sample" % newname,
429 compare=c,
430 lines_diff=ld,
431 )
432 elif test.startswith("sim_size"):
433 c = "sim_size"
434 tn = test.split(":")[1].strip()
435 if tn > "":
436 if "." in tn:
437 delta = None
438 delta_frac = min(1.0, float(tn))
439 else:
440 delta = int(tn)
441 delta_frac = None
442 tp = gxtp.TestOutput(
443 name=newname,
444 value="%s_sample" % newname,
445 compare=c,
446 delta=delta,
447 delta_frac=delta_frac,
448 )
449 self.testparam.append(tp)
450 for p in self.infiles:
451 newname = p[ICLPOS]
452 newfmt = p[IFMTPOS]
453 ndash = self.getNdash(newname)
454 if not len(p[ILABPOS]) > 0:
455 alab = p[ICLPOS]
456 else:
457 alab = p[ILABPOS]
458 aninput = gxtp.DataParam(
459 newname,
460 optional=False,
461 label=alab,
462 help=p[IHELPOS],
463 format=newfmt,
464 multiple=False,
465 num_dashes=ndash,
466 )
467 aninput.positional = self.is_positional
468 self.tinputs.append(aninput)
469 tparm = gxtp.TestParam(name=newname, value="%s_sample" % newname)
470 self.testparam.append(tparm)
471 for p in self.addpar:
472 (
473 newname,
474 newval,
475 newlabel,
476 newhelp,
477 newtype,
478 newcl,
479 override,
480 oldcl,
481 ) = p
482 if not len(newlabel) > 0:
483 newlabel = newname
484 ndash = self.getNdash(newname)
485 if newtype == "text":
486 aparm = gxtp.TextParam(
487 newname,
488 label=newlabel,
489 help=newhelp,
490 value=newval,
491 num_dashes=ndash,
492 )
493 elif newtype == "integer":
494 aparm = gxtp.IntegerParam(
495 newname,
496 label=newname,
497 help=newhelp,
498 value=newval,
499 num_dashes=ndash,
500 )
501 elif newtype == "float":
502 aparm = gxtp.FloatParam(
503 newname,
504 label=newname,
505 help=newhelp,
506 value=newval,
507 num_dashes=ndash,
508 )
509 else:
510 raise ValueError(
511 'Unrecognised parameter type "%s" for\
512 additional parameter %s in makeXML'
513 % (newtype, newname)
514 )
515 aparm.positional = self.is_positional
516 if self.is_positional:
517 aparm.positional = int(oldcl)
518 self.tinputs.append(aparm)
519 tparm = gxtp.TestParam(newname, value=newval)
520 self.testparam.append(tparm)
521
522 def doNoXMLparam(self):
523 """filter style package - stdin to stdout"""
524 if len(self.infiles) > 0:
525 alab = self.infiles[0][ILABPOS]
526 if len(alab) == 0:
527 alab = self.infiles[0][ICLPOS]
528 max1s = (
529 "Maximum one input if parampass is 0 but multiple input files supplied - %s"
530 % str(self.infiles)
531 )
532 assert len(self.infiles) == 1, max1s
533 newname = self.infiles[0][ICLPOS]
534 aninput = gxtp.DataParam(
535 newname,
536 optional=False,
537 label=alab,
538 help=self.infiles[0][IHELPOS],
539 format=self.infiles[0][IFMTPOS],
540 multiple=False,
541 num_dashes=0,
542 )
543 aninput.command_line_override = "< $%s" % newname
544 aninput.positional = self.is_positional
545 self.tinputs.append(aninput)
546 tp = gxtp.TestParam(name=newname, value="%s_sample" % newname)
547 self.testparam.append(tp)
548 if len(self.outfiles) > 0:
549 newname = self.outfiles[0][OCLPOS]
550 newfmt = self.outfiles[0][OFMTPOS]
551 anout = gxtp.OutputData(newname, format=newfmt, num_dashes=0)
552 anout.command_line_override = "> $%s" % newname
553 anout.positional = self.is_positional
554 self.toutputs.append(anout)
555 tp = gxtp.TestOutput(
556 name=newname, value="%s_sample" % newname
557 )
558 self.testparam.append(tp)
559
560 def makeXML(self):
561 """
562 Create a Galaxy xml tool wrapper for the new script
563 Uses galaxyhtml
564 Hmmm. How to get the command line into correct order...
565 """
566 if self.command_override:
567 self.newtool.command_override = self.command_override # config file
568 else:
569 self.newtool.command_override = self.xmlcl
570 cite = gxtp.Citations()
571 acite = gxtp.Citation(type="doi", value="10.1093/bioinformatics/bts573")
572 cite.append(acite)
573 self.newtool.citations = cite
574 safertext = ""
575 if self.args.help_text:
576 helptext = open(self.args.help_text, "r").readlines()
577 safertext = "\n".join([cheetah_escape(x) for x in helptext])
578 if len(safertext.strip()) == 0:
579 safertext = (
580 "Ask the tool author (%s) to rebuild with help text please\n"
581 % (self.args.user_email)
582 )
583 if self.args.script_path:
584 if len(safertext) > 0:
585 safertext = safertext + "\n\n------\n" # transition allowed!
586 scr = [x for x in self.spacedScript if x.strip() > ""]
587 scr.insert(0, "\n\nScript::\n")
588 if len(scr) > 300:
589 scr = (
590 scr[:100] \
591 + [" >300 lines - stuff deleted", " ......"] \
592 + scr[-100:]
593 )
594 scr.append("\n")
595 safertext = safertext + "\n".join(scr)
596 self.newtool.help = safertext
597 self.newtool.version_command = f'echo "{self.args.tool_version}"'
598 requirements = gxtp.Requirements()
599 if self.args.packages:
600 for d in self.args.packages.split(","):
601 ver = ""
602 d = d.replace("==", ":")
603 d = d.replace("=", ":")
604 if ":" in d:
605 packg, ver = d.split(":")
606 else:
607 packg = d
608 requirements.append(
609 gxtp.Requirement("package", packg.strip(), ver.strip())
610 )
611 self.newtool.requirements = requirements
612 if self.args.parampass == "0":
613 self.doNoXMLparam()
614 else:
615 self.doXMLparam()
616 self.newtool.outputs = self.toutputs
617 self.newtool.inputs = self.tinputs
618 if self.args.script_path:
619 configfiles = gxtp.Configfiles()
620 configfiles.append(
621 gxtp.Configfile(name="runme", text="\n".join(self.escapedScript))
622 )
623 self.newtool.configfiles = configfiles
624 tests = gxtp.Tests()
625 test_a = gxtp.Test()
626 for tp in self.testparam:
627 test_a.append(tp)
628 tests.append(test_a)
629 self.newtool.tests = tests
630 self.newtool.add_comment(
631 "Created by %s at %s using the Galaxy Tool Factory."
632 % (self.args.user_email, timenow())
633 )
634 self.newtool.add_comment("Source in git at: %s" % (toolFactoryURL))
635 exml0 = self.newtool.export()
636 exml = exml0.replace(FAKEEXE, "") # temporary work around until PR accepted
637 if (
638 self.test_override
639 ): # cannot do this inside galaxyxml as it expects lxml objects for tests
640 part1 = exml.split("<tests>")[0]
641 part2 = exml.split("</tests>")[1]
642 fixed = "%s\n%s\n%s" % (part1, self.test_override, part2)
643 exml = fixed
644 # exml = exml.replace('range="1:"', 'range="1000:"')
645 xf = open("%s.xml" % self.tool_name, "w")
646 xf.write(exml)
647 xf.write("\n")
648 xf.close()
649 # ready for the tarball
650
651 def run(self):
652 """
653 generate test outputs by running a command line
654 won't work if command or test override in play - planemo is the
655 easiest way to generate test outputs for that case so is
656 automagically selected
657 """
658 scl = " ".join(self.cl)
659 err = None
660 if self.args.parampass != "0":
661 if os.path.exists(self.elog):
662 ste = open(self.elog, "a")
663 else:
664 ste = open(self.elog, "w")
665 if self.lastclredirect:
666 sto = open(self.lastclredirect[1], "wb") # is name of an output file
667 else:
668 if os.path.exists(self.tlog):
669 sto = open(self.tlog, "a")
670 else:
671 sto = open(self.tlog, "w")
672 sto.write(
673 "## Executing Toolfactory generated command line = %s\n" % scl
674 )
675 sto.flush()
676 subp = subprocess.run(
677 self.cl, env=self.ourenv, shell=False, stdout=sto, stderr=ste
678 )
679 sto.close()
680 ste.close()
681 retval = subp.returncode
682 else: # work around special case - stdin and write to stdout
683 if len(self.infiles) > 0:
684 sti = open(self.infiles[0][IPATHPOS], "rb")
685 else:
686 sti = sys.stdin
687 if len(self.outfiles) > 0:
688 sto = open(self.outfiles[0][ONAMEPOS], "wb")
689 else:
690 sto = sys.stdout
691 subp = subprocess.run(
692 self.cl, env=self.ourenv, shell=False, stdout=sto, stdin=sti
693 )
694 sto.write("## Executing Toolfactory generated command line = %s\n" % scl)
695 retval = subp.returncode
696 sto.close()
697 sti.close()
698 if os.path.isfile(self.tlog) and os.stat(self.tlog).st_size == 0:
699 os.unlink(self.tlog)
700 if os.path.isfile(self.elog) and os.stat(self.elog).st_size == 0:
701 os.unlink(self.elog)
702 if retval != 0 and err: # problem
703 sys.stderr.write(err)
704 logging.debug("run done")
705 return retval
706
707 def shedLoad(self):
708 """
709 use bioblend to create new repository
710 or update existing
711
712 """
713 if os.path.exists(self.tlog):
714 sto = open(self.tlog, "a")
715 else:
716 sto = open(self.tlog, "w")
717
718 ts = toolshed.ToolShedInstance(
719 url=self.args.toolshed_url,
720 key=self.args.toolshed_api_key,
721 verify=False,
722 )
723 repos = ts.repositories.get_repositories()
724 rnames = [x.get("name", "?") for x in repos]
725 rids = [x.get("id", "?") for x in repos]
726 tfcat = "ToolFactory generated tools"
727 if self.tool_name not in rnames:
728 tscat = ts.categories.get_categories()
729 cnames = [x.get("name", "?").strip() for x in tscat]
730 cids = [x.get("id", "?") for x in tscat]
731 catID = None
732 if tfcat.strip() in cnames:
733 ci = cnames.index(tfcat)
734 catID = cids[ci]
735 res = ts.repositories.create_repository(
736 name=self.args.tool_name,
737 synopsis="Synopsis:%s" % self.args.tool_desc,
738 description=self.args.tool_desc,
739 type="unrestricted",
740 remote_repository_url=self.args.toolshed_url,
741 homepage_url=None,
742 category_ids=catID,
743 )
744 tid = res.get("id", None)
745 sto.write(f"#create_repository {self.args.tool_name} tid={tid} res={res}\n")
746 else:
747 i = rnames.index(self.tool_name)
748 tid = rids[i]
749 try:
750 res = ts.repositories.update_repository(
751 id=tid, tar_ball_path=self.newtarpath, commit_message=None
752 )
753 sto.write(f"#update res id {id} ={res}\n")
754 except ConnectionError:
755 sto.write(
756 "####### Is the toolshed running and the API key correct? Bioblend shed upload failed\n"
757 )
758 sto.close()
759
760 def eph_galaxy_load(self):
761 """
762 use ephemeris to load the new tool from the local toolshed after planemo uploads it
763 """
764 if os.path.exists(self.tlog):
765 tout = open(self.tlog, "a")
766 else:
767 tout = open(self.tlog, "w")
768 cll = [
769 "shed-tools",
770 "install",
771 "-g",
772 self.args.galaxy_url,
773 "--latest",
774 "-a",
775 self.args.galaxy_api_key,
776 "--name",
777 self.tool_name,
778 "--owner",
779 "fubar",
780 "--toolshed",
781 self.args.toolshed_url,
782 "--section_label",
783 "ToolFactory",
784 ]
785 tout.write("running\n%s\n" % " ".join(cll))
786 subp = subprocess.run(
787 cll,
788 env=self.ourenv,
789 cwd=self.ourcwd,
790 shell=False,
791 stderr=tout,
792 stdout=tout,
793 )
794 tout.write(
795 "installed %s - got retcode %d\n" % (self.tool_name, subp.returncode)
796 )
797 tout.close()
798 return subp.returncode
799
800 def writeShedyml(self):
801 """for planemo"""
802 yuser = self.args.user_email.split("@")[0]
803 yfname = os.path.join(self.tooloutdir, ".shed.yml")
804 yamlf = open(yfname, "w")
805 odict = {
806 "name": self.tool_name,
807 "owner": yuser,
808 "type": "unrestricted",
809 "description": self.args.tool_desc,
810 "synopsis": self.args.tool_desc,
811 "category": "TF Generated Tools",
812 }
813 yaml.dump(odict, yamlf, allow_unicode=True)
814 yamlf.close()
815
816 def makeTool(self):
817 """write xmls and input samples into place"""
818 self.makeXML()
819 if self.args.script_path:
820 stname = os.path.join(self.tooloutdir, "%s" % (self.sfile))
821 if not os.path.exists(stname):
822 shutil.copyfile(self.sfile, stname)
823 xreal = "%s.xml" % self.tool_name
824 xout = os.path.join(self.tooloutdir, xreal)
825 shutil.copyfile(xreal, xout)
826 for p in self.infiles:
827 pth = p[IPATHPOS]
828 dest = os.path.join(self.testdir, "%s_sample" % p[ICLPOS])
829 shutil.copyfile(pth, dest)
830
831 def makeToolTar(self):
832 """move outputs into test-data and prepare the tarball"""
833 excludeme = "_planemo_test_report.html"
834
835 def exclude_function(tarinfo):
836 filename = tarinfo.name
837 return None if filename.endswith(excludeme) else tarinfo
838
839 if os.path.exists(self.tlog):
840 tout = open(self.tlog, "a")
841 else:
842 tout = open(self.tlog, "w")
843 for p in self.outfiles:
844 oname = p[ONAMEPOS]
845 tdest = os.path.join(self.testdir, "%s_sample" % oname)
846 if not os.path.isfile(tdest):
847 src = os.path.join(self.testdir, oname)
848 if os.path.isfile(src):
849 shutil.copyfile(src, tdest)
850 dest = os.path.join(self.repdir, "%s.sample" % (oname))
851 shutil.copyfile(src, dest)
852 else:
853 tout.write(
854 "###Output file %s not found in testdir %s. This is normal during the first Planemo run that generates test outputs"
855 % (tdest, self.testdir)
856 )
857 tf = tarfile.open(self.newtarpath, "w:gz")
858 tf.add(
859 name=self.tooloutdir,
860 arcname=self.tool_name,
861 filter=exclude_function,
862 )
863 tf.close()
864 shutil.copyfile(self.newtarpath, self.args.new_tool)
865
866 def moveRunOutputs(self):
867 """need to move planemo or run outputs into toolfactory collection"""
868 with os.scandir(self.tooloutdir) as outs:
869 for entry in outs:
870 if not entry.is_file():
871 continue
872 if "." in entry.name:
873 nayme, ext = os.path.splitext(entry.name)
874 if ext in [".yml", ".xml", ".json", ".yaml"]:
875 ext = f"{ext}.txt"
876 else:
877 ext = ".txt"
878 ofn = "%s%s" % (entry.name.replace(".", "_"), ext)
879 dest = os.path.join(self.repdir, ofn)
880 src = os.path.join(self.tooloutdir, entry.name)
881 shutil.copyfile(src, dest)
882 with os.scandir(self.testdir) as outs:
883 for entry in outs:
884 if (
885 (not entry.is_file()) \
886 or entry.name.endswith("_sample") \
887 or entry.name.endswith("_planemo_test_report.html")
888 ):
889 continue
890 if "." in entry.name:
891 nayme, ext = os.path.splitext(entry.name)
892 else:
893 ext = ".txt"
894 newname = f"{entry.name}{ext}"
895 dest = os.path.join(self.repdir, newname)
896 src = os.path.join(self.testdir, entry.name)
897 shutil.copyfile(src, dest)
898
899 def planemo_test(self, genoutputs=True):
900 """planemo is a requirement so is available for testing but needs a
901 different call if in the biocontainer - see above
902 and for generating test outputs if command or test overrides are
903 supplied test outputs are sent to repdir for display
904 """
905 xreal = "%s.xml" % self.tool_name
906 tool_test_path = os.path.join(
907 self.repdir, f"{self.tool_name}_planemo_test_report.html"
908 )
909 if os.path.exists(self.tlog):
910 tout = open(self.tlog, "a")
911 else:
912 tout = open(self.tlog, "w")
913 if genoutputs:
914 dummy, tfile = tempfile.mkstemp()
915 cll = [
916 "planemo",
917 "test",
918 "--test_data",
919 os.path.abspath(self.testdir),
920 "--test_output",
921 os.path.abspath(tool_test_path),
922 "--skip_venv",
923 "--galaxy_root",
924 self.args.galaxy_root,
925 "--update_test_data",
926 os.path.abspath(xreal),
927 ]
928 p = subprocess.run(
929 cll,
930 env=self.ourenv,
931 shell=False,
932 cwd=self.tooloutdir,
933 stderr=dummy,
934 stdout=dummy,
935 )
936
937 else:
938 cll = [
939 "planemo",
940 "test",
941 "--test_data",
942 os.path.abspath(self.testdir),
943 "--test_output",
944 os.path.abspath(tool_test_path),
945 "--skip_venv",
946 "--galaxy_root",
947 self.args.galaxy_root,
948 os.path.abspath(xreal),
949 ]
950 p = subprocess.run(
951 cll,
952 shell=False,
953 env=self.ourenv,
954 cwd=self.tooloutdir,
955 stderr=tout,
956 stdout=tout,
957 )
958 tout.close()
959 return p.returncode
960
961
962 def main():
963 """
964 This is a Galaxy wrapper.
965 It expects to be called by a special purpose tool.xml
966
967 """
968 parser = argparse.ArgumentParser()
969 a = parser.add_argument
970 a("--script_path", default=None)
971 a("--history_test", default=None)
972 a("--cl_prefix", default=None)
973 a("--sysexe", default=None)
974 a("--packages", default=None)
975 a("--tool_name", default="newtool")
976 a("--tool_dir", default=None)
977 a("--input_files", default=[], action="append")
978 a("--output_files", default=[], action="append")
979 a("--user_email", default="Unknown")
980 a("--bad_user", default=None)
981 a("--make_Tool", default="runonly")
982 a("--help_text", default=None)
983 a("--tool_desc", default=None)
984 a("--tool_version", default=None)
985 a("--citations", default=None)
986 a("--command_override", default=None)
987 a("--test_override", default=None)
988 a("--additional_parameters", action="append", default=[])
989 a("--edit_additional_parameters", action="store_true", default=False)
990 a("--parampass", default="positional")
991 a("--tfout", default="./tfout")
992 a("--new_tool", default="new_tool")
993 a("--galaxy_url", default="http://localhost:8080")
994 a("--toolshed_url", default="http://localhost:9009")
995 # make sure this is identical to tool_sheds_conf.xml
996 # localhost != 127.0.0.1 so validation fails
997 a("--toolshed_api_key", default="fakekey")
998 a("--galaxy_api_key", default="fakekey")
999 a("--galaxy_root", default="/galaxy-central")
1000 a("--galaxy_venv", default="/galaxy_venv")
1001 args = parser.parse_args()
1002 assert not args.bad_user, (
1003 'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy \
1004 admin adds %s to "admin_users" in the galaxy.yml Galaxy configuration file'
1005 % (args.bad_user, args.bad_user)
1006 )
1007 assert args.tool_name, "## Tool Factory expects a tool name - eg --tool_name=DESeq"
1008 assert (
1009 args.sysexe or args.packages
1010 ), "## Tool Factory wrapper expects an interpreter \
1011 or an executable package in --sysexe or --packages"
1012 args.input_files = [x.replace('"', "").replace("'", "") for x in args.input_files]
1013 # remove quotes we need to deal with spaces in CL params
1014 for i, x in enumerate(args.additional_parameters):
1015 args.additional_parameters[i] = args.additional_parameters[i].replace('"', "")
1016 r = ScriptRunner(args)
1017 r.writeShedyml()
1018 r.makeTool()
1019 if args.make_Tool == "generate":
1020 retcode = r.run()
1021 r.moveRunOutputs()
1022 r.makeToolTar()
1023 else:
1024 retcode = r.planemo_test(genoutputs=True) # this fails :( - see PR
1025 r.moveRunOutputs()
1026 r.makeToolTar()
1027 retcode = r.planemo_test(genoutputs=False)
1028 r.moveRunOutputs()
1029 r.makeToolTar()
1030 print(f"second planemo_test returned {retcode}")
1031 if args.make_Tool == "gentestinstall":
1032 r.shedLoad()
1033 r.eph_galaxy_load()
1034
1035
1036 if __name__ == "__main__":
1037 main()