Mercurial > repos > fubar_too > toolfactory
comparison toolfactory/test-data/input1_sample @ 0:03df06784e56 draft
Uploaded
author | fubar_too |
---|---|
date | Tue, 18 May 2021 08:42:29 +0000 |
parents | |
children |
comparison
equal
deleted
inserted
replaced
-1:000000000000 | 0:03df06784e56 |
---|---|
1 # see https://github.com/fubar2/toolfactory | |
2 # | |
3 # copyright ross lazarus (ross stop lazarus at gmail stop com) May 2012 | |
4 # | |
5 # all rights reserved | |
6 # Licensed under the LGPL | |
7 # suggestions for improvement and bug fixes welcome at | |
8 # https://github.com/fubar2/toolfactory | |
9 # | |
10 # April 2021: Refactored into two tools - generate and test/install | |
11 # as part of GTN tutorial development and biocontainer adoption | |
12 # The tester runs planemo on a non-tested archive, creates the test outputs | |
13 # and returns a new proper tool with test. | |
14 | |
15 | |
16 | |
17 import argparse | |
18 import copy | |
19 import fcntl | |
20 import json | |
21 import os | |
22 import re | |
23 import shlex | |
24 import shutil | |
25 import subprocess | |
26 import sys | |
27 import tarfile | |
28 import tempfile | |
29 import time | |
30 | |
31 from bioblend import galaxy | |
32 | |
33 import galaxyxml.tool as gxt | |
34 import galaxyxml.tool.parameters as gxtp | |
35 | |
36 import lxml.etree as ET | |
37 | |
38 import yaml | |
39 | |
40 myversion = "V2.3 April 2021" | |
41 verbose = True | |
42 debug = True | |
43 toolFactoryURL = "https://github.com/fubar2/toolfactory" | |
44 FAKEEXE = "~~~REMOVE~~~ME~~~" | |
45 # need this until a PR/version bump to fix galaxyxml prepending the exe even | |
46 # with override. | |
47 | |
48 | |
49 def timenow(): | |
50 """return current time as a string""" | |
51 return time.strftime("%d/%m/%Y %H:%M:%S", time.localtime(time.time())) | |
52 | |
53 | |
54 cheetah_escape_table = {"$": "\\$", "#": "\\#"} | |
55 | |
56 | |
57 def cheetah_escape(text): | |
58 """Produce entities within text.""" | |
59 return "".join([cheetah_escape_table.get(c, c) for c in text]) | |
60 | |
61 | |
62 def parse_citations(citations_text): | |
63 """""" | |
64 citations = [c for c in citations_text.split("**ENTRY**") if c.strip()] | |
65 citation_tuples = [] | |
66 for citation in citations: | |
67 if citation.startswith("doi"): | |
68 citation_tuples.append(("doi", citation[len("doi") :].strip())) | |
69 else: | |
70 citation_tuples.append(("bibtex", citation[len("bibtex") :].strip())) | |
71 return citation_tuples | |
72 | |
73 class Locker: | |
74 """ | |
75 multiple instances of the TF may try to update tool_conf.xml so use a simple lockfile | |
76 to prevent overwriting mix ups. | |
77 """ | |
78 def __enter__ (self): | |
79 lockfile = "/tmp/.toolfactory_lockfile.lck" | |
80 if not os.path.exists(lockfile): | |
81 try: | |
82 os.utime(lockfile, None) | |
83 except OSError: | |
84 open(lockfile, 'a').close() | |
85 self.fp = open(lockfile) | |
86 fcntl.flock(self.fp.fileno(), fcntl.LOCK_EX) | |
87 | |
88 def __exit__ (self, _type, value, tb): | |
89 fcntl.flock(self.fp.fileno(), fcntl.LOCK_UN) | |
90 self.fp.close() | |
91 | |
92 | |
93 class Tool_Conf_Updater: | |
94 | |
95 """# update config/tool_conf.xml with a new tool unpacked in /tools | |
96 # requires highly insecure docker settings - like write to tool_conf.xml and to tools ! | |
97 # if in a container possibly not so courageous. | |
98 # Fine on your own laptop but security red flag for most production instances | |
99 Note potential race condition for tool_conf.xml update - uses a file lock. | |
100 """ | |
101 | |
102 def __init__( | |
103 self, args, tool_conf_path, new_tool_archive_path, new_tool_name, local_tool_dir, run_test | |
104 ): | |
105 self.args = args | |
106 self.tool_conf_path = os.path.join(args.galaxy_root, tool_conf_path) | |
107 self.tool_dir = os.path.join(args.galaxy_root, local_tool_dir,'TFtools') | |
108 self.out_section = "ToolFactory Generated Tools" | |
109 tff = tarfile.open(new_tool_archive_path, "r:*") | |
110 flist = tff.getnames() | |
111 ourdir = os.path.commonpath(flist) # eg pyrevpos | |
112 self.tool_id = ourdir # they are the same for TF tools | |
113 ourxml = [x for x in flist if x.lower().endswith(".xml")] | |
114 tff.extractall() | |
115 tff.close() | |
116 self.run_rsync(ourdir, self.tool_dir) | |
117 with Locker(): | |
118 self.update_toolconf(ourdir, ourxml) | |
119 | |
120 def run_rsync(self, srcf, dstf): | |
121 src = os.path.abspath(srcf) | |
122 dst = os.path.abspath(dstf) | |
123 if os.path.isdir(src): | |
124 cll = ["rsync", "-r", src, dst] | |
125 else: | |
126 cll = ["rsync", src, dst] | |
127 subprocess.run( | |
128 cll, | |
129 capture_output=False, | |
130 encoding="utf8", | |
131 shell=False, | |
132 ) | |
133 | |
134 def update_toolconf(self, ourdir, ourxml): # path is relative to tools | |
135 localconf = "./local_tool_conf.xml" | |
136 self.run_rsync(self.tool_conf_path, localconf) | |
137 tree = ET.parse(localconf) | |
138 root = tree.getroot() | |
139 hasTF = False | |
140 TFsection = None | |
141 for e in root.findall("section"): | |
142 if e.attrib["name"] == self.out_section: | |
143 hasTF = True | |
144 TFsection = e | |
145 if not hasTF: | |
146 TFsection = ET.Element("section", {"id":self.out_section, "name":self.out_section}) | |
147 root.insert(0, TFsection) # at the top! | |
148 our_tools = TFsection.findall("tool") | |
149 conf_tools = [x.attrib["file"] for x in our_tools] | |
150 for xml in ourxml: # may be > 1 | |
151 if xml not in conf_tools: # new | |
152 ET.SubElement(TFsection, "tool", {"file": os.path.join('TFtools', xml)}) | |
153 newconf = f"{self.tool_id}_conf" | |
154 tree.write(newconf, pretty_print=True) | |
155 self.run_rsync(newconf, self.tool_conf_path) | |
156 | |
157 | |
158 | |
159 | |
160 class Tool_Factory: | |
161 """Wrapper for an arbitrary script | |
162 uses galaxyxml | |
163 | |
164 """ | |
165 | |
166 def __init__(self, args=None): # noqa | |
167 """ | |
168 prepare command line cl for running the tool here | |
169 and prepare elements needed for galaxyxml tool generation | |
170 """ | |
171 self.ourcwd = os.getcwd() | |
172 self.collections = [] | |
173 if len(args.collection) > 0: | |
174 try: | |
175 self.collections = [ | |
176 json.loads(x) for x in args.collection if len(x.strip()) > 1 | |
177 ] | |
178 except Exception: | |
179 print( | |
180 f"--collections parameter {str(args.collection)} is malformed - should be a dictionary" | |
181 ) | |
182 try: | |
183 self.infiles = [ | |
184 json.loads(x) for x in args.input_files if len(x.strip()) > 1 | |
185 ] | |
186 except Exception: | |
187 print( | |
188 f"--input_files parameter {str(args.input_files)} is malformed - should be a dictionary" | |
189 ) | |
190 try: | |
191 self.outfiles = [ | |
192 json.loads(x) for x in args.output_files if len(x.strip()) > 1 | |
193 ] | |
194 except Exception: | |
195 print( | |
196 f"--output_files parameter {args.output_files} is malformed - should be a dictionary" | |
197 ) | |
198 try: | |
199 self.addpar = [ | |
200 json.loads(x) for x in args.additional_parameters if len(x.strip()) > 1 | |
201 ] | |
202 except Exception: | |
203 print( | |
204 f"--additional_parameters {args.additional_parameters} is malformed - should be a dictionary" | |
205 ) | |
206 try: | |
207 self.selpar = [ | |
208 json.loads(x) for x in args.selecttext_parameters if len(x.strip()) > 1 | |
209 ] | |
210 except Exception: | |
211 print( | |
212 f"--selecttext_parameters {args.selecttext_parameters} is malformed - should be a dictionary" | |
213 ) | |
214 self.args = args | |
215 self.cleanuppar() | |
216 self.lastxclredirect = None | |
217 self.xmlcl = [] | |
218 self.is_positional = self.args.parampass == "positional" | |
219 if self.args.sysexe: | |
220 if " " in self.args.sysexe: | |
221 self.executeme = self.args.sysexe.split(" ") | |
222 else: | |
223 self.executeme = [ | |
224 self.args.sysexe, | |
225 ] | |
226 else: | |
227 if self.args.packages: | |
228 self.executeme = [ | |
229 self.args.packages.split(",")[0].split(":")[0].strip(), | |
230 ] | |
231 else: | |
232 self.executeme = None | |
233 aXCL = self.xmlcl.append | |
234 assert args.parampass in [ | |
235 "0", | |
236 "argparse", | |
237 "positional", | |
238 ], 'args.parampass must be "0","positional" or "argparse"' | |
239 self.tool_name = re.sub("[^a-zA-Z0-9_]+", "", args.tool_name) | |
240 self.tool_id = self.tool_name | |
241 self.newtool = gxt.Tool( | |
242 self.tool_name, | |
243 self.tool_id, | |
244 self.args.tool_version, | |
245 self.args.tool_desc, | |
246 FAKEEXE, | |
247 ) | |
248 self.tooloutdir = "./tfout" | |
249 self.repdir = "./toolgen" | |
250 self.newtarpath = os.path.join(self.tooloutdir, "%s_not_tested.toolshed.gz" % self.tool_name) | |
251 self.testdir = os.path.join(self.tooloutdir, "test-data") | |
252 if not os.path.exists(self.tooloutdir): | |
253 os.mkdir(self.tooloutdir) | |
254 if not os.path.exists(self.testdir): | |
255 os.mkdir(self.testdir) | |
256 if not os.path.exists(self.repdir): | |
257 os.mkdir(self.repdir) | |
258 self.tinputs = gxtp.Inputs() | |
259 self.toutputs = gxtp.Outputs() | |
260 self.testparam = [] | |
261 if self.args.script_path: | |
262 self.prepScript() | |
263 if self.args.command_override: | |
264 scos = open(self.args.command_override, "r").readlines() | |
265 self.command_override = [x.rstrip() for x in scos] | |
266 else: | |
267 self.command_override = None | |
268 if self.args.test_override: | |
269 stos = open(self.args.test_override, "r").readlines() | |
270 self.test_override = [x.rstrip() for x in stos] | |
271 else: | |
272 self.test_override = None | |
273 if self.args.script_path: | |
274 for ex in self.executeme: | |
275 aXCL(ex) | |
276 aXCL("$runme") | |
277 else: | |
278 for ex in self.executeme: | |
279 aXCL(ex) | |
280 | |
281 if self.args.parampass == "0": | |
282 self.clsimple() | |
283 else: | |
284 if self.args.parampass == "positional": | |
285 self.prepclpos() | |
286 self.clpositional() | |
287 else: | |
288 self.prepargp() | |
289 self.clargparse() | |
290 | |
291 def clsimple(self): | |
292 """no parameters or repeats - uses < and > for i/o""" | |
293 aXCL = self.xmlcl.append | |
294 if len(self.infiles) > 0: | |
295 aXCL("<") | |
296 aXCL("$%s" % self.infiles[0]["infilename"]) | |
297 if len(self.outfiles) > 0: | |
298 aXCL(">") | |
299 aXCL("$%s" % self.outfiles[0]["name"]) | |
300 if self.args.cl_user_suffix: # DIY CL end | |
301 clp = shlex.split(self.args.cl_user_suffix) | |
302 for c in clp: | |
303 aXCL(c) | |
304 | |
305 def prepargp(self): | |
306 xclsuffix = [] | |
307 for i, p in enumerate(self.infiles): | |
308 nam = p["infilename"] | |
309 if p["origCL"].strip().upper() == "STDIN": | |
310 xappendme = [ | |
311 nam, | |
312 nam, | |
313 "< $%s" % nam, | |
314 ] | |
315 else: | |
316 rep = p["repeat"] == "1" | |
317 over = "" | |
318 if rep: | |
319 over = f'#for $rep in $R_{nam}:\n--{nam} "$rep.{nam}"\n#end for' | |
320 xappendme = [p["CL"], "$%s" % p["CL"], over] | |
321 xclsuffix.append(xappendme) | |
322 for i, p in enumerate(self.outfiles): | |
323 if p["origCL"].strip().upper() == "STDOUT": | |
324 self.lastxclredirect = [">", "$%s" % p["name"]] | |
325 else: | |
326 xclsuffix.append([p["name"], "$%s" % p["name"], ""]) | |
327 for p in self.addpar: | |
328 nam = p["name"] | |
329 rep = p["repeat"] == "1" | |
330 if rep: | |
331 over = f'#for $rep in $R_{nam}:\n--{nam} "$rep.{nam}"\n#end for' | |
332 else: | |
333 over = p["override"] | |
334 xclsuffix.append([p["CL"], '"$%s"' % nam, over]) | |
335 for p in self.selpar: | |
336 xclsuffix.append([p["CL"], '"$%s"' % p["name"], p["override"]]) | |
337 self.xclsuffix = xclsuffix | |
338 | |
339 def prepclpos(self): | |
340 xclsuffix = [] | |
341 for i, p in enumerate(self.infiles): | |
342 if p["origCL"].strip().upper() == "STDIN": | |
343 xappendme = [ | |
344 "999", | |
345 p["infilename"], | |
346 "< $%s" % p["infilename"], | |
347 ] | |
348 else: | |
349 xappendme = [p["CL"], "$%s" % p["infilename"], ""] | |
350 xclsuffix.append(xappendme) | |
351 for i, p in enumerate(self.outfiles): | |
352 if p["origCL"].strip().upper() == "STDOUT": | |
353 self.lastxclredirect = [">", "$%s" % p["name"]] | |
354 else: | |
355 xclsuffix.append([p["CL"], "$%s" % p["name"], ""]) | |
356 for p in self.addpar: | |
357 nam = p["name"] | |
358 rep = p["repeat"] == "1" # repeats make NO sense | |
359 if rep: | |
360 print( | |
361 f"### warning. Repeats for {nam} ignored - not permitted in positional parameter command lines!" | |
362 ) | |
363 over = p["override"] | |
364 xclsuffix.append([p["CL"], '"$%s"' % nam, over]) | |
365 for p in self.selpar: | |
366 xclsuffix.append([p["CL"], '"$%s"' % p["name"], p["override"]]) | |
367 xclsuffix.sort() | |
368 self.xclsuffix = xclsuffix | |
369 | |
370 def prepScript(self): | |
371 rx = open(self.args.script_path, "r").readlines() | |
372 rx = [x.rstrip() for x in rx] | |
373 rxcheck = [x.strip() for x in rx if x.strip() > ""] | |
374 assert len(rxcheck) > 0, "Supplied script is empty. Cannot run" | |
375 self.script = "\n".join(rx) | |
376 fhandle, self.sfile = tempfile.mkstemp( | |
377 prefix=self.tool_name, suffix="_%s" % (self.executeme[0]) | |
378 ) | |
379 tscript = open(self.sfile, "w") | |
380 tscript.write(self.script) | |
381 tscript.close() | |
382 self.spacedScript = [f" {x}" for x in rx if x.strip() > ""] | |
383 rx.insert(0, "#raw") | |
384 rx.append("#end raw") | |
385 self.escapedScript = rx | |
386 art = "%s.%s" % (self.tool_name, self.executeme[0]) | |
387 artifact = open(art, "wb") | |
388 artifact.write(bytes(self.script, "utf8")) | |
389 artifact.close() | |
390 | |
391 def cleanuppar(self): | |
392 """ positional parameters are complicated by their numeric ordinal""" | |
393 if self.args.parampass == "positional": | |
394 for i, p in enumerate(self.infiles): | |
395 assert ( | |
396 p["CL"].isdigit() or p["CL"].strip().upper() == "STDIN" | |
397 ), "Positional parameters must be ordinal integers - got %s for %s" % ( | |
398 p["CL"], | |
399 p["label"], | |
400 ) | |
401 for i, p in enumerate(self.outfiles): | |
402 assert ( | |
403 p["CL"].isdigit() or p["CL"].strip().upper() == "STDOUT" | |
404 ), "Positional parameters must be ordinal integers - got %s for %s" % ( | |
405 p["CL"], | |
406 p["name"], | |
407 ) | |
408 for i, p in enumerate(self.addpar): | |
409 assert p[ | |
410 "CL" | |
411 ].isdigit(), "Positional parameters must be ordinal integers - got %s for %s" % ( | |
412 p["CL"], | |
413 p["name"], | |
414 ) | |
415 for i, p in enumerate(self.infiles): | |
416 infp = copy.copy(p) | |
417 infp["origCL"] = infp["CL"] | |
418 if self.args.parampass in ["positional", "0"]: | |
419 infp["infilename"] = infp["label"].replace(" ", "_") | |
420 else: | |
421 infp["infilename"] = infp["CL"] | |
422 self.infiles[i] = infp | |
423 for i, p in enumerate(self.outfiles): | |
424 outfp = copy.copy(p) | |
425 outfp["origCL"] = outfp["CL"] # keep copy | |
426 self.outfiles[i] = outfp | |
427 for i, p in enumerate(self.addpar): | |
428 addp = copy.copy(p) | |
429 addp["origCL"] = addp["CL"] | |
430 self.addpar[i] = addp | |
431 | |
432 def clpositional(self): | |
433 # inputs in order then params | |
434 aXCL = self.xmlcl.append | |
435 for (k, v, koverride) in self.xclsuffix: | |
436 aXCL(v) | |
437 if self.lastxclredirect: | |
438 for cl in self.lastxclredirect: | |
439 aXCL(cl) | |
440 if self.args.cl_user_suffix: # DIY CL end | |
441 clp = shlex.split(self.args.cl_user_suffix) | |
442 for c in clp: | |
443 aXCL(c) | |
444 | |
445 def clargparse(self): | |
446 """argparse style""" | |
447 aXCL = self.xmlcl.append | |
448 # inputs then params in argparse named form | |
449 | |
450 for (k, v, koverride) in self.xclsuffix: | |
451 if koverride > "": | |
452 k = koverride | |
453 aXCL(k) | |
454 else: | |
455 if len(k.strip()) == 1: | |
456 k = "-%s" % k | |
457 else: | |
458 k = "--%s" % k | |
459 aXCL(k) | |
460 aXCL(v) | |
461 if self.lastxclredirect: | |
462 for cl in self.lastxclredirect: | |
463 aXCL(cl) | |
464 if self.args.cl_user_suffix: # DIY CL end | |
465 clp = shlex.split(self.args.cl_user_suffix) | |
466 for c in clp: | |
467 aXCL(c) | |
468 | |
469 def getNdash(self, newname): | |
470 if self.is_positional: | |
471 ndash = 0 | |
472 else: | |
473 ndash = 2 | |
474 if len(newname) < 2: | |
475 ndash = 1 | |
476 return ndash | |
477 | |
478 def doXMLparam(self): # noqa | |
479 """Add all needed elements to tool""" | |
480 for p in self.outfiles: | |
481 newname = p["name"] | |
482 newfmt = p["format"] | |
483 newcl = p["CL"] | |
484 test = p["test"] | |
485 oldcl = p["origCL"] | |
486 test = test.strip() | |
487 ndash = self.getNdash(newcl) | |
488 aparm = gxtp.OutputData( | |
489 name=newname, format=newfmt, num_dashes=ndash, label=newname | |
490 ) | |
491 aparm.positional = self.is_positional | |
492 if self.is_positional: | |
493 if oldcl.upper() == "STDOUT": | |
494 aparm.positional = 9999999 | |
495 aparm.command_line_override = "> $%s" % newname | |
496 else: | |
497 aparm.positional = int(oldcl) | |
498 aparm.command_line_override = "$%s" % newname | |
499 self.toutputs.append(aparm) | |
500 ld = None | |
501 if test.strip() > "": | |
502 if test.startswith("diff"): | |
503 c = "diff" | |
504 ld = 0 | |
505 if test.split(":")[1].isdigit: | |
506 ld = int(test.split(":")[1]) | |
507 tp = gxtp.TestOutput( | |
508 name=newname, | |
509 value="%s_sample" % newname, | |
510 compare=c, | |
511 lines_diff=ld, | |
512 ) | |
513 elif test.startswith("sim_size"): | |
514 c = "sim_size" | |
515 tn = test.split(":")[1].strip() | |
516 if tn > "": | |
517 if "." in tn: | |
518 delta = None | |
519 delta_frac = min(1.0, float(tn)) | |
520 else: | |
521 delta = int(tn) | |
522 delta_frac = None | |
523 tp = gxtp.TestOutput( | |
524 name=newname, | |
525 value="%s_sample" % newname, | |
526 compare=c, | |
527 delta=delta, | |
528 delta_frac=delta_frac, | |
529 ) | |
530 else: | |
531 c = test | |
532 tp = gxtp.TestOutput( | |
533 name=newname, | |
534 value="%s_sample" % newname, | |
535 compare=c, | |
536 ) | |
537 self.testparam.append(tp) | |
538 for p in self.infiles: | |
539 newname = p["infilename"] | |
540 newfmt = p["format"] | |
541 ndash = self.getNdash(newname) | |
542 reps = p.get("repeat", "0") == "1" | |
543 if not len(p["label"]) > 0: | |
544 alab = p["CL"] | |
545 else: | |
546 alab = p["label"] | |
547 aninput = gxtp.DataParam( | |
548 newname, | |
549 optional=False, | |
550 label=alab, | |
551 help=p["help"], | |
552 format=newfmt, | |
553 multiple=False, | |
554 num_dashes=ndash, | |
555 ) | |
556 aninput.positional = self.is_positional | |
557 if self.is_positional: | |
558 if p["origCL"].upper() == "STDIN": | |
559 aninput.positional = 9999998 | |
560 aninput.command_line_override = "> $%s" % newname | |
561 else: | |
562 aninput.positional = int(p["origCL"]) | |
563 aninput.command_line_override = "$%s" % newname | |
564 if reps: | |
565 repe = gxtp.Repeat( | |
566 name=f"R_{newname}", title=f"Add as many {alab} as needed" | |
567 ) | |
568 repe.append(aninput) | |
569 self.tinputs.append(repe) | |
570 tparm = gxtp.TestRepeat(name=f"R_{newname}") | |
571 tparm2 = gxtp.TestParam(newname, value="%s_sample" % newname) | |
572 tparm.append(tparm2) | |
573 self.testparam.append(tparm) | |
574 else: | |
575 self.tinputs.append(aninput) | |
576 tparm = gxtp.TestParam(newname, value="%s_sample" % newname) | |
577 self.testparam.append(tparm) | |
578 for p in self.addpar: | |
579 newname = p["name"] | |
580 newval = p["value"] | |
581 newlabel = p["label"] | |
582 newhelp = p["help"] | |
583 newtype = p["type"] | |
584 newcl = p["CL"] | |
585 oldcl = p["origCL"] | |
586 reps = p["repeat"] == "1" | |
587 if not len(newlabel) > 0: | |
588 newlabel = newname | |
589 ndash = self.getNdash(newname) | |
590 if newtype == "text": | |
591 aparm = gxtp.TextParam( | |
592 newname, | |
593 label=newlabel, | |
594 help=newhelp, | |
595 value=newval, | |
596 num_dashes=ndash, | |
597 ) | |
598 elif newtype == "integer": | |
599 aparm = gxtp.IntegerParam( | |
600 newname, | |
601 label=newlabel, | |
602 help=newhelp, | |
603 value=newval, | |
604 num_dashes=ndash, | |
605 ) | |
606 elif newtype == "float": | |
607 aparm = gxtp.FloatParam( | |
608 newname, | |
609 label=newlabel, | |
610 help=newhelp, | |
611 value=newval, | |
612 num_dashes=ndash, | |
613 ) | |
614 elif newtype == "boolean": | |
615 aparm = gxtp.BooleanParam( | |
616 newname, | |
617 label=newlabel, | |
618 help=newhelp, | |
619 value=newval, | |
620 num_dashes=ndash, | |
621 ) | |
622 else: | |
623 raise ValueError( | |
624 'Unrecognised parameter type "%s" for\ | |
625 additional parameter %s in makeXML' | |
626 % (newtype, newname) | |
627 ) | |
628 aparm.positional = self.is_positional | |
629 if self.is_positional: | |
630 aparm.positional = int(oldcl) | |
631 if reps: | |
632 repe = gxtp.Repeat( | |
633 name=f"R_{newname}", title=f"Add as many {newlabel} as needed" | |
634 ) | |
635 repe.append(aparm) | |
636 self.tinputs.append(repe) | |
637 tparm = gxtp.TestRepeat(name=f"R_{newname}") | |
638 tparm2 = gxtp.TestParam(newname, value=newval) | |
639 tparm.append(tparm2) | |
640 self.testparam.append(tparm) | |
641 else: | |
642 self.tinputs.append(aparm) | |
643 tparm = gxtp.TestParam(newname, value=newval) | |
644 self.testparam.append(tparm) | |
645 for p in self.selpar: | |
646 newname = p["name"] | |
647 newval = p["value"] | |
648 newlabel = p["label"] | |
649 newhelp = p["help"] | |
650 newtype = p["type"] | |
651 newcl = p["CL"] | |
652 if not len(newlabel) > 0: | |
653 newlabel = newname | |
654 ndash = self.getNdash(newname) | |
655 if newtype == "selecttext": | |
656 newtext = p["texts"] | |
657 aparm = gxtp.SelectParam( | |
658 newname, | |
659 label=newlabel, | |
660 help=newhelp, | |
661 num_dashes=ndash, | |
662 ) | |
663 for i in range(len(newval)): | |
664 anopt = gxtp.SelectOption( | |
665 value=newval[i], | |
666 text=newtext[i], | |
667 ) | |
668 aparm.append(anopt) | |
669 aparm.positional = self.is_positional | |
670 if self.is_positional: | |
671 aparm.positional = int(newcl) | |
672 self.tinputs.append(aparm) | |
673 tparm = gxtp.TestParam(newname, value=newval) | |
674 self.testparam.append(tparm) | |
675 else: | |
676 raise ValueError( | |
677 'Unrecognised parameter type "%s" for\ | |
678 selecttext parameter %s in makeXML' | |
679 % (newtype, newname) | |
680 ) | |
681 for p in self.collections: | |
682 newkind = p["kind"] | |
683 newname = p["name"] | |
684 newlabel = p["label"] | |
685 newdisc = p["discover"] | |
686 collect = gxtp.OutputCollection(newname, label=newlabel, type=newkind) | |
687 disc = gxtp.DiscoverDatasets( | |
688 pattern=newdisc, directory=f"{newname}", visible="false" | |
689 ) | |
690 collect.append(disc) | |
691 self.toutputs.append(collect) | |
692 try: | |
693 tparm = gxtp.TestOutputCollection(newname) # broken until PR merged. | |
694 self.testparam.append(tparm) | |
695 except Exception: | |
696 print( | |
697 "#### WARNING: Galaxyxml version does not have the PR merged yet - tests for collections must be over-ridden until then!" | |
698 ) | |
699 | |
700 def doNoXMLparam(self): | |
701 """filter style package - stdin to stdout""" | |
702 if len(self.infiles) > 0: | |
703 alab = self.infiles[0]["label"] | |
704 if len(alab) == 0: | |
705 alab = self.infiles[0]["infilename"] | |
706 max1s = ( | |
707 "Maximum one input if parampass is 0 but multiple input files supplied - %s" | |
708 % str(self.infiles) | |
709 ) | |
710 assert len(self.infiles) == 1, max1s | |
711 newname = self.infiles[0]["infilename"] | |
712 aninput = gxtp.DataParam( | |
713 newname, | |
714 optional=False, | |
715 label=alab, | |
716 help=self.infiles[0]["help"], | |
717 format=self.infiles[0]["format"], | |
718 multiple=False, | |
719 num_dashes=0, | |
720 ) | |
721 aninput.command_line_override = "< $%s" % newname | |
722 aninput.positional = True | |
723 self.tinputs.append(aninput) | |
724 tp = gxtp.TestParam(name=newname, value="%s_sample" % newname) | |
725 self.testparam.append(tp) | |
726 if len(self.outfiles) > 0: | |
727 newname = self.outfiles[0]["name"] | |
728 newfmt = self.outfiles[0]["format"] | |
729 anout = gxtp.OutputData(newname, format=newfmt, num_dashes=0) | |
730 anout.command_line_override = "> $%s" % newname | |
731 anout.positional = self.is_positional | |
732 self.toutputs.append(anout) | |
733 tp = gxtp.TestOutput(name=newname, value="%s_sample" % newname) | |
734 self.testparam.append(tp) | |
735 | |
736 def makeXML(self): # noqa | |
737 """ | |
738 Create a Galaxy xml tool wrapper for the new script | |
739 Uses galaxyhtml | |
740 Hmmm. How to get the command line into correct order... | |
741 """ | |
742 if self.command_override: | |
743 self.newtool.command_override = self.command_override # config file | |
744 else: | |
745 self.newtool.command_override = self.xmlcl | |
746 cite = gxtp.Citations() | |
747 acite = gxtp.Citation(type="doi", value="10.1093/bioinformatics/bts573") | |
748 cite.append(acite) | |
749 self.newtool.citations = cite | |
750 safertext = "" | |
751 if self.args.help_text: | |
752 helptext = open(self.args.help_text, "r").readlines() | |
753 safertext = "\n".join([cheetah_escape(x) for x in helptext]) | |
754 if len(safertext.strip()) == 0: | |
755 safertext = ( | |
756 "Ask the tool author (%s) to rebuild with help text please\n" | |
757 % (self.args.user_email) | |
758 ) | |
759 if self.args.script_path: | |
760 if len(safertext) > 0: | |
761 safertext = safertext + "\n\n------\n" # transition allowed! | |
762 scr = [x for x in self.spacedScript if x.strip() > ""] | |
763 scr.insert(0, "\n\nScript::\n") | |
764 if len(scr) > 300: | |
765 scr = ( | |
766 scr[:100] | |
767 + [" >300 lines - stuff deleted", " ......"] | |
768 + scr[-100:] | |
769 ) | |
770 scr.append("\n") | |
771 safertext = safertext + "\n".join(scr) | |
772 self.newtool.help = safertext | |
773 self.newtool.version_command = f'echo "{self.args.tool_version}"' | |
774 std = gxtp.Stdios() | |
775 std1 = gxtp.Stdio() | |
776 std.append(std1) | |
777 self.newtool.stdios = std | |
778 requirements = gxtp.Requirements() | |
779 if self.args.packages: | |
780 try: | |
781 for d in self.args.packages.split(","): | |
782 ver = None | |
783 packg = None | |
784 d = d.replace("==", ":") | |
785 d = d.replace("=", ":") | |
786 if ":" in d: | |
787 packg, ver = d.split(":") | |
788 ver = ver.strip() | |
789 packg = packg.strip() | |
790 else: | |
791 packg = d.strip() | |
792 ver = None | |
793 if ver == "": | |
794 ver = None | |
795 if packg: | |
796 requirements.append( | |
797 gxtp.Requirement("package", packg.strip(), ver) | |
798 ) | |
799 except Exception: | |
800 print( | |
801 "### malformed packages string supplied - cannot parse =", | |
802 self.args.packages, | |
803 ) | |
804 sys.exit(2) | |
805 self.newtool.requirements = requirements | |
806 if self.args.parampass == "0": | |
807 self.doNoXMLparam() | |
808 else: | |
809 self.doXMLparam() | |
810 self.newtool.outputs = self.toutputs | |
811 self.newtool.inputs = self.tinputs | |
812 if self.args.script_path: | |
813 configfiles = gxtp.Configfiles() | |
814 configfiles.append( | |
815 gxtp.Configfile(name="runme", text="\n".join(self.escapedScript)) | |
816 ) | |
817 self.newtool.configfiles = configfiles | |
818 tests = gxtp.Tests() | |
819 test_a = gxtp.Test() | |
820 for tp in self.testparam: | |
821 test_a.append(tp) | |
822 tests.append(test_a) | |
823 self.newtool.tests = tests | |
824 self.newtool.add_comment( | |
825 "Created by %s at %s using the Galaxy Tool Factory." | |
826 % (self.args.user_email, timenow()) | |
827 ) | |
828 self.newtool.add_comment("Source in git at: %s" % (toolFactoryURL)) | |
829 exml0 = self.newtool.export() | |
830 exml = exml0.replace(FAKEEXE, "") # temporary work around until PR accepted | |
831 if ( | |
832 self.test_override | |
833 ): # cannot do this inside galaxyxml as it expects lxml objects for tests | |
834 part1 = exml.split("<tests>")[0] | |
835 part2 = exml.split("</tests>")[1] | |
836 fixed = "%s\n%s\n%s" % (part1, "\n".join(self.test_override), part2) | |
837 exml = fixed | |
838 # exml = exml.replace('range="1:"', 'range="1000:"') | |
839 with open("%s.xml" % self.tool_name, "w") as xf: | |
840 xf.write(exml) | |
841 xf.write("\n") | |
842 with open(self.args.untested_tool_out, 'w') as outf: | |
843 outf.write(exml) | |
844 outf.write('\n') | |
845 # ready for the tarball | |
846 | |
847 def writeShedyml(self): | |
848 """for planemo""" | |
849 yuser = self.args.user_email.split("@")[0] | |
850 yfname = os.path.join(self.tooloutdir, ".shed.yml") | |
851 yamlf = open(yfname, "w") | |
852 odict = { | |
853 "name": self.tool_name, | |
854 "owner": yuser, | |
855 "type": "unrestricted", | |
856 "description": self.args.tool_desc, | |
857 "synopsis": self.args.tool_desc, | |
858 "category": "TF Generated Tools", | |
859 } | |
860 yaml.dump(odict, yamlf, allow_unicode=True) | |
861 yamlf.close() | |
862 | |
863 def makeTool(self): | |
864 """write xmls and input samples into place""" | |
865 if self.args.parampass == 0: | |
866 self.doNoXMLparam() | |
867 else: | |
868 self.makeXML() | |
869 if self.args.script_path: | |
870 stname = os.path.join(self.tooloutdir, self.sfile) | |
871 if not os.path.exists(stname): | |
872 shutil.copyfile(self.sfile, stname) | |
873 xreal = "%s.xml" % self.tool_name | |
874 xout = os.path.join(self.tooloutdir, xreal) | |
875 shutil.copyfile(xreal, xout) | |
876 #xout = os.path.join(self.repdir, xreal) | |
877 #shutil.copyfile(xreal, xout) | |
878 for p in self.infiles: | |
879 pth = p["name"] | |
880 dest = os.path.join(self.testdir, "%s_sample" % p["infilename"]) | |
881 shutil.copyfile(pth, dest) | |
882 dest = os.path.join( | |
883 self.repdir, "%s_sample.%s" % (p["infilename"], p["format"]) | |
884 ) | |
885 shutil.copyfile(pth, dest) | |
886 | |
887 def makeToolTar(self, report_fail=False): | |
888 """move outputs into test-data and prepare the tarball""" | |
889 excludeme = "_planemo_test_report.html" | |
890 | |
891 def exclude_function(tarinfo): | |
892 filename = tarinfo.name | |
893 return None if filename.endswith(excludeme) else tarinfo | |
894 | |
895 for p in self.outfiles: | |
896 oname = p["name"] | |
897 tdest = os.path.join(self.testdir, "%s_sample" % oname) | |
898 src = os.path.join(self.testdir, oname) | |
899 if not os.path.isfile(tdest): | |
900 if os.path.isfile(src): | |
901 shutil.copyfile(src, tdest) | |
902 dest = os.path.join(self.repdir, "%s.sample.%s" % (oname,p['format'])) | |
903 shutil.copyfile(src, dest) | |
904 else: | |
905 if report_fail: | |
906 print( | |
907 "###Tool may have failed - output file %s not found in testdir after planemo run %s." | |
908 % (tdest, self.testdir) | |
909 ) | |
910 tf = tarfile.open(self.newtarpath, "w:gz") | |
911 tf.add( | |
912 name=self.tooloutdir, | |
913 arcname=self.tool_name, | |
914 filter=exclude_function, | |
915 ) | |
916 shutil.copy(self.newtarpath, os.path.join(self.tooloutdir, f"{self.tool_name}_untested.toolshed.gz")) | |
917 tf.close() | |
918 | |
919 | |
920 def main(): | |
921 """ | |
922 This is a Galaxy wrapper. | |
923 It expects to be called by a special purpose tool.xml | |
924 | |
925 """ | |
926 parser = argparse.ArgumentParser() | |
927 a = parser.add_argument | |
928 a("--script_path", default=None) | |
929 a("--history_test", default=None) | |
930 a("--cl_user_suffix", default=None) | |
931 a("--sysexe", default=None) | |
932 a("--packages", default=None) | |
933 a("--tool_name", default="newtool") | |
934 a("--tool_dir", default=None) | |
935 a("--input_files", default=[], action="append") | |
936 a("--output_files", default=[], action="append") | |
937 a("--user_email", default="Unknown") | |
938 a("--bad_user", default=None) | |
939 a("--help_text", default=None) | |
940 a("--tool_desc", default=None) | |
941 a("--tool_version", default=None) | |
942 a("--citations", default=None) | |
943 a("--command_override", default=None) | |
944 a("--test_override", default=None) | |
945 a("--additional_parameters", action="append", default=[]) | |
946 a("--selecttext_parameters", action="append", default=[]) | |
947 a("--edit_additional_parameters", action="store_true", default=False) | |
948 a("--parampass", default="positional") | |
949 a("--tfout", default="./tfout") | |
950 a("--galaxy_root", default="/galaxy-central") | |
951 a("--galaxy_venv", default="/galaxy_venv") | |
952 a("--collection", action="append", default=[]) | |
953 a("--include_tests", default=False, action="store_true") | |
954 a("--install", default="1") | |
955 a("--admin_only", default=False, action="store_true") | |
956 a("--untested_tool_out", default=None) | |
957 a("--local_tools", default="tools") # relative to $__root_dir__ | |
958 a("--tool_conf_path", default="config/tool_conf.xml") # relative to $__root_dir__ | |
959 args = parser.parse_args() | |
960 if args.admin_only: | |
961 assert not args.bad_user, ( | |
962 'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy \ | |
963 admin adds %s to "admin_users" in the galaxy.yml Galaxy configuration file' | |
964 % (args.bad_user, args.bad_user) | |
965 ) | |
966 assert args.tool_name, "## Tool Factory expects a tool name - eg --tool_name=DESeq" | |
967 r = Tool_Factory(args) | |
968 r.writeShedyml() | |
969 r.makeTool() | |
970 r.makeToolTar() | |
971 if args.install == "1": | |
972 TCU = Tool_Conf_Updater( | |
973 args=args, | |
974 local_tool_dir=args.local_tools, | |
975 new_tool_archive_path=r.newtarpath, | |
976 tool_conf_path=args.tool_conf_path, | |
977 new_tool_name=r.tool_name, | |
978 run_test = args.run_test | |
979 ) | |
980 | |
981 if __name__ == "__main__": | |
982 main() |