Mercurial > repos > fubar > tool_factory_2

comparison rgToolFactory2.py @ 10:8de2b7571d98 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Uploaded
author fubar
date Thu, 15 Jan 2015 07:35:06 -0500
parents ce5ec1d989fd
children
comparison
equal deleted inserted replaced
9:ce5ec1d989fd 10:8de2b7571d98
4 # copyright ross lazarus (ross stop lazarus at gmail stop com) May 2012 4 # copyright ross lazarus (ross stop lazarus at gmail stop com) May 2012
5 # 5 #
6 # all rights reserved 6 # all rights reserved
7 # Licensed under the LGPL 7 # Licensed under the LGPL
8 # suggestions for improvement and bug fixes welcome at https://bitbucket.org/fubar/galaxytoolfactory/wiki/Home 8 # suggestions for improvement and bug fixes welcome at https://bitbucket.org/fubar/galaxytoolfactory/wiki/Home
9 #
10 # January 2015
11 # in the process of building a complex tool
12 # added ability to choose one of the current toolshed package_r or package_perl or package_python dependencies and source that package
13 # need to add that package to tool_dependencies
9 # 14 #
10 # sept 2014 added additional params from 15 # sept 2014 added additional params from
11 # https://bitbucket.org/mvdbeek/dockertoolfactory/src/d4863bcf7b521532c7e8c61b6333840ba5393f73/DockerToolFactory.py?at=default 16 # https://bitbucket.org/mvdbeek/dockertoolfactory/src/d4863bcf7b521532c7e8c61b6333840ba5393f73/DockerToolFactory.py?at=default
12 # passing them is complex 17 # passing them is complex
13 # and they are restricted to NOT contain commas or double quotes to ensure that they can be safely passed together on 18 # and they are restricted to NOT contain commas or double quotes to ensure that they can be safely passed together on
105 debug = False 110 debug = False
106 toolFactoryURL = 'https://bitbucket.org/fubar/galaxytoolfactory' 111 toolFactoryURL = 'https://bitbucket.org/fubar/galaxytoolfactory'
107 112
108 # if we do html we need these dependencies specified in a tool_dependencies.xml file and referred to in the generated 113 # if we do html we need these dependencies specified in a tool_dependencies.xml file and referred to in the generated
109 # tool xml 114 # tool xml
110 toolhtmldepskel = """<?xml version="1.0"?>
111 <tool_dependency>
112 <package name="ghostscript" version="9.10">
113 <repository name="package_ghostscript_9_10" owner="devteam" prior_installation_required="True" />
114 </package>
115 <package name="graphicsmagick" version="1.3.18">
116 <repository name="package_graphicsmagick_1_3" owner="iuc" prior_installation_required="True" />
117 </package>
118 <readme>
119 %s
120 </readme>
121 </tool_dependency>
122 """
123
124 emptytoolhtmldepskel = """<?xml version="1.0"?>
125 <tool_dependency>
126 <readme>
127 %s
128 </readme>
129 </tool_dependency>
130 """
131
132 protorequirements = """<requirements>
133 <requirement type="package" version="9.10">ghostscript</requirement>
134 <requirement type="package" version="1.3.18">graphicsmagick</requirement>
135 </requirements>"""
136 115
137 def timenow(): 116 def timenow():
138 """return current time as a string 117 """return current time as a string
139 """ 118 """
140 return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time())) 119 return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time()))
186 newenv = dict((line.split("=", 1) for line in outl)) 165 newenv = dict((line.split("=", 1) for line in outl))
187 os.environ.update(newenv) 166 os.environ.update(newenv)
188 167
189 class ScriptRunner: 168 class ScriptRunner:
190 """class is a wrapper for an arbitrary script 169 """class is a wrapper for an arbitrary script
170 note funky templating. this should all be done proper.
171 Problem is, this kludge developed quite naturally and seems to work ok with
172 little overhead...
173
191 """ 174 """
175
192 176
193 def __init__(self,opts=None,treatbashSpecial=True): 177 def __init__(self,opts=None,treatbashSpecial=True):
194 """ 178 """
195 cleanup inputs, setup some outputs 179 cleanup inputs, setup some outputs
196 180
197 """ 181 """
182
183 self.toolhtmldepinterpskel = """<?xml version="1.0"?>
184 <tool_dependency>
185 <package name="ghostscript" version="9.10">
186 <repository name="package_ghostscript_9_10" owner="devteam" prior_installation_required="True" />
187 </package>
188 <package name="graphicsmagick" version="1.3.18">
189 <repository name="package_graphicsmagick_1_3" owner="iuc" prior_installation_required="True" />
190 </package>
191 <package name="%(interpreter)s" version="%(interpreter_version)s">
192 <repository name="%(interpreter_name)s" owner="%(interpreter_owner)s" prior_installation_required="True" />
193 </package>
194
195 <readme>
196 %(readme)s
197 </readme>
198 </tool_dependency>
199 """
200
201 self.toolhtmldepskel = """<?xml version="1.0"?>
202 <tool_dependency>
203 <package name="ghostscript" version="9.10">
204 <repository name="package_ghostscript_9_10" owner="devteam" prior_installation_required="True" />
205 </package>
206 <package name="graphicsmagick" version="1.3.18">
207 <repository name="package_graphicsmagick_1_3" owner="iuc" prior_installation_required="True" />
208 </package>
209 <readme>
210 %(readme)s
211 </readme>
212 </tool_dependency>
213 """
214
215 self.emptytoolhtmldepskel = """<?xml version="1.0"?>
216 <tool_dependency>
217 <readme>
218 %(readme)s
219 </readme>
220 </tool_dependency>
221 """
222
223 self.protorequirements = """<requirements>
224 <requirement type="package" version="9.10">ghostscript</requirement>
225 <requirement type="package" version="1.3.18">graphicsmagick</requirement>
226 </requirements>"""
227
228 self.protorequirements_interpreter = """<requirements>
229 <requirement type="package" version="9.10">ghostscript</requirement>
230 <requirement type="package" version="1.3.18">graphicsmagick</requirement>
231 <requirement type="package" version="%(interpreter_version)s">%(interpreter_name)s</requirement>
232 </requirements>"""
233
234
235 self.newCommand="""
236 %(toolname)s.py --script_path "$runMe" --interpreter "%(interpreter)s"
237 --tool_name "%(toolname)s"
238 %(command_inputs)s
239 %(command_outputs)s
240 """
241
242 self.tooltestsTabOnly = """
243 <test>
244 %(test1Inputs)s
245 <param name="job_name" value="test1"/>
246 <param name="runMe" value="$runMe"/>
247 <output name="output1="%(test1Output)s" ftype="tabular"/>
248 %(additionalParams)s
249 </test>
250 """
251
252 self.tooltestsHTMLOnly = """
253 <test>
254 %(test1Inputs)s
255 <param name="job_name" value="test1"/>
256 <param name="runMe" value="$runMe"/>
257 %(additionalParams)s
258 <output name="html_file" file="%(test1HTML)s" ftype="html" lines_diff="5"/>
259 </test>
260 """
261
262 self.tooltestsBoth = """
263 <test>
264 %(test1Inputs)s
265 <param name="job_name" value="test1"/>
266 <param name="runMe" value="$runMe"/>
267 %(additionalParams)s
268 <output name="output1" file="%(test1Output)s" ftype="tabular" />
269 <output name="html_file" file="%(test1HTML)s" ftype="html" lines_diff="10"/>
270 </test>
271 """
272
273 self.newXML="""<tool id="%(toolid)s" name="%(toolname)s" version="%(tool_version)s">
274 %(tooldesc)s
275 %(requirements)s
276 <command interpreter="python">
277 %(command)s
278 </command>
279 <inputs>
280 %(inputs)s
281 %(additionalInputs)s
282 </inputs>
283 <outputs>
284 %(outputs)s
285 </outputs>
286 <configfiles>
287 <configfile name="runMe">
288 %(script)s
289 </configfile>
290 </configfiles>
291 <tests>
292 %(tooltests)s
293 </tests>
294 <help>
295
296 %(help)s
297
298 </help>
299 <citations>
300 %(citations)s
301 <citation type="doi">10.1093/bioinformatics/bts573</citation>
302 </citations>
303 </tool>"""
304
198 self.useGM = cmd_exists('gm') 305 self.useGM = cmd_exists('gm')
199 self.useIM = cmd_exists('convert') 306 self.useIM = cmd_exists('convert')
200 self.useGS = cmd_exists('gs') 307 self.useGS = cmd_exists('gs')
201 self.temp_warned = False # we want only one warning if $TMP not set 308 self.temp_warned = False # we want only one warning if $TMP not set
202 self.treatbashSpecial = treatbashSpecial 309 self.treatbashSpecial = treatbashSpecial
313 a('%s="%s"' % (param,value)) 420 a('%s="%s"' % (param,value))
314 self.cl.insert(4+i,'%s="%s"' % (param,value)) 421 self.cl.insert(4+i,'%s="%s"' % (param,value))
315 else: 422 else:
316 a('%s=%s' % (param,value)) 423 a('%s=%s' % (param,value))
317 self.cl.insert(4+i,'%s=%s' % (param,value)) 424 self.cl.insert(4+i,'%s=%s' % (param,value))
318 425 self.interp_owner = None
319 426 self.interp_pack = None
427 self.interp_revision = None
428 self.interp_version = None
429 if opts.envshpath <> 'system': # need to parse out details for our tool_dependency
430 try:
431 packdetails = opts.envshpath.split(os.path.sep)[-4:-1] # eg ['fubar', 'package_r_3_1_1', '63cdb9b2234c']
432 self.interpreter_owner = packdetails[0]
433 self.interpreter_pack = packdetails[1]
434 self.interpreter_revision = packdetails[2]
435 self.interpreter_version = '.'.join(self.interpreter_pack.split('_')[2:])
436 # hope our naming convention as at jan 2015 = package_[interp]_v0_v1_v2... = version v0.v1.v2.. is in play
437 except:
438 pass
320 self.outFormats = opts.output_format 439 self.outFormats = opts.output_format
321 self.inputFormats = opts.input_formats 440 self.inputFormats = opts.input_formats
322 self.test1Output = '%s_test1_output.xls' % self.toolname 441 self.test1Output = '%s_test1_output.xls' % self.toolname
323 self.test1HTML = '%s_test1_output.html' % self.toolname 442 self.test1HTML = '%s_test1_output.html' % self.toolname
324 443
369 </configfile> 488 </configfile>
370 </configfiles> 489 </configfiles>
371 </tool> 490 </tool>
372 491
373 """ 492 """
374 newXML="""<tool id="%(toolid)s" name="%(toolname)s" version="%(tool_version)s"> 493
375 %(tooldesc)s 494 # these templates need a dict with the right keys to match the parameters - outputs, help, code...
376 %(requirements)s 495
377 <command interpreter="python">
378 %(command)s
379 </command>
380 <inputs>
381 %(inputs)s
382 %(additionalInputs)s
383 </inputs>
384 <outputs>
385 %(outputs)s
386 </outputs>
387 <configfiles>
388 <configfile name="runMe">
389 %(script)s
390 </configfile>
391 </configfiles>
392 <tests>
393 %(tooltests)s
394 </tests>
395 <help>
396
397 %(help)s
398
399 </help>
400 <citations>
401 %(citations)s
402 <citation type="doi">10.1093/bioinformatics/bts573</citation>
403 </citations>
404 </tool>"""
405 # needs a dict with toolname, toolid, interpreter, scriptname, command, inputs as a multi line string ready to write, outputs ditto, help ditto
406
407 newCommand="""
408 %(toolname)s.py --script_path "$runMe" --interpreter "%(interpreter)s"
409 --tool_name "%(toolname)s"
410 %(command_inputs)s
411 %(command_outputs)s
412 """
413 # may NOT be an input or htmlout - appended later
414 tooltestsTabOnly = """
415 <test>
416 %(test1Inputs)s
417 <param name="job_name" value="test1"/>
418 <param name="runMe" value="$runMe"/>
419 <output name="output1="%(test1Output)s" ftype="tabular"/>
420 %(additionalParams)s
421 </test>
422 """
423 tooltestsHTMLOnly = """
424 <test>
425 %(test1Inputs)s
426 <param name="job_name" value="test1"/>
427 <param name="runMe" value="$runMe"/>
428 %(additionalParams)s
429 <output name="html_file" file="%(test1HTML)s" ftype="html" lines_diff="5"/>
430 </test>
431 """
432 tooltestsBoth = """
433 <test>
434 %(test1Inputs)s
435 <param name="job_name" value="test1"/>
436 <param name="runMe" value="$runMe"/>
437 %(additionalParams)s
438 <output name="output1" file="%(test1Output)s" ftype="tabular" />
439 <output name="html_file" file="%(test1HTML)s" ftype="html" lines_diff="10"/>
440 </test>
441 """
442 xdict = {} 496 xdict = {}
443 xdict['additionalParams'] = '' 497 xdict['additionalParams'] = ''
444 xdict['additionalInputs'] = '' 498 xdict['additionalInputs'] = ''
445 if self.opts.additional_parameters: 499 if self.opts.additional_parameters:
446 if self.opts.edit_additional_parameters: # add to new tool form with default value set to original value 500 if self.opts.edit_additional_parameters: # add to new tool form with default value set to original value
447 xdict['additionalInputs'] = '\n'.join(['<param name="%s" value="%s" label="%s" help="%s" type="%s"/>' % \ 501 xdict['additionalInputs'] = '\n'.join(['<param name="%s" value="%s" label="%s" help="%s" type="%s"/>' % \
448 (x.split(',')[0],html_escape(x.split(',')[1]),html_escape(x.split(',')[2]),html_escape(x.split(',')[3]), x.split(',')[4]) for x in self.opts.additional_parameters]) 502 (x.split(',')[0],html_escape(x.split(',')[1]),html_escape(x.split(',')[2]),html_escape(x.split(',')[3]), x.split(',')[4]) for x in self.opts.additional_parameters])
449 xdict['additionalParams'] = '\n'.join(['<param name="%s" value="%s" />' % (x.split(',')[0],html_escape(x.split(',')[1])) for x in self.opts.additional_parameters]) 503 xdict['additionalParams'] = '\n'.join(['<param name="%s" value="%s" />' % (x.split(',')[0],html_escape(x.split(',')[1])) for x in self.opts.additional_parameters])
504 xdict['interpreter_owner'] = self.interpreter_owner
505 xdict['interpreter_version'] = self.interpreter_version
506 xdict['interpreter_name'] = self.interpreter_pack
450 xdict['requirements'] = '' 507 xdict['requirements'] = ''
451 if self.opts.make_HTML: 508 if self.opts.include_dependencies == "yes":
452 if self.opts.include_dependencies == "yes": 509 if self.opts.envshpath <> 'system':
453 xdict['requirements'] = protorequirements 510 xdict['requirements'] = self.protorequirements_interpreter % xdict
511 else:
512 xdict['requirements'] = self.protorequirements
454 xdict['tool_version'] = self.opts.tool_version 513 xdict['tool_version'] = self.opts.tool_version
455 xdict['test1HTML'] = self.test1HTML 514 xdict['test1HTML'] = self.test1HTML
456 xdict['test1Output'] = self.test1Output 515 xdict['test1Output'] = self.test1Output
457 xdict['test1Inputs'] = self.test1Inputs 516 xdict['test1Inputs'] = self.test1Inputs
458 if self.opts.make_HTML and self.opts.output_tab: 517 if self.opts.make_HTML and self.opts.output_tab:
459 xdict['tooltests'] = tooltestsBoth % xdict 518 xdict['tooltests'] = self.tooltestsBoth % xdict
460 elif self.opts.make_HTML: 519 elif self.opts.make_HTML:
461 xdict['tooltests'] = tooltestsHTMLOnly % xdict 520 xdict['tooltests'] = self.tooltestsHTMLOnly % xdict
462 else: 521 else:
463 xdict['tooltests'] = tooltestsTabOnly % xdict 522 xdict['tooltests'] = self.tooltestsTabOnly % xdict
464 xdict['script'] = self.escapedScript 523 xdict['script'] = self.escapedScript
465 # configfile is least painful way to embed script to avoid external dependencies 524 # configfile is least painful way to embed script to avoid external dependencies
466 # but requires escaping of <, > and $ to avoid Mako parsing 525 # but requires escaping of <, > and $ to avoid Mako parsing
467 if self.opts.help_text: 526 if self.opts.help_text:
468 helptext = open(self.opts.help_text,'r').readlines() 527 helptext = open(self.opts.help_text,'r').readlines()
518 else: 577 else:
519 xdict['command_outputs'] += ' --output_dir "./"' 578 xdict['command_outputs'] += ' --output_dir "./"'
520 if self.opts.output_tab: 579 if self.opts.output_tab:
521 xdict['command_outputs'] += ' --output_tab "$output1"' 580 xdict['command_outputs'] += ' --output_tab "$output1"'
522 xdict['outputs'] += ' <data format="%s" name="output1" label="${job_name}"/>\n' % self.outFormats 581 xdict['outputs'] += ' <data format="%s" name="output1" label="${job_name}"/>\n' % self.outFormats
523 xdict['command'] = newCommand % xdict 582 xdict['command'] = self.newCommand % xdict
524 if self.opts.citations: 583 if self.opts.citations:
525 citationstext = open(self.opts.citations,'r').read() 584 citationstext = open(self.opts.citations,'r').read()
526 citation_tuples = parse_citations(citationstext) 585 citation_tuples = parse_citations(citationstext)
527 citations_xml = "" 586 citations_xml = ""
528 for citation_type, citation_content in citation_tuples: 587 for citation_type, citation_content in citation_tuples:
529 citation_xml = """<citation type="%s">%s</citation>""" % (citation_type, html_escape(citation_content)) 588 citation_xml = """<citation type="%s">%s</citation>""" % (citation_type, html_escape(citation_content))
530 citations_xml += citation_xml 589 citations_xml += citation_xml
531 xdict['citations'] = citations_xml 590 xdict['citations'] = citations_xml
532 else: 591 else:
533 xdict['citations'] = "" 592 xdict['citations'] = ""
534 xmls = newXML % xdict 593 xmls = self.newXML % xdict
535 xf = open(self.xmlfile,'w') 594 xf = open(self.xmlfile,'w')
536 xf.write(xmls) 595 xf.write(xmls)
537 xf.write('\n') 596 xf.write('\n')
538 xf.close() 597 xf.close()
539 # ready for the tarball 598 # ready for the tarball
553 self.makeXML() 612 self.makeXML()
554 if self.opts.help_text: 613 if self.opts.help_text:
555 hlp = open(self.opts.help_text,'r').read() 614 hlp = open(self.opts.help_text,'r').read()
556 else: 615 else:
557 hlp = 'Please ask the tool author for help as none was supplied at tool generation\n' 616 hlp = 'Please ask the tool author for help as none was supplied at tool generation\n'
617 readme_dict = {'readme':hlp,'interpreter':self.opts.interpreter,'interpreter_version':self.interpreter_version,'interpreter_name':self.interpreter_pack,
618 'interpreter_owner':self.interpreter_owner}
558 if self.opts.include_dependencies == "yes": 619 if self.opts.include_dependencies == "yes":
559 tooldepcontent = toolhtmldepskel % hlp 620 if self.opts.envshpath == 'system':
560 else: 621 tooldepcontent = self.toolhtmldepskel % readme_dict
561 tooldepcontent = emptytoolhtmldepskel % hlp 622 else:
623 tooldepcontent = self.toolhtmldepinterpskel % readme_dict
624 else:
625 tooldepcontent = self.emptytoolhtmldepskel % readme_dictls -l
562 depf = open(os.path.join(tdir,'tool_dependencies.xml'),'w') 626 depf = open(os.path.join(tdir,'tool_dependencies.xml'),'w')
563 depf.write(tooldepcontent) 627 depf.write(tooldepcontent)
564 depf.write('\n') 628 depf.write('\n')
565 depf.close() 629 depf.close()
566 if self.opts.input_tab: # no reproducible test otherwise? TODO: maybe.. 630 if self.opts.input_tab: # no reproducible test otherwise? TODO: maybe..