Mercurial > repos > fubar > tool_factory_2

comparison rgToolFactory2.py @ 15:dd6cf2ddaac7 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Uploaded
author fubar
date Wed, 28 Jan 2015 19:28:32 -0500
parents 3635f4518c4d
children
comparison
equal deleted inserted replaced
14:3635f4518c4d 15:dd6cf2ddaac7
6 # all rights reserved 6 # all rights reserved
7 # Licensed under the LGPL 7 # Licensed under the LGPL
8 # suggestions for improvement and bug fixes welcome at https://bitbucket.org/fubar/galaxytoolfactory/wiki/Home 8 # suggestions for improvement and bug fixes welcome at https://bitbucket.org/fubar/galaxytoolfactory/wiki/Home
9 # 9 #
10 # January 2015 10 # January 2015
11 # unified all setups by passing the script on the cl rather than via a PIPE - no need for treat_bash_special so removed
12 #
11 # in the process of building a complex tool 13 # in the process of building a complex tool
12 # added ability to choose one of the current toolshed package_r or package_perl or package_python dependencies and source that package 14 # added ability to choose one of the current toolshed package_r or package_perl or package_python dependencies and source that package
13 # add that package to tool_dependencies 15 # add that package to tool_dependencies
14 # Note that once the generated tool is loaded, it will have that package's env.sh loaded automagically so there is no 16 # Note that once the generated tool is loaded, it will have that package's env.sh loaded automagically so there is no
15 # --envshpath in the parameters for the generated tool and it uses the system one which will be first on the adjusted path. 17 # --envshpath in the parameters for the generated tool and it uses the system one which will be first on the adjusted path.
118 def timenow(): 120 def timenow():
119 """return current time as a string 121 """return current time as a string
120 """ 122 """
121 return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time())) 123 return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time()))
122 124
125 def quote_non_numeric(s):
126 """return a prequoted string for non-numerics
127 useful for perl and Rscript parameter passing?
128 """
129 try:
130 res = float(s)
131 return s
132 except ValueError:
133 return '"%s"' % s
134
123 html_escape_table = { 135 html_escape_table = {
124 "&": "&", 136 "&": "&",
125 ">": ">", 137 ">": ">",
126 "<": "&lt;", 138 "<": "&lt;",
127 "$": "\$" 139 "$": "\$"
152 else: 164 else:
153 citation_tuples.append( ("bibtex", citation[len("bibtex"):].strip() ) ) 165 citation_tuples.append( ("bibtex", citation[len("bibtex"):].strip() ) )
154 return citation_tuples 166 return citation_tuples
155 167
156 def shell_source(script): 168 def shell_source(script):
157 """need a way to source a Galaxy tool interpreter env.sh so we can use that dependency 169 """need a way to source a Galaxy tool interpreter env.sh to point at the right dependency package
158 package 170 This based on the idea in http://pythonwise.blogspot.fr/2010/04/sourcing-shell-script.html
159 see http://pythonwise.blogspot.fr/2010/04/sourcing-shell-script.html 171 Note that we have to finesse any wierdly quoted newlines in automagic exports using nulls (env -0) as newlines"""
160 Sometime you want to emulate the action of "source" in bash,
161 settings some environment variables. Here is a way to do it.
162 Note that we have to finesse the automagic exports using nulls as newlines for env"""
163 pipe = subprocess.Popen("env -i ; . %s ; env -0" % script, stdout=subprocess.PIPE, shell=True) 172 pipe = subprocess.Popen("env -i ; . %s ; env -0" % script, stdout=subprocess.PIPE, shell=True)
164 output = pipe.communicate()[0] 173 output = pipe.communicate()[0]
165 outl = output.split('\0') 174 outl = output.split('\0')
166 outl = [x for x in outl if len(x.split("=")) == 2] 175 outl = [x for x in outl if len(x.split("=")) == 2]
167 newenv = dict((line.split("=", 1) for line in outl)) 176 newenv = dict((line.split("=", 1) for line in outl))
174 little overhead... 183 little overhead...
175 184
176 """ 185 """
177 186
178 187
179 def __init__(self,opts=None,treatbashSpecial=True): 188 def __init__(self,opts=None):
180 """ 189 """
181 cleanup inputs, setup some outputs 190 cleanup inputs, setup some outputs
182 191
183 """ 192 """
184 193
311 320
312 self.useGM = cmd_exists('gm') 321 self.useGM = cmd_exists('gm')
313 self.useIM = cmd_exists('convert') 322 self.useIM = cmd_exists('convert')
314 self.useGS = cmd_exists('gs') 323 self.useGS = cmd_exists('gs')
315 self.temp_warned = False # we want only one warning if $TMP not set 324 self.temp_warned = False # we want only one warning if $TMP not set
316 self.treatbashSpecial = treatbashSpecial
317 if opts.output_dir: # simplify for the tool tarball 325 if opts.output_dir: # simplify for the tool tarball
318 os.chdir(opts.output_dir) 326 os.chdir(opts.output_dir)
319 self.thumbformat = 'png' 327 self.thumbformat = 'png'
320 self.opts = opts 328 self.opts = opts
321 self.toolname = re.sub('[^a-zA-Z0-9_]+', '', opts.tool_name) # a sanitizer now does this but.. 329 self.toolname = re.sub('[^a-zA-Z0-9_]+', '', opts.tool_name) # a sanitizer now does this but..
378 psplit = p.split(',') 386 psplit = p.split(',')
379 param = html_unescape(psplit[0]) 387 param = html_unescape(psplit[0])
380 value = html_unescape(psplit[1]) 388 value = html_unescape(psplit[1])
381 a('%s="%s"' % (param,value)) 389 a('%s="%s"' % (param,value))
382 if (self.opts.interpreter == 'Rscript'): 390 if (self.opts.interpreter == 'Rscript'):
383 # pass params on command line 391 # pass params on command line as expressions which the script evaluates - see sample
384 if self.opts.input_tab: 392 if self.opts.input_tab:
385 a('INPATHS="%s"' % self.infile_paths) 393 a('INPATHS="%s"' % self.infile_paths)
386 a('INNAMES="%s"' % self.infile_names) 394 a('INNAMES="%s"' % self.infile_names)
387 if self.opts.output_tab: 395 if self.opts.output_tab:
388 a('OUTPATH="%s"' % self.opts.output_tab) 396 a('OUTPATH="%s"' % self.opts.output_tab)
389 for p in opts.additional_parameters: 397 for p in opts.additional_parameters:
390 p = p.replace('"','') 398 p = p.replace('"','')
391 psplit = p.split(',') 399 psplit = p.split(',')
392 param = html_unescape(psplit[0]) 400 param = html_unescape(psplit[0])
393 value = html_unescape(psplit[1]) 401 value = html_unescape(psplit[1])
394 a('%s="%s"' % (param,value)) 402 a('%s=%s' % (param,quote_non_numeric(value)))
395 if (self.opts.interpreter == 'perl'): 403 if (self.opts.interpreter == 'perl'):
396 # pass params on command line 404 # pass positional params on command line - perl script needs to discombobulate the path/name lists
397 if self.opts.input_tab: 405 if self.opts.input_tab:
398 a('%s' % self.infile_paths) 406 a('%s' % self.infile_paths)
399 a('%s' % self.infile_names) 407 a('%s' % self.infile_names)
400 if self.opts.output_tab: 408 if self.opts.output_tab:
401 a('%s' % self.opts.output_tab) 409 a('%s' % self.opts.output_tab)
402 for p in opts.additional_parameters: 410 for p in opts.additional_parameters:
411 # followed by any additional name=value parameter pairs
403 p = p.replace('"','') 412 p = p.replace('"','')
404 psplit = p.split(',') 413 psplit = p.split(',')
405 param = html_unescape(psplit[0]) 414 param = html_unescape(psplit[0])
406 value = html_unescape(psplit[1]) 415 value = html_unescape(psplit[1])
407 if (value.find(' ') <> -1): 416 a('%s=%s' % (param,quote_non_numeric(value)))
408 a('%s="%s"' % (param,value))
409 else:
410 a('%s=%s' % (param,value))
411 if self.opts.interpreter == 'sh' or self.opts.interpreter == 'bash': 417 if self.opts.interpreter == 'sh' or self.opts.interpreter == 'bash':
412 # more is better - now move all params into environment AND drop on to command line. 418 # more is better - now move all params into environment AND drop on to command line.
413 self.cl.insert(0,'env') 419 self.cl.insert(0,'env')
414 if self.opts.input_tab: 420 if self.opts.input_tab:
415 self.cl.insert(1,'INPATHS=%s' % (self.infile_paths)) 421 self.cl.insert(1,'INPATHS=%s' % (self.infile_paths))
421 # additional params appear in CL - yes, it's confusing 427 # additional params appear in CL - yes, it's confusing
422 for i,p in enumerate(opts.additional_parameters): 428 for i,p in enumerate(opts.additional_parameters):
423 psplit = p.split(',') 429 psplit = p.split(',')
424 param = html_unescape(psplit[0]) 430 param = html_unescape(psplit[0])
425 value = html_unescape(psplit[1]) 431 value = html_unescape(psplit[1])
426 if (value.find(' ') <> -1): 432 a('%s=%s' % (param,quote_non_numeric(value)))
427 a('%s="%s"' % (param,value)) 433 self.cl.insert(4+i,'%s=%s' % (param,quote_non_numeric(value)))
428 self.cl.insert(4+i,'%s="%s"' % (param,value)) 434 self.interpreter_owner = 'SYSTEM'
429 else: 435 self.interpreter_pack = 'SYSTEM'
430 a('%s=%s' % (param,value)) 436 self.interpreter_name = 'SYSTEM'
431 self.cl.insert(4+i,'%s=%s' % (param,value)) 437 self.interpreter_version = 'SYSTEM'
432 self.interp_owner = None 438 self.interpreter_revision = 'SYSTEM'
433 self.interp_pack = None
434 self.interp_revision = None
435 self.interp_version = None
436 if opts.envshpath <> 'system': # need to parse out details for our tool_dependency 439 if opts.envshpath <> 'system': # need to parse out details for our tool_dependency
437 try: # fragile - depends on common naming convention as at jan 2015 = package_[interp]_v0_v1_v2... = version v0.v1.v2.. is in play 440 try: # fragile - depends on common naming convention as at jan 2015 = package_[interp]_v0_v1_v2... = version v0.v1.v2.. is in play
438 441 # this ONLY happens at tool generation by an admin - the generated tool always uses the default of system so path is from local env.sh
439 packdetails = opts.envshpath.split(os.path.sep)[-4:-1] # eg ['fubar', 'package_r_3_1_1', '63cdb9b2234c'] 442 packdetails = opts.envshpath.split(os.path.sep)[-4:-1] # eg ['fubar', 'package_r_3_1_1', '63cdb9b2234c']
440 self.interpreter_owner = packdetails[0] 443 self.interpreter_owner = packdetails[0]
441 self.interpreter_pack = packdetails[1] 444 self.interpreter_pack = packdetails[1]
442 self.interpreter_name = packdetails[1].split('_')[1].upper() 445 self.interpreter_name = packdetails[1].split('_')[1].upper()
443 self.interpreter_revision = packdetails[2] 446 self.interpreter_revision = packdetails[2]
629 if self.opts.envshpath == 'system': 632 if self.opts.envshpath == 'system':
630 tooldepcontent = self.toolhtmldepskel % readme_dict 633 tooldepcontent = self.toolhtmldepskel % readme_dict
631 else: 634 else:
632 tooldepcontent = self.toolhtmldepinterpskel % readme_dict 635 tooldepcontent = self.toolhtmldepinterpskel % readme_dict
633 else: 636 else:
634 tooldepcontent = self.emptytoolhtmldepskel % readme_dictls -l 637 tooldepcontent = self.emptytoolhtmldepskel % readme_dict
635 depf = open(os.path.join(tdir,'tool_dependencies.xml'),'w') 638 depf = open(os.path.join(tdir,'tool_dependencies.xml'),'w')
636 depf.write(tooldepcontent) 639 depf.write(tooldepcontent)
637 depf.write('\n') 640 depf.write('\n')
638 depf.close() 641 depf.close()
639 testdir = os.path.join(tdir,'test-data') 642 testdir = os.path.join(tdir,'test-data')
858 861
859 862
860 863
861 def run(self): 864 def run(self):
862 """ 865 """
863 scripts must be small enough not to fill the pipe! 866 Some devteam tools have this defensive stderr read so I'm keeping with the faith
867 Feel free to update.
864 """ 868 """
865 if self.opts.envshpath <> 'system': 869 if self.opts.envshpath <> 'system':
866 shell_source(self.opts.envshpath) 870 shell_source(self.opts.envshpath)
867 if self.treatbashSpecial and self.opts.interpreter in ['bash','sh']: 871 # this only happens at tool generation - the generated tool relies on the dependencies all being set up
868 retval = self.runBash() 872 # at toolshed installation by sourcing local env.sh
869 else: 873 if self.opts.output_dir:
870 if self.opts.output_dir: 874 ste = open(self.elog,'wb')
871 ste = open(self.elog,'w') 875 sto = open(self.tlog,'wb')
872 sto = open(self.tlog,'w') 876 s = ' '.join(self.cl)
873 sto.write('## Toolfactory generated command line = %s\n' % ' '.join(self.cl)) 877 sto.write('## Executing Toolfactory generated command line = %s\n' % s)
874 sto.flush() 878 sto.flush()
875 p = subprocess.Popen(self.cl,shell=False,stdout=sto,stderr=ste,cwd=self.opts.output_dir) 879 p = subprocess.Popen(self.cl,shell=False,stdout=sto,stderr=ste,cwd=self.opts.output_dir)
876 else:
877 p = subprocess.Popen(self.cl,shell=False)
878 retval = p.wait() 880 retval = p.wait()
879 if self.opts.output_dir: 881 sto.close()
880 sto.close() 882 ste.close()
881 ste.close() 883 tmp_stderr = open( self.elog, 'rb' )
882 err = open(self.elog,'r').readlines() 884 err = ''
883 if retval <> 0 and err: # problem 885 buffsize = 1048576
884 print >> sys.stderr,err 886 try:
885 if self.opts.make_HTML: 887 while True:
886 self.makeHtml() 888 err += tmp_stderr.read( buffsize )
887 return retval 889 if not err or len( stderr ) % buffsize != 0:
888 890 break
889 def runBash(self): 891 except OverflowError:
890 """ 892 pass
891 cannot use - for bash so use self.sfile 893 tmp_stderr.close()
892 """ 894 else:
895 p = subprocess.Popen(self.cl,shell=False)
896 retval = p.wait()
893 if self.opts.output_dir: 897 if self.opts.output_dir:
894 s = '## Toolfactory generated command line = %s\n' % ' '.join(self.cl) 898 if retval <> 0 and err: # problem
895 sto = open(self.tlog,'w') 899 print >> sys.stderr,err
896 sto.write(s)
897 sto.flush()
898 p = subprocess.Popen(self.cl,shell=False,stdout=sto,stderr=sto,cwd=self.opts.output_dir)
899 else:
900 p = subprocess.Popen(self.cl,shell=False)
901 retval = p.wait()
902 if self.opts.output_dir:
903 sto.close()
904 if self.opts.make_HTML: 900 if self.opts.make_HTML:
905 self.makeHtml() 901 self.makeHtml()
906 return retval 902 return retval
903
907 904
908 905
909 def main(): 906 def main():
910 u = """ 907 u = """
911 This is a Galaxy wrapper. It expects to be called by a special purpose tool.xml as: 908 This is a Galaxy wrapper. It expects to be called by a special purpose tool.xml as: