|
0
|
1 #!/usr/bin/python
|
|
|
2 import sys
|
|
|
3 import os
|
|
|
4 import subprocess
|
|
|
5 import optparse as op
|
|
|
6 import xml.etree.cElementTree as et
|
|
|
7
|
|
|
8 TRACE=False
|
|
|
9 #
|
|
|
10 # Turn on tracing to dump out __input__.xml and __settings__.xml somewhere
|
|
|
11 #
|
|
|
12 #TRACE=True
|
|
|
13 #TRACE_PATH='/home/UNIXHOME/jsorenson'
|
|
|
14
|
|
|
15 class SmrtpipeGalaxy:
|
|
|
16 """Wrapper for running smrtpipe under galaxy"""
|
|
|
17 def __init__( self, argv ):
|
|
|
18 self.__parseOptions( argv )
|
|
|
19
|
|
|
20 def __parseOptions( self, argv ):
|
|
|
21 usage = 'Usage: %prog [--help] [options] smrtpipe.ini'
|
|
|
22 parser = op.OptionParser( usage=usage, description=SmrtpipeGalaxy.__doc__ )
|
|
|
23 parser.add_option( "--output",
|
|
|
24 help="Designate a file generated by smrtpipe as the expected output for galaxy" )
|
|
|
25 parser.add_option( "--nproc", type="int",
|
|
|
26 help="Number of processes to use (-D NPROC)" )
|
|
|
27 parser.add_option( "--galaxy_output",
|
|
|
28 help="File name provided by galaxy where output should be placed" )
|
|
|
29 parser.add_option( "--dry_run", action="store_true",
|
|
|
30 help="Create auxiliary XML files and exit" )
|
|
|
31 parser.add_option( "--dat_extension",
|
|
|
32 help="Soft link .dat files to have this extension (some pipelines require certain extensions)" )
|
|
|
33
|
|
|
34 parser.set_defaults( output=None, dry_run=False, galaxy_output=None,
|
|
|
35 dat_extension=None, nproc=0 )
|
|
|
36 self.options, self.args = parser.parse_args( argv )
|
|
|
37
|
|
|
38 if len(self.args)!=2:
|
|
|
39 parser.error( 'Expected 1 argument' )
|
|
|
40
|
|
|
41 self.configFile = self.args[1]
|
|
|
42
|
|
|
43 def __parseConfig( self ):
|
|
|
44 infile = open( self.configFile, 'r' )
|
|
|
45 section = None
|
|
|
46 sections = []
|
|
|
47 for line in infile:
|
|
|
48 l = line.strip()
|
|
|
49 if len(l)==0 or line.startswith('#'):
|
|
|
50 continue
|
|
|
51 if l.startswith('[') and l.endswith(']'):
|
|
|
52 section = section_factory( l[1:-1] )
|
|
|
53 sections.append(section)
|
|
|
54 continue
|
|
|
55 if section is None:
|
|
|
56 continue
|
|
|
57 if '=' in l:
|
|
|
58 section.addParameterLine(l)
|
|
|
59 else:
|
|
|
60 section.addLine(l)
|
|
|
61 infile.close()
|
|
|
62 return sections
|
|
|
63
|
|
|
64 def transferOutput( self ):
|
|
|
65 if not self.options.output or not self.options.galaxy_output:
|
|
|
66 return True, ''
|
|
|
67 if not os.path.exists(self.options.output):
|
|
|
68 return False, "Can't find file %s (job error?)" % self.options.output
|
|
|
69 os.system( 'cp %s %s' % (self.options.output, self.options.galaxy_output ))
|
|
|
70 return True, ''
|
|
|
71
|
|
|
72 def run( self ):
|
|
|
73 if not os.path.exists( self.configFile ):
|
|
|
74 print >>sys.stderr, "Can't find config file %s" % self.configFile
|
|
|
75 return 1
|
|
|
76
|
|
|
77 sections = self.__parseConfig()
|
|
|
78
|
|
|
79 if len(sections)==0:
|
|
|
80 print >>sys.stderr, "No sections found in %s" % self.configFile
|
|
|
81 return 1
|
|
|
82 if sections[0].name != 'input':
|
|
|
83 print >>sys.stderr, "No [input] section found in %s" % self.configFile
|
|
|
84 return 1
|
|
|
85
|
|
|
86 INPUT_FILE = '__input__.xml'
|
|
|
87 SETTINGS_FILE = '__settings__.xml'
|
|
|
88
|
|
|
89 sections[0].softLinkDats( self.options.dat_extension )
|
|
|
90 inputXml = sections[0].makeXmlElement()
|
|
|
91 write_xml_to_file( INPUT_FILE, inputXml )
|
|
|
92 if TRACE:
|
|
|
93 write_xml_to_file( os.path.join(TRACE_PATH,INPUT_FILE), inputXml )
|
|
|
94
|
|
|
95 settings = et.Element( 'smrtpipeSettings' )
|
|
|
96 for s in sections[1:]:
|
|
|
97 s.makeXmlElement( settings )
|
|
|
98
|
|
|
99 write_xml_to_file( SETTINGS_FILE, settings )
|
|
|
100 if TRACE:
|
|
|
101 write_xml_to_file( os.path.join(TRACE_PATH,SETTINGS_FILE), settings )
|
|
|
102
|
|
|
103 nproc = '-D NPROC=%d' % self.options.nproc if self.options.nproc>0 else ''
|
|
|
104 cmd = 'smrtpipe.py %s --params=%s xml:%s > smrtpipe.err 2>1' % \
|
|
|
105 ( nproc, SETTINGS_FILE, INPUT_FILE )
|
|
|
106
|
|
|
107 if self.options.dry_run:
|
|
|
108 print 'Command to run:'
|
|
|
109 print cmd
|
|
|
110 return 0
|
|
|
111
|
|
|
112 out, errCode, errMsg = backticks( cmd )
|
|
|
113 if errCode!=0:
|
|
|
114 print >>sys.stderr, "error while running: %s" % cmd
|
|
|
115 print >>sys.stderr, errMsg
|
|
|
116 if os.path.exists('log/smrtpipe.log'):
|
|
|
117 print >>sys.stderr, 'Log:'
|
|
|
118 infile = open('log/smrtpipe.log','r')
|
|
|
119 for line in infile: sys.stderr.write(line)
|
|
|
120 infile.close()
|
|
|
121 return errCode
|
|
|
122
|
|
|
123 success, errMsg = self.transferOutput()
|
|
|
124 if not success:
|
|
|
125 print >>sys.stderr, errMsg
|
|
|
126 return 1
|
|
|
127
|
|
|
128 return 0
|
|
|
129
|
|
|
130 def write_xml_to_file( fileName, root ):
|
|
|
131 outfile = open( fileName, 'w' )
|
|
|
132 outfile.write( '<?xml version="1.0"?>\n' )
|
|
|
133 outfile.write( et.tostring(root) + '\n' )
|
|
|
134 outfile.close()
|
|
|
135
|
|
|
136 def section_factory( name ):
|
|
|
137 if name=='input':
|
|
|
138 return InputSection(name)
|
|
|
139 else:
|
|
|
140 return Section(name)
|
|
|
141
|
|
|
142 class Section:
|
|
|
143 def __init__( self, name ):
|
|
|
144 self._name = name
|
|
|
145 self._lines = []
|
|
|
146 self._vars = {}
|
|
|
147
|
|
|
148 @property
|
|
|
149 def name(self):
|
|
|
150 return self._name
|
|
|
151
|
|
|
152 def addLine( self, line ):
|
|
|
153 self._lines.append(line)
|
|
|
154
|
|
|
155 def addParameterLine( self, line ):
|
|
|
156 self.addLine(line)
|
|
|
157 i = line.find( '=' )
|
|
|
158 key = line[:i].strip()
|
|
|
159 value = line[i+1:].strip()
|
|
|
160 self._vars[key] = value
|
|
|
161
|
|
|
162 def makeXmlElement( self, settings ):
|
|
|
163 if self._name=='global':
|
|
|
164 root = et.SubElement( settings, "protocol", {'name':'generic'} )
|
|
|
165 else:
|
|
|
166 root = et.SubElement( settings, "module", {'name':self._name} )
|
|
|
167 for k,v in self._vars.iteritems():
|
|
|
168 param = et.SubElement( root, 'param', {'name':k} )
|
|
|
169 val = et.SubElement( param, 'value' )
|
|
|
170 val.text = v
|
|
|
171 return None
|
|
|
172
|
|
|
173 def __str__( self ):
|
|
|
174 "for debugging"
|
|
|
175 buffer = [ 'S { name=' ]
|
|
|
176 buffer.append(self._name)
|
|
|
177 buffer.append('; lines=%s' % ','.join(self._lines) )
|
|
|
178 for k,v in self._vars.iteritems():
|
|
|
179 buffer.append('; %s=%s' % (k,v) )
|
|
|
180 buffer.append(' }')
|
|
|
181 return ''.join(buffer)
|
|
|
182
|
|
|
183 class InputSection( Section ):
|
|
|
184 def __init__( self, name ):
|
|
|
185 Section.__init__(self,name)
|
|
|
186
|
|
|
187 def softLinkDats( self, newExtension ):
|
|
|
188 if not newExtension:
|
|
|
189 return
|
|
|
190 newLines = []
|
|
|
191 for l in self._lines:
|
|
|
192 if ':' in l:
|
|
|
193 protocol = l[:l.find(':')+1]
|
|
|
194 file = l[l.find(':')+1:]
|
|
|
195 else:
|
|
|
196 protocol = ''
|
|
|
197 file = l
|
|
|
198 if os.path.exists(file) and file.endswith('.dat'):
|
|
|
199 newFile = '%s.%s' % ( file, newExtension )
|
|
|
200 if not os.path.exists(newFile):
|
|
|
201 os.system( 'ln -s %s %s' % ( file, newFile ) )
|
|
|
202 newLines.append(protocol+newFile)
|
|
|
203 else:
|
|
|
204 newLines.append(l)
|
|
|
205 self._lines = newLines
|
|
|
206
|
|
|
207 def makeXmlElement( self, parent=None ):
|
|
|
208 root = et.Element( "pacbioAnalysisInputs" )
|
|
|
209 data = et.SubElement( root, 'dataReferences' )
|
|
|
210 iRef = 0
|
|
|
211 for l in self._lines:
|
|
|
212 def add(x,iRef):
|
|
|
213 if len(x)==0: return iRef
|
|
|
214 node = et.SubElement( data, 'url' )
|
|
|
215 if ':' in x:
|
|
|
216 node.attrib[ 'ref' ] = x
|
|
|
217 else:
|
|
|
218 node.attrib[ 'ref' ] = 'run:0000000-%04d' % iRef
|
|
|
219 node2 = et.SubElement( node, 'location' )
|
|
|
220 node2.text = x
|
|
|
221 return iRef+1
|
|
|
222 if l.endswith('fofn') and os.path.exists(l):
|
|
|
223 infile = open(l,'r')
|
|
|
224 for j,line in enumerate(infile): iRef=add(line.strip(),iRef)
|
|
|
225 infile.close()
|
|
|
226 else:
|
|
|
227 iRef=add(l,iRef)
|
|
|
228 return root
|
|
|
229
|
|
|
230 def backticks( cmd, merge_stderr=True ):
|
|
|
231 """
|
|
|
232 Simulates the perl backticks (``) command with error-handling support
|
|
|
233 Returns ( command output as sequence of strings, error code, error message )
|
|
|
234 """
|
|
|
235 if merge_stderr:
|
|
|
236 _stderr = subprocess.STDOUT
|
|
|
237 else:
|
|
|
238 _stderr = subprocess.PIPE
|
|
|
239
|
|
|
240 p = subprocess.Popen( cmd, shell=True, stdin=subprocess.PIPE,
|
|
|
241 stdout=subprocess.PIPE, stderr=_stderr,
|
|
|
242 close_fds=True )
|
|
|
243
|
|
|
244 out = [ l[:-1] for l in p.stdout.readlines() ]
|
|
|
245
|
|
|
246 p.stdout.close()
|
|
|
247 if not merge_stderr:
|
|
|
248 p.stderr.close()
|
|
|
249
|
|
|
250 # need to allow process to terminate
|
|
|
251 p.wait()
|
|
|
252
|
|
|
253 errCode = p.returncode and p.returncode or 0
|
|
|
254 if p.returncode>0:
|
|
|
255 errorMessage = os.linesep.join(out)
|
|
|
256 output = []
|
|
|
257 else:
|
|
|
258 errorMessage = ''
|
|
|
259 output = out
|
|
|
260
|
|
|
261 return output, errCode, errorMessage
|
|
|
262
|
|
|
263 if __name__=='__main__':
|
|
|
264 app = SmrtpipeGalaxy( sys.argv )
|
|
|
265 sys.exit( app.run() )
|
