annotate SMART/DiffExpAnal/fastq_groomer_parallel_unSQL.py @ 34:529e3e6a0954

Deleted selected files
author m-zytnicki
date Tue, 30 Apr 2013 14:35:27 -0400
parents 94ab73e8a190
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
18
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
1 import sys, os, optparse,shutil, random
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
2 from commons.core.launcher.Launcher import Launcher
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
3 from commons.core.sql.TableJobAdaptatorFactory import TableJobAdaptatorFactory
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
4 from commons.core.utils.FileUtils import FileUtils
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
5
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
6 def _map(iLauncher, cmd, cmdStart, cmdFinish ):
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
7 lCmds = []
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
8 lCmds.extend(cmd)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
9 lCmdStart = []
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
10 lCmdStart.extend(cmdStart)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
11 lCmdFinish = []
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
12 lCmdFinish.extend(cmdFinish)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
13 return(iLauncher.prepareCommands_withoutIndentation(lCmds, lCmdStart, lCmdFinish))
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
14
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
15 def splitFastQ(fileName, nbOfSeqPerBatch):
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
16 nbOfLinesPerFile = nbOfSeqPerBatch * 4
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
17 lOutput = []
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
18 filePrefix, fileExt = os.path.splitext(os.path.basename(fileName))
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
19 resDir = os.path.dirname(fileName)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
20 with open(fileName) as inF:
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
21 fileNb = 1
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
22 line = inF.readline()
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
23 if not line or nbOfLinesPerFile == 0:
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
24 outFileName = "%s/%s-%s%s" %(resDir, filePrefix, fileNb, fileExt)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
25 lOutput.append(outFileName)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
26 f = open(outFileName, "wb")
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
27 shutil.copyfileobj(open(fileName, "rb"), f)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
28 f.close()
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
29 else:
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
30 while line:
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
31 outFileName = "%s/%s-%s%s" %(resDir, filePrefix, fileNb, fileExt)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
32 lOutput.append(outFileName)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
33 with open(outFileName, "w") as outF:
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
34 lineNb = 1
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
35 while lineNb <= nbOfLinesPerFile and line:
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
36 outF.write(line)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
37 line = inF.readline()
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
38 lineNb += 1
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
39 fileNb += 1
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
40 return lOutput
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
41
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
42 def joinFastQ(dCutOut2Out):
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
43 for key in dCutOut2Out.keys():
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
44 FileUtils.catFilesFromList(dCutOut2Out[key],key, False)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
45
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
46 def _createFastqGroomerCode(outGroomerNames, inputFileNames, input_type, output_type, force_quality_encoding, summarize_input):
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
47 cmd2Launch = []
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
48 cmd2Launch.append("log = 0")
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
49 cmd2Launch.append("from galaxy_utils.sequence.fastq import fastqReader, fastqVerboseErrorReader, fastqAggregator, fastqWriter")
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
50 cmd2Launch.append("aggregator = fastqAggregator()")
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
51 cmd2Launch.append("out = fastqWriter( open( '%s', 'wb' ), format = '%s', force_quality_encoding = '%s')" % (outGroomerNames,output_type,force_quality_encoding))
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
52 cmd2Launch.append("read_count = None")
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
53 if summarize_input:
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
54 cmd2Launch.append("reader = fastqVerboseErrorReader")
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
55 else:
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
56 cmd2Launch.append("reader = fastqReader")
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
57 cmd2Launch.append("for read_count, fastq_read in enumerate( reader( open( '%s' ), format = '%s', apply_galaxy_conventions = True ) ):" % (inputFileNames, input_type))
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
58 if summarize_input:
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
59 cmd2Launch.append("\taggregator.consume_read( fastq_read )")
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
60 cmd2Launch.append("\tout.write( fastq_read )")
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
61 cmd2Launch.append("out.close()")
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
62 cmd2Launch.append("if read_count is not None:")
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
63 #cmd2Launch.append("\tprint 'Groomed %s %s reads into %s reads.' % ( read_count + 1, %s, %s )" % ('%i', '%s', '%s', input_type,output_type))
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
64 cmd2Launch.append("\tif '%s' != '%s' and 'solexa' in [ '%s', '%s' ]:" % (input_type, output_type, input_type, output_type))
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
65 cmd2Launch.append("\t\tprint 'Converted between Solexa and PHRED scores.'")
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
66 if summarize_input:
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
67 cmd2Launch.append("\tprint 'Based upon quality and sequence, the input data is valid for: %s' % ( ', '.join( aggregator.get_valid_formats() ) or 'None' )")
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
68 cmd2Launch.append("\tascii_range = aggregator.get_ascii_range()")
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
69 cmd2Launch.append("\tdecimal_range = aggregator.get_decimal_range()")
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
70 cmd2Launch.append("\tprint 'Input ASCII range: %s(%i) - %s(%i)' % ( repr( ascii_range[0] ), ord( ascii_range[0] ), repr( ascii_range[1] ), ord( ascii_range[1] ) )")
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
71 cmd2Launch.append("\tprint 'Input decimal range: %i - %i' % ( decimal_range[0], decimal_range[1] ) ")
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
72 cmd2Launch.append("else:")
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
73 cmd2Launch.append("\tprint 'No valid FASTQ reads were provided.'")
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
74 cmd2Launch.append("\tlog = 255")
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
75 return cmd2Launch
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
76
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
77 def stop_err(msg):
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
78 sys.stderr.write("%s\n" % msg)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
79 sys.exit()
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
80
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
81 def main():
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
82
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
83 input_filename = sys.argv[1] #a txt file
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
84 input_type = sys.argv[2]
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
85 output_filename = sys.argv[3] #a txt file
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
86 output_type = sys.argv[4]
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
87 force_quality_encoding = sys.argv[5]
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
88 summarize_input = sys.argv[6] == 'summarize_input'
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
89 pairedEnd_input = sys.argv[7]
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
90 if pairedEnd_input == 'None':
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
91 pairedEnd_input = None
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
92 else:
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
93 output_pairedEndFileName = sys.argv[8]
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
94
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
95 if force_quality_encoding == 'None':
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
96 force_quality_encoding = None
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
97
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
98 #Parse the input txt file and read a list of fastq files
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
99 file = open(input_filename, "r")
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
100 lines = file.readlines()
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
101 inputFileNames = []
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
102 outGroomerNames = []
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
103 resDirName = os.path.dirname(output_filename) + "/"
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
104 #Write output txt file and define all output groomer file names
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
105 outFile = open(output_filename, "w")
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
106 for line in lines:
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
107 tab = line.split()
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
108 inputFileNames.append(tab[1])
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
109 outGroomerName = resDirName + tab[0] + '_outGroomer_%s.fastq' % random.randrange(0, 10000)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
110 outGroomerNames.append(outGroomerName)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
111 outFile.write(tab[0] + '\t' + outGroomerName + '\n')
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
112 outFile.close()
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
113 file.close()
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
114
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
115 if pairedEnd_input != None:
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
116 inPairedFile = open(pairedEnd_input, "r")
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
117 lines = inPairedFile.readlines()
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
118 inputPairedEndFileNames = []
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
119 outGroomerPairedEndNames = []
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
120 outPairedEndFile = open(output_pairedEndFileName, "w")
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
121 for line in lines:
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
122 tab = line.split()
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
123 inputPairedEndFileNames.append(tab[1])
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
124 outGroomerPairedEndName = resDirName + tab[0] + '_outGroomer_pairedEnd_%s.fastq' % random.randrange(0, 10000)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
125 outGroomerPairedEndNames.append(outGroomerPairedEndName)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
126 outPairedEndFile.write(tab[0] + '\t' + outGroomerPairedEndName + '\n')
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
127 outPairedEndFile.close()
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
128 inPairedFile.close()
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
129
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
130 acronym = "fastqGroomer"
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
131 jobdb = TableJobAdaptatorFactory.createJobInstance()
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
132 iLauncher = Launcher(jobdb, os.getcwd(), "", "", os.getcwd(), os.getcwd(), "jobs", "", acronym, acronym, False, True)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
133 lCmdsTuples = []
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
134 dCutOut2Out = {}
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
135 lAllFile2remove = []
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
136 # Write output file
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
137 for i in range(len(outGroomerNames)):
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
138 lCutInputFile = splitFastQ(inputFileNames[i], 20000)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
139 lAllFile2remove.extend(lCutInputFile)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
140 lCutOutput = []
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
141 for cutInput in lCutInputFile:
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
142 cutOutput = "%s_out" % cutInput
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
143 lCutOutput.append(cutOutput)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
144 lAllFile2remove.extend(lCutOutput)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
145 cmd2Launch = _createFastqGroomerCode(cutOutput, cutInput, input_type, output_type, force_quality_encoding, summarize_input)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
146 cmdStart = []
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
147 cmdFinish = []
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
148 lCmdsTuples.append(_map(iLauncher, cmd2Launch, cmdStart, cmdFinish))
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
149 dCutOut2Out[outGroomerNames[i]] = lCutOutput
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
150 if pairedEnd_input != None:
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
151 lCutInputFile = splitFastQ(inputPairedEndFileNames[i], 20000)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
152 lAllFile2remove.extend(lCutInputFile)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
153 lCutOutput = []
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
154 for cutInput in lCutInputFile:
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
155 cutOutput = "%s_out" % cutInput
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
156 lCutOutput.append(cutOutput)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
157 lAllFile2remove.extend(lCutOutput)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
158 cmd2Launch = _createFastqGroomerCode(cutOutput, cutInput, input_type, output_type, force_quality_encoding, summarize_input)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
159 cmdStart = []
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
160 cmdFinish = []
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
161 lCmdsTuples.append(_map(iLauncher, cmd2Launch, cmdStart, cmdFinish))
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
162 dCutOut2Out[outGroomerPairedEndNames[i]] = lCutOutput
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
163 iLauncher.runLauncherForMultipleJobs(acronym, lCmdsTuples, False)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
164
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
165 joinFastQ(dCutOut2Out)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
166 FileUtils.removeFilesFromListIfExist(lAllFile2remove)
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
167
94ab73e8a190 Uploaded
m-zytnicki
parents:
diff changeset
168 if __name__ == "__main__": main()