# HG changeset patch # User linda.bakker@wur.nl # Date 1427358471 -3600 # Node ID 94b62c8be01e943fb3f3c8c98f6e4b9e35886874 # Parent 05ff1c55db84776913547e407dc832d16e40ed34 restored pdfread module diff -r 05ff1c55db84 -r 94b62c8be01e GCMS/combine_output.py --- a/GCMS/combine_output.py Fri Mar 20 17:11:04 2015 +0100 +++ b/GCMS/combine_output.py Thu Mar 26 09:27:51 2015 +0100 @@ -5,7 +5,6 @@ ''' import csv -import re import sys import math import pprint @@ -81,13 +80,15 @@ # The ID in the RankFilter output contains the following 5 fields: rf_id = rankfilter['ID'].split('-') try: + if 'Formula' not in rankfilter: + raise Exception("Error: old Rankfilter format detected (the selected Rankfilter data does not contain the column 'Formula'). Solution: rerun Rankfilter again.") hit = [rf_id[0], # Centrotype rf_id[1], # cent.Factor rf_id[2], # scan nr rf_id[3], # R.T. (umin) rf_id[4], # nr. Peaks + rankfilter['R.T.'], # Appending other fields - rankfilter['R.T.'], rankfilter['Name'], rankfilter['Formula'], rankfilter['Library'].strip(), diff -r 05ff1c55db84 -r 94b62c8be01e GCMS/combine_output.xml --- a/GCMS/combine_output.xml Fri Mar 20 17:11:04 2015 +0100 +++ b/GCMS/combine_output.xml Thu Mar 26 09:27:51 2015 +0100 @@ -15,13 +15,13 @@ -Performs a combination of output files from the 'RankFilter' and 'Lookup RI for CAS' tools into two tab-separated files. +Performs a combination of given 'RankFilter' and 'Lookup RI for CAS' files into two tab-separated files. -Merges data from both input dictionaries based on the Centrotype field. +This combination is a merge of the given files based on the Centrotype field. In the 'RIQC-RankFilter output' the centrotype is found in the 'ID' field (first part before the "-"). In the 'RIQC-Lookup RI for CAS output' the centrotype is found in the 'Centrotype' field. -The files produced are contain either all hits for a compound on a single line (Single) or on separate lines +The files produced contain either all hits for a compound on a single line (Single) or on separate lines (Multi). .. class:: infomark diff -r 05ff1c55db84 -r 94b62c8be01e rankfilter_GCMS/pdfread.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/rankfilter_GCMS/pdfread.py Thu Mar 26 09:27:51 2015 +0100 @@ -0,0 +1,203 @@ +""" +Copyright (C) 2011 by Velitchka Mihaleva, Wageningen University + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import sys +import csv + +def getPDF(filename, print_progress): + ''' + Parses NIST PDF file + @param filename: PDF file to parse + ''' + NistInput = {} + NistInput_missed = {} + nist_input = open(filename, 'r').read() + + hitid = [] + rt = [] + name = [] + forward = [] + cas = [] + reverse = [] + prob = [] + lib_id = [] + nist_id = [] + missed_compounds = [] + id_missed_compounds = [] + formula = [] + + hit_list = nist_input.split('** Search Report Page 1 of 1 **') + hit_list.pop(0) + #number_hits = range(10) + line_id = 0 + for line in hit_list: + line = line.strip().translate(None, '\r') + if line != '': + hits = line.replace('\n', ' ').replace('\x0c', '').replace('^L', '').split('Hit') #solution? : if we wouldn't replace the \n by ' ' but by some special sign, then reading formula would be simpler! + #strange....code seems fine actually...debug! See test/data/download.pdf + # strange thing is that it looks like the new line does not end up in the text file, eventhough it looks like there is a new line in the pdf...perhaps a bug in the pdf2text command in linux? + spec_id = hits.pop(0).split(' ')[1] + j = 0 + for hh in hits: + cell = hh.split(';') + if print_progress == True: + print 'Processing line: ', line_id, ' with length: ', len(cell), ':\n\t', cell + line_id += 1 + if len(cell) == 7: # the compound has CAS number + if len(cell[1].split(':')) == 2: + forward.append((cell[1].split(':')[1]).strip()) + # indication that the name contains the ":". Should join the cells of name_tmp from 1 till end + if len(cell[0].split(':')) > 2: + name_tmp = ':'.join(cell[0].split(':')[1:]) + else: + name_tmp = cell[0].split(':')[1] + + name.append(name_tmp.replace(" ", " ").strip()) + name_tmp = name_tmp.strip().split(' ') + if name_tmp: + # if the name ends with a word that starts with C, F or H, then assume this last word is a formula: + if name_tmp[-1][0] == 'C' or name_tmp[-1][0] == 'F' or name_tmp[-1][0] == 'H': + formule = (name_tmp[-1]) + else: + formule = ('not_def') + else: + formule = ('not_def') + formula.append(formule.replace(" ", " ")) + reverse.append((cell[2].split(':')[1]).strip()) + prob.append(cell[3].split(' ')[2].replace('%', '')) + cas.append((cell[4].split(':')[1]).strip()) + lib_id.append((cell[5].split(':')[1]).strip()) + nist_id.append(cell[6].split(':')[1].replace('.', '').strip()) + j = j + 1 + else: + missed_compounds.append(hh) + id_missed_compounds.append(spec_id) + + elif len(cell) == 6: # the compound has no CAS number + if len(cell[1].split(':')) == 2: + + forward.append((cell[1].split(':')[1]).strip()) + # indication that the name contains the ":". Should join the cells of name_tmp from 1 till end + if len(cell[0].split(':')) > 2: + name_tmp = ':'.join(cell[0].split(':')[1:]) + else: + name_tmp = cell[0].split(':')[1] + + name.append(name_tmp.replace(" ", " ").strip()) + name_tmp = name_tmp.strip().split(' ') + if name_tmp: + # if the name ends with a word that starts with C, F or H, then assume this last word is a formula: + if name_tmp[-1][0] == 'C' or name_tmp[-1][0] == 'F' or name_tmp[-1][0] == 'H': + formule = (name_tmp[-1]) + else: + formule = ('not_def') + else: + formule = ('not_def') + formula.append(formule.replace(" ", " ")) + reverse.append((cell[2].split(':')[1]).strip()) + prob.append(cell[3].split(' ')[2].replace('%', '')) + cas.append('undef') + lib_id.append((cell[4].split(':')[1]).strip()) + nist_id.append(cell[5].split(':')[1].replace('.', '').strip()) + j = j + 1 + + else: + missed_compounds.append(hh) + id_missed_compounds.append(spec_id) + + else: # Missing columns, report and quit + missed_compounds.append(hh) + id_missed_compounds.append(spec_id) + + for _ in range(j): + hitid.append(str(spec_id.replace(" ", " "))) + #NB: this is the RT as found in the "id" generated by e.g. msclust, so NOT the RT of the library hit: + rt.append(str(float(spec_id.split('-')[3]) / 1e+06)) + + NistInput['ID'] = hitid + NistInput['R.T.'] = rt + NistInput['Name'] = name + NistInput['CAS'] = cas + NistInput['Formula'] = formula + NistInput['Forward'] = forward + NistInput['Reverse'] = reverse + NistInput['Probability'] = prob + NistInput['Library'] = lib_id + NistInput['Library ID'] = nist_id + NistInput_missed['Missed Compounds'] = missed_compounds + NistInput_missed['ID missed Compounds'] = id_missed_compounds + + return NistInput, NistInput_missed + + +def convert_pdftotext2tabular(filename, output_file, error_file, print_progress): + ''' + Converts NIST PDF file to tabular format + @param filename: PDF file to parse + @param output_file: output file for the hits + @param error_file: output file for failed hits + ''' + [HitList, HitList_missed] = getPDF(filename, print_progress) + # save Hitlist as tab seperate file + Hitlist_as_text = "\t".join(HitList.keys()) + "\n" + Hitlist_array_of_array = ([HitList[row] for row in HitList.keys()]) + Hitlist_as_text += str("\n".join(["\t".join(e) for e in zip(*Hitlist_array_of_array)])) + output_fh = open(output_file, 'wb') + output_fh.write(Hitlist_as_text) + output_fh.close() + + out_missed_pdf = open(error_file, 'wb') + for x, y in zip(HitList_missed['Missed Compounds'], HitList_missed['ID missed Compounds']): + out_missed_pdf.write("Line with incorrect format or unexpected number of fields:\n") + out_missed_pdf.write('%s\n' % '\t'.join([y, x])) + out_missed_pdf.close() + + + + + +def read_tabular_old(filename): + ''' + Function to read tabular format (created by convert_pdftotext2tabular) + and output a dict with header of columns as key and value is columns of tabular as list + @param filename: tabular file to read + ''' + input_fh = None + try: + input_fh = open(filename, 'r') + except IOError, error: + raise error + colnames = input_fh.readline().strip().split('\t') + cells = [] + for line in input_fh.readlines(): + cells.append(line.strip().split('\t')) + #transform from row oriented structure to column oriented structure + cells = zip(*cells) + #store the list of list in form of final output + RankFilterGC_format = {} + for colnumber in range(len(colnames)): + RankFilterGC_format[colnames[colnumber]] = cells[colnumber] + return RankFilterGC_format + + +if __name__ == '__main__': + convert_pdftotext2tabular(sys.argv[1], sys.argv[2], sys.argv[3], True) diff -r 05ff1c55db84 -r 94b62c8be01e rankfilter_GCMS/rankfilter.py --- a/rankfilter_GCMS/rankfilter.py Fri Mar 20 17:11:04 2015 +0100 +++ b/rankfilter_GCMS/rankfilter.py Thu Mar 26 09:27:51 2015 +0100 @@ -142,6 +142,8 @@ # Convert 'Name' data to list in order to be indexed # library_data['Name']=list(library_data['Name']) + # tries to match on CAS first. If this is not possible (cas is 'undef' + # or not found) then tries to match on name: for hit_cas, hit_name in zip(hit_list['CAS'], hit_list['Name']): index = 0 if hit_cas != 'undef':