|
0
|
1 #!/usr/bin/env python
|
|
|
2 # This tool takes a tab-delimited text file as input and creates filters on columns based on certain properties.
|
|
|
3 # The tool will skip over invalid lines within the file, informing the user about the number of lines skipped.
|
|
|
4
|
|
|
5 from __future__ import division
|
|
|
6 import sys, re, os.path
|
|
|
7 from galaxy import eggs
|
|
|
8
|
|
|
9 # Older py compatibility
|
|
|
10 try:
|
|
|
11 set()
|
|
|
12 except:
|
|
|
13 from sets import Set as set
|
|
|
14
|
|
|
15 assert sys.version_info[:2] >= ( 2, 4 )
|
|
|
16
|
|
|
17 def get_operands( filter_condition ):
|
|
|
18 # Note that the order of all_operators is important
|
|
|
19 items_to_strip = ['+', '-', '**', '*', '//', '/', '%', '<<', '>>', '&', '|', '^', '~', '<=', '<', '>=', '>', '==', '!=', '<>', ' and ', ' or ', ' not ', ' is ', ' is not ', ' in ', ' not in ']
|
|
|
20 for item in items_to_strip:
|
|
|
21 if filter_condition.find( item ) >= 0:
|
|
|
22 filter_condition = filter_condition.replace( item, ' ' )
|
|
|
23 operands = set( filter_condition.split( ' ' ) )
|
|
|
24 return operands
|
|
|
25
|
|
|
26 def stop_err( msg ):
|
|
|
27 sys.stderr.write( msg )
|
|
|
28 sys.exit()
|
|
|
29
|
|
|
30 in_fname = sys.argv[1]
|
|
|
31 out_fname = sys.argv[2]
|
|
|
32 cond_text = sys.argv[3]
|
|
|
33 try:
|
|
|
34 in_columns = int( sys.argv[4] )
|
|
|
35 assert sys.argv[5] #check to see that the column types variable isn't null
|
|
|
36 in_column_types = sys.argv[5].split( ',' )
|
|
|
37 except:
|
|
|
38 stop_err( "Data does not appear to be tabular. This tool can only be used with tab-delimited data." )
|
|
|
39
|
|
|
40 # Unescape if input has been escaped
|
|
|
41 mapped_str = {
|
|
|
42 '__lt__': '<',
|
|
|
43 '__le__': '<=',
|
|
|
44 '__eq__': '==',
|
|
|
45 '__ne__': '!=',
|
|
|
46 '__gt__': '>',
|
|
|
47 '__ge__': '>=',
|
|
|
48 '__sq__': '\'',
|
|
|
49 '__dq__': '"',
|
|
|
50 }
|
|
|
51 for key, value in mapped_str.items():
|
|
|
52 cond_text = cond_text.replace( key, value )
|
|
|
53
|
|
|
54 # Attempt to determine if the condition includes executable stuff and, if so, exit
|
|
|
55 secured = dir()
|
|
|
56 operands = get_operands(cond_text)
|
|
|
57 for operand in operands:
|
|
|
58 try:
|
|
|
59 check = int( operand )
|
|
|
60 except:
|
|
|
61 if operand in secured:
|
|
|
62 stop_err( "Illegal value '%s' in condition '%s'" % ( operand, cond_text ) )
|
|
|
63
|
|
|
64 # Work out which columns are used in the filter (save using 1 based counting)
|
|
|
65 used_cols = sorted(set(int(match.group()[1:]) \
|
|
|
66 for match in re.finditer('c(\d)+', cond_text)))
|
|
|
67 largest_col_index = max(used_cols)
|
|
|
68
|
|
|
69 # Prepare the column variable names and wrappers for column data types. Only
|
|
|
70 # cast columns used in the filter.
|
|
|
71 cols, type_casts = [], []
|
|
|
72 for col in range( 1, largest_col_index + 1 ):
|
|
|
73 col_name = "c%d" % col
|
|
|
74 cols.append( col_name )
|
|
|
75 col_type = in_column_types[ col - 1 ]
|
|
|
76 if col in used_cols:
|
|
|
77 type_cast = "%s(%s)" % ( col_type, col_name )
|
|
|
78 else:
|
|
|
79 #If we don't use this column, don't cast it.
|
|
|
80 #Otherwise we get errors on things like optional integer columns.
|
|
|
81 type_cast = col_name
|
|
|
82 type_casts.append( type_cast )
|
|
|
83
|
|
|
84 col_str = ', '.join( cols ) # 'c1, c2, c3, c4'
|
|
|
85 type_cast_str = ', '.join( type_casts ) # 'str(c1), int(c2), int(c3), str(c4)'
|
|
|
86 assign = "%s, = line.split( '\\t' )[:%i]" % ( col_str, largest_col_index )
|
|
|
87 wrap = "%s = %s" % ( col_str, type_cast_str )
|
|
|
88 skipped_lines = 0
|
|
|
89 invalid_lines = 0
|
|
|
90 first_invalid_line = 0
|
|
|
91 invalid_line = None
|
|
|
92 lines_kept = 0
|
|
|
93 total_lines = 0
|
|
|
94 out = open( out_fname, 'wt' )
|
|
|
95
|
|
|
96 # Read and filter input file, skipping invalid lines
|
|
|
97 code = '''
|
|
|
98 for i, line in enumerate( file( in_fname ) ):
|
|
|
99 total_lines += 1
|
|
|
100 line = line.rstrip( '\\r\\n' )
|
|
|
101 if not line or line.startswith( '#' ):
|
|
|
102 skipped_lines += 1
|
|
|
103 continue
|
|
|
104 try:
|
|
|
105 %s
|
|
|
106 %s
|
|
|
107 if %s:
|
|
|
108 lines_kept += 1
|
|
|
109 print >> out, line
|
|
|
110 except:
|
|
|
111 invalid_lines += 1
|
|
|
112 if not invalid_line:
|
|
|
113 first_invalid_line = i + 1
|
|
|
114 invalid_line = line
|
|
|
115 ''' % ( assign, wrap, cond_text )
|
|
|
116
|
|
|
117 valid_filter = True
|
|
|
118 try:
|
|
|
119 exec code
|
|
|
120 except Exception, e:
|
|
|
121 out.close()
|
|
|
122 if str( e ).startswith( 'invalid syntax' ):
|
|
|
123 valid_filter = False
|
|
|
124 stop_err( 'Filter condition "%s" likely invalid. See tool tips, syntax and examples.' % cond_text )
|
|
|
125 else:
|
|
|
126 stop_err( str( e ) )
|
|
|
127
|
|
|
128 if valid_filter:
|
|
|
129 out.close()
|
|
|
130 valid_lines = total_lines - skipped_lines
|
|
|
131 print 'Filtering with %s, ' % cond_text
|
|
|
132 if valid_lines > 0:
|
|
|
133 print 'kept %4.2f%% of %d valid lines (%d total lines).' % ( 100.0*lines_kept/valid_lines, valid_lines, total_lines )
|
|
|
134 else:
|
|
|
135 print 'Possible invalid filter condition "%s" or non-existent column referenced. See tool tips, syntax and examples.' % cond_text
|
|
|
136 if invalid_lines:
|
|
|
137 print 'Skipped %d invalid line(s) starting at line #%d: "%s"' % ( invalid_lines, first_invalid_line, invalid_line )
|
|
|
138 if skipped_lines:
|
|
|
139 print 'Skipped %i comment (starting with #) or blank line(s)' % skipped_lines
|
