annotate tools/stats/column_maker.py @ 0:9071e359b9a3

Uploaded
author xuebing
date Fri, 09 Mar 2012 19:37:19 -0500
parents
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
1 #!/usr/bin/env python
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
2 # This tool takes a tab-delimited textfile as input and creates another column in the file which is the result of
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
3 # a computation performed on every row in the original file. The tool will skip over invalid lines within the file,
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
4 # informing the user about the number of lines skipped.
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
5 import sys, re, os.path
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
6 from galaxy import eggs
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
7 from galaxy.tools import validation
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
8 from galaxy.datatypes import metadata
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
9 from math import log,exp,sqrt,ceil,floor
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
10
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
11 assert sys.version_info[:2] >= ( 2, 4 )
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
12
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
13 def stop_err( msg ):
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
14 sys.stderr.write( msg )
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
15 sys.exit()
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
16
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
17 inp_file = sys.argv[1]
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
18 out_file = sys.argv[2]
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
19 expr = sys.argv[3]
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
20 round_result = sys.argv[4]
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
21 try:
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
22 in_columns = int( sys.argv[5] )
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
23 except:
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
24 stop_err( "Missing or invalid 'columns' metadata value, click the pencil icon in the history item and select the Auto-detect option to correct it. This tool can only be used with tab-delimited data." )
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
25 if in_columns < 2:
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
26 # To be considered tabular, data must fulfill requirements of the sniff.is_column_based() method.
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
27 stop_err( "Missing or invalid 'columns' metadata value, click the pencil icon in the history item and select the Auto-detect option to correct it. This tool can only be used with tab-delimited data." )
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
28 try:
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
29 in_column_types = sys.argv[6].split( ',' )
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
30 except:
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
31 stop_err( "Missing or invalid 'column_types' metadata value, click the pencil icon in the history item and select the Auto-detect option to correct it. This tool can only be used with tab-delimited data." )
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
32 if len( in_column_types ) != in_columns:
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
33 stop_err( "The 'columns' metadata setting does not conform to the 'column_types' metadata setting, click the pencil icon in the history item and select the Auto-detect option to correct it. This tool can only be used with tab-delimited data." )
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
34
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
35 # Unescape if input has been escaped
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
36 mapped_str = {
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
37 '__lt__': '<',
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
38 '__le__': '<=',
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
39 '__eq__': '==',
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
40 '__ne__': '!=',
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
41 '__gt__': '>',
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
42 '__ge__': '>=',
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
43 '__sq__': '\'',
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
44 '__dq__': '"',
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
45 }
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
46 for key, value in mapped_str.items():
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
47 expr = expr.replace( key, value )
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
48
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
49 # Prepare the column variable names and wrappers for column data types
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
50 cols, type_casts = [], []
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
51 for col in range( 1, in_columns + 1 ):
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
52 col_name = "c%d" % col
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
53 cols.append( col_name )
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
54 col_type = in_column_types[ col - 1 ].strip()
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
55 if round_result == 'no' and col_type == 'int':
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
56 col_type = 'float'
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
57 type_cast = "%s(%s)" % ( col_type, col_name )
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
58 type_casts.append( type_cast )
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
59
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
60 col_str = ', '.join( cols ) # 'c1, c2, c3, c4'
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
61 type_cast_str = ', '.join( type_casts ) # 'str(c1), int(c2), int(c3), str(c4)'
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
62 assign = "%s = line.split( '\\t' )" % col_str
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
63 wrap = "%s = %s" % ( col_str, type_cast_str )
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
64 skipped_lines = 0
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
65 first_invalid_line = 0
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
66 invalid_line = None
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
67 lines_kept = 0
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
68 total_lines = 0
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
69 out = open( out_file, 'wt' )
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
70
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
71 # Read input file, skipping invalid lines, and perform computation that will result in a new column
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
72 code = '''
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
73 for i, line in enumerate( file( inp_file ) ):
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
74 total_lines += 1
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
75 line = line.rstrip( '\\r\\n' )
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
76 if not line or line.startswith( '#' ):
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
77 skipped_lines += 1
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
78 if not invalid_line:
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
79 first_invalid_line = i + 1
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
80 invalid_line = line
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
81 continue
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
82 try:
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
83 %s
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
84 %s
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
85 new_val = %s
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
86 if round_result == "yes":
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
87 new_val = int( round( new_val ) )
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
88 new_line = line + '\\t' + str( new_val )
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
89 print >> out, new_line
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
90 lines_kept += 1
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
91 except:
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
92 skipped_lines += 1
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
93 if not invalid_line:
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
94 first_invalid_line = i + 1
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
95 invalid_line = line
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
96 ''' % ( assign, wrap, expr )
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
97
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
98 valid_expr = True
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
99 try:
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
100 exec code
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
101 except Exception, e:
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
102 out.close()
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
103 if str( e ).startswith( 'invalid syntax' ):
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
104 valid_expr = False
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
105 stop_err( 'Expression "%s" likely invalid. See tool tips, syntax and examples.' % expr )
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
106 else:
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
107 stop_err( str( e ) )
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
108
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
109 if valid_expr:
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
110 out.close()
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
111 valid_lines = total_lines - skipped_lines
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
112 print 'Creating column %d with expression %s' % ( in_columns + 1, expr )
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
113 if valid_lines > 0:
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
114 print 'kept %4.2f%% of %d lines.' % ( 100.0*lines_kept/valid_lines, total_lines )
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
115 else:
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
116 print 'Possible invalid expression "%s" or non-existent column referenced. See tool tips, syntax and examples.' % expr
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
117 if skipped_lines > 0:
9071e359b9a3 Uploaded
xuebing
parents:
diff changeset
118 print 'Skipped %d invalid lines starting at line #%d: "%s"' % ( skipped_lines, first_invalid_line, invalid_line )