Mercurial > repos > proteore > proteore_filter_keywords_values

annotate filter_kw_val.py @ 0:a55e8b137c6b draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
author proteore
date Wed, 19 Sep 2018 05:01:15 -0400
parents
children 52a7afd01c6d
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
1 import argparse, re, csv
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
2
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
3 def options():
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
4 """
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
5 Parse options:
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
6 -i, --input Input filename and boolean value if the file contains header ["filename,true/false"]
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
7 --kw Keyword to be filtered, the column number where this filter applies,
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
8 boolean value if the keyword should be filtered in exact ["keyword,ncol,true/false"].
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
9 This option can be repeated: --kw "kw1,c1,true" --kw "kw2,c1,false" --kw "kw3,c2,true"
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
10 --kwfile A file that contains keywords to be filter, the column where this filter applies and
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
11 boolean value if the keyword should be filtered in exact ["filename,ncol,true/false"]
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
12 --value The value to be filtered, the column number where this filter applies and the
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
13 operation symbol ["value,ncol,=/>/>=/</<=/!="]
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
14 --values_range range of values to be keep, example : --values_range 5 20 c1 true
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
15 --operator The operator used to filter with several keywords/values : AND or OR
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
16 --o --output The output filename
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
17 --filtered_file The file contains removed lines
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
18 -s --sort_col Used column to sort the file, ",true" for reverse sorting, ",false" otherwise example : c1,false
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
19 """
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
20 parser = argparse.ArgumentParser()
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
21 parser.add_argument("-i", "--input", help="Input file", required=True)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
22 parser.add_argument("--kw", nargs="+", action="append", help="")
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
23 parser.add_argument("--kw_file", nargs="+", action="append", help="")
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
24 parser.add_argument("--value", nargs="+", action="append", help="")
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
25 parser.add_argument("--values_range", nargs="+", action="append", help="")
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
26 parser.add_argument("--operator", default="OR", type=str, choices=['AND','OR'],help='')
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
27 parser.add_argument("-o", "--output", default="output.txt")
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
28 parser.add_argument("--filtered_file", default="filtered_output.txt")
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
29 parser.add_argument("-s","--sort_col", help="")
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
30
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
31 args = parser.parse_args()
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
32 filters(args)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
33
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
34 def str_to_bool(v):
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
35 if v.lower() in ('yes', 'true', 't', 'y', '1'):
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
36 return True
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
37 elif v.lower() in ('no', 'false', 'f', 'n', '0'):
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
38 return False
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
39 else:
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
40 raise argparse.ArgumentTypeError('Boolean value expected.')
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
41
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
42 #Check if a variable is a float or an integer
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
43 def is_number(number_format, n):
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
44 float_format = re.compile(r"^[-]?[0-9][0-9]*.?[0-9]+$")
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
45 int_format = re.compile(r"^[-]?[0-9][0-9]*$")
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
46 test = ""
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
47 if number_format == "int":
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
48 test = re.match(int_format, n)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
49 elif number_format == "float":
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
50 test = re.match(float_format, n)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
51 if test:
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
52 return True
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
53
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
54 #Filter the document
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
55 def filters(args):
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
56 filename = args.input.split(",")[0]
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
57 header = str_to_bool(args.input.split(",")[1])
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
58 csv_file = read_file(filename)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
59 results_dict = {}
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
60
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
61 if args.kw:
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
62 keywords = args.kw
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
63 for k in keywords:
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
64 results_dict=filter_keyword(csv_file, header, results_dict, k[0], k[1], k[2])
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
65
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
66 if args.kw_file:
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
67 key_files = args.kw_file
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
68 for kf in key_files:
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
69 keywords = read_option(kf[0])
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
70 results_dict=filter_keyword(csv_file, header, results_dict, keywords, kf[1], kf[2])
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
71
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
72 if args.value:
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
73 for v in args.value:
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
74 if is_number("float", v[0]):
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
75 results_dict = filter_value(csv_file, header, results_dict, v[0], v[1], v[2])
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
76 else:
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
77 raise ValueError("Please enter a number in filter by value")
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
78
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
79 if args.values_range:
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
80 for vr in args.values_range:
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
81 if (is_number("float", vr[0]) or is_number("int", vr[0])) and (is_number("float",vr[1]) or is_number("int",vr[1])):
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
82 results_dict = filter_values_range(csv_file, header, results_dict, vr[0], vr[1], vr[2], vr[3])
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
83
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
84 remaining_lines=[]
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
85 filtered_lines=[]
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
86
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
87 if header is True :
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
88 remaining_lines.append(csv_file[0])
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
89 filtered_lines.append(csv_file[0])
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
90
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
91 for id_line,line in enumerate(csv_file) :
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
92 if id_line in results_dict : #skip header and empty lines
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
93 if args.operator == 'OR' :
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
94 if any(results_dict[id_line]) :
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
95 filtered_lines.append(line)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
96 else :
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
97 remaining_lines.append(line)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
98
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
99 elif args.operator == "AND" :
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
100 if all(results_dict[id_line]) :
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
101 filtered_lines.append(line)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
102 else :
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
103 remaining_lines.append(line)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
104
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
105 #sort of results by column
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
106 if args.sort_col :
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
107 sort_col=args.sort_col.split(",")[0]
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
108 sort_col=column_from_txt(sort_col)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
109 reverse=str_to_bool(args.sort_col.split(",")[1])
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
110 remaining_lines= sort_by_column(remaining_lines,sort_col,reverse,header)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
111 filtered_lines = sort_by_column(filtered_lines,sort_col,reverse,header)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
112
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
113 # Write results to output
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
114 with open(args.output,"w") as output :
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
115 writer = csv.writer(output,delimiter="\t")
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
116 writer.writerows(remaining_lines)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
117
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
118 # Write filtered lines to filtered_output
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
119 with open(args.filtered_file,"w") as filtered_output :
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
120 writer = csv.writer(filtered_output,delimiter="\t")
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
121 writer.writerows(filtered_lines)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
122
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
123 #function to sort the csv_file by value in a specific column
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
124 def sort_by_column(tab,sort_col,reverse,header):
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
125
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
126 if len(tab) > 1 : #if there's more than just a header or 1 row
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
127 if header is True :
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
128 head=tab[0]
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
129 tab=tab[1:]
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
130
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
131 if is_number("int",tab[0][sort_col]) :
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
132 tab = sorted(tab, key=lambda row: int(row[sort_col]), reverse=reverse)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
133 elif is_number("float",tab[0][sort_col]) :
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
134 tab = sorted(tab, key=lambda row: float(row[sort_col]), reverse=reverse)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
135 else :
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
136 tab = sorted(tab, key=lambda row: row[sort_col], reverse=reverse)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
137
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
138 if header is True : tab = [head]+tab
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
139
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
140 return tab
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
141
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
142 #Read the keywords file to extract the list of keywords
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
143 def read_option(filename):
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
144 with open(filename, "r") as f:
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
145 filter_list=f.read().splitlines()
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
146 filter_list=[key for key in filter_list if len(key.replace(' ',''))!=0]
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
147 filters=";".join(filter_list)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
148
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
149 return filters
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
150
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
151 # Read input file
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
152 def read_file(filename):
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
153 with open(filename,"r") as f :
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
154 reader=csv.reader(f,delimiter="\t")
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
155 tab=list(reader)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
156
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
157 # Remove empty lines (contain only space or new line or "")
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
158 #[tab.remove(blank) for blank in tab if blank.isspace() or blank == ""]
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
159 tab=[line for line in tab if len("".join(line).replace(" ","")) !=0 ]
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
160
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
161 return tab
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
162
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
163 #seek for keywords in rows of csvfile, return a dictionary of boolean (true if keyword found, false otherwise)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
164 def filter_keyword(csv_file, header, results_dict, keywords, ncol, match):
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
165 match=str_to_bool(match)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
166 ncol=column_from_txt(ncol)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
167
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
168 keywords = keywords.upper().split(";") # Split list of filter keyword
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
169 [keywords.remove(blank) for blank in keywords if blank.isspace() or blank == ""] # Remove blank keywords
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
170 keywords = [k.strip() for k in keywords] # Remove space from 2 heads of keywords
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
171
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
172 for id_line,line in enumerate(csv_file):
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
173 if header is True and id_line == 0 : continue
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
174 #line = line.replace("\n", "")
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
175 keyword_inline = line[ncol].replace('"', "").split(";")
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
176 #line = line + "\n"
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
177
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
178 #Perfect match or not
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
179 if match is True :
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
180 found_in_line = any(pid.upper() in keywords for pid in keyword_inline)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
181 else:
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
182 found_in_line = any(ft in pid.upper() for pid in keyword_inline for ft in keywords)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
183
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
184 #if the keyword is found in line
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
185 if id_line in results_dict : results_dict[id_line].append(found_in_line)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
186 else : results_dict[id_line]=[found_in_line]
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
187
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
188 return results_dict
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
189
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
190 #filter ba determined value in rows of csvfile, return a dictionary of boolean (true if value filtered, false otherwise)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
191 def filter_value(csv_file, header, results_dict, filter_value, ncol, opt):
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
192
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
193 filter_value = float(filter_value)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
194 ncol=column_from_txt(ncol)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
195
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
196 for id_line,line in enumerate(csv_file):
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
197 if header is True and id_line == 0 : continue
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
198 value = line[ncol].replace('"', "").strip()
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
199 if value.replace(".", "", 1).isdigit():
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
200 to_filter=value_compare(value,filter_value,opt)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
201
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
202 #adding the result to the dictionary
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
203 if id_line in results_dict : results_dict[id_line].append(to_filter)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
204 else : results_dict[id_line]=[to_filter]
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
205
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
206 return results_dict
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
207
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
208 #filter ba determined value in rows of csvfile, return a dictionary of boolean (true if value filtered, false otherwise)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
209 def filter_values_range(csv_file, header, results_dict, bottom_value, top_value, ncol, inclusive):
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
210 inclusive=str_to_bool(inclusive)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
211 bottom_value = float(bottom_value)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
212 top_value=float(top_value)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
213 ncol=column_from_txt(ncol)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
214
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
215 for id_line, line in enumerate(csv_file):
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
216 if header is True and id_line == 0 : continue
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
217 value = line[ncol].replace('"', "").strip()
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
218 if value.replace(".", "", 1).isdigit():
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
219 value=float(value)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
220 if inclusive is True:
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
221 in_range = not (bottom_value <= value <= top_value)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
222 else :
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
223 in_range = not (bottom_value < value < top_value)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
224
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
225 #adding the result to the dictionary
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
226 if id_line in results_dict : results_dict[id_line].append(in_range)
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
227 else : results_dict[id_line]=[in_range]
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
228
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
229 return results_dict
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
230
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
231 def column_from_txt(ncol):
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
232 if is_number("int", ncol.replace("c", "")):
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
233 ncol = int(ncol.replace("c", "")) - 1
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
234 else:
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
235 raise ValueError("Please specify the column where "
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
236 "you would like to apply the filter "
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
237 "with valid format")
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
238 return ncol
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
239
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
240 #return True if value is in the determined values, false otherwise
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
241 def value_compare(value,filter_value,opt):
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
242 test_value=False
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
243
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
244 if opt == "<":
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
245 if float(value) < filter_value:
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
246 test_value = True
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
247 elif opt == "<=":
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
248 if float(value) <= filter_value:
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
249 test_value = True
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
250 elif opt == ">":
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
251 if float(value) > filter_value:
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
252 test_value = True
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
253 elif opt == ">=":
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
254 if float(value) >= filter_value:
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
255 test_value = True
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
256 elif opt == "=":
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
257 if float(value) == filter_value:
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
258 test_value = True
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
259 elif opt == "!=":
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
260 if float(value) != filter_value:
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
261 test_value = True
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
262
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
263 return test_value
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
264
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
265 if __name__ == "__main__":
a55e8b137c6b planemo upload commit 688c456ca57914a63c20eba942ec5fe81e896099-dirty
proteore
parents:
diff changeset
266 options()