|
0
|
1 #!/usr/bin/env python
|
|
|
2
|
|
|
3 # https://github.com/ross/requests-futures
|
|
|
4 # http://docs.python-requests.org/en/master/user/quickstart/#more-complicated-post-requests
|
|
|
5
|
|
|
6 import sys, os, uuid, optparse, requests, json, time
|
|
|
7 #from requests_futures.sessions import FuturesSession
|
|
|
8
|
|
|
9 #### NN14 ####
|
|
|
10 SERVICE_URL = "http://nn14.galaxyproject.org:8080/";
|
|
|
11 #service_url = "http://127.0.0.1:8082/";
|
|
4
|
12 QUERY_URL = SERVICE_URL+"tree/<tree_id>/query";
|
|
0
|
13 STATUS_URL = SERVICE_URL+"status/<query_id>";
|
|
|
14 ##############
|
|
|
15 # query delay in seconds
|
|
|
16 QUERY_DELAY = 30;
|
|
|
17 ##############
|
|
|
18
|
|
|
19 __version__ = "1.0.0";
|
|
|
20 VALID_CHARS = '.-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ '
|
|
|
21 # in the case of collections, exitcodes equal to 0 and 1 are not considered errors
|
|
|
22 ERR_EXIT_CODE = 2;
|
|
|
23 OK_EXIT_CODE = 0;
|
|
|
24
|
|
|
25 def raiseException( exitcode, message, output_dir_path, errorfilename ):
|
|
|
26 errorfilepath = os.path.join(output_dir_path, errorfilename+"_txt");
|
|
|
27 with open(errorfilepath, 'w') as out:
|
|
|
28 out.write(message);
|
|
|
29 sys.exit(exitcode);
|
|
|
30
|
|
|
31 def query_request( options, args, payload ):
|
|
|
32 output_dir_path = options.outputdir;
|
|
|
33 # add additional parameters to the payload
|
|
|
34 #payload["tree_id"] = str(options.treeid);
|
|
|
35 payload["search_mode"] = str(options.search);
|
|
|
36 payload["exact_algorithm"] = int(options.exact);
|
|
|
37 payload["search_threshold"] = float(options.sthreshold);
|
|
10
|
38 payload["sort"] = int(options.sortcontrol);
|
|
0
|
39 # set the content type to application/json
|
|
|
40 headers = {'Content-type': 'application/json'};
|
|
|
41
|
|
|
42 # create a session
|
|
|
43 session = requests.Session();
|
|
|
44 # make a synchronous post request to the query route
|
|
4
|
45 QUERY_URL.replace("<tree_id>", str(options.treeid));
|
|
|
46 req = session.post(QUERY_URL.replace("<tree_id>", str(options.treeid)), headers=headers, json=payload);
|
|
0
|
47 resp_code = req.status_code;
|
|
|
48 #print(str(req.content)+"\n\n");
|
|
|
49 if resp_code == requests.codes.ok:
|
|
|
50 resp_content = str(req.content);
|
|
|
51 # convert out to json
|
|
|
52 json_content = json.loads(resp_content);
|
|
|
53 # retrieve query id
|
|
|
54 query_id = json_content['query_id'];
|
|
|
55 query_processed = False;
|
|
|
56 # results json content
|
|
|
57 json_status_content = None;
|
|
|
58 while query_processed is False:
|
|
|
59 # create a new session
|
|
|
60 session = requests.Session();
|
|
|
61 # make a synchronous get request to the status route
|
|
|
62 status_query_url = STATUS_URL.replace("<query_id>", query_id);
|
|
|
63 status_req = session.get(status_query_url);
|
|
|
64 status_resp_content = str(status_req.content);
|
|
|
65 #print(status_resp_content+"\n\n");
|
|
|
66 # convert out to json
|
|
|
67 json_status_content = json.loads(status_resp_content);
|
|
|
68 # take a look at the state
|
|
|
69 # state attribute is always available
|
|
|
70 if json_status_content['state'] == 'SUCCESS':
|
|
|
71 query_processed = True;
|
|
|
72 break;
|
|
|
73 elif json_status_content['state'] in ['FAILURE', 'REVOKED']:
|
|
|
74 return raiseException( ERR_EXIT_CODE, "Query ID: "+str(query_id)+"\nQuery status: "+str(json_status_content['state']), output_dir_path, str(options.errorfile) );
|
|
|
75 else:
|
|
|
76 time.sleep(QUERY_DELAY); # in seconds
|
|
|
77
|
|
|
78 out_file_format = "tabular";
|
|
|
79 for block in json_status_content['results']:
|
|
|
80 seq_id = block['sequence_id'];
|
|
|
81 # put response block in the output collection
|
|
|
82 output_file_path = os.path.join(output_dir_path, seq_id + "_" + out_file_format);
|
|
|
83 accessions_list = "";
|
|
|
84 hits_block = block['hits'];
|
|
8
|
85 accessions_dict = { };
|
|
4
|
86 is_sabutan = False;
|
|
0
|
87 for hit in hits_block:
|
|
|
88 if type(hit) is dict: # sabutan
|
|
4
|
89 #accessions_list = accessions_list + str(hit['accession_number']) + "\t" + str(hit['score']) + "\n";
|
|
|
90 accession_number = hit['accession_number'];
|
|
12
|
91 #------------
|
|
|
92 #score = hit['score'];
|
|
|
93 #score_split = score.split("/");
|
|
|
94 #accessions_dict[accession_number] = "{0:.6f}".format(float(score_split[0])/float(score_split[1]));
|
|
|
95 #------------
|
|
|
96 fraction = hit['fraction'];
|
|
4
|
97 score = hit['score'];
|
|
12
|
98 accession_scores = {
|
|
|
99 "fraction": str(fraction),
|
|
|
100 "score": float(score)
|
|
|
101 }
|
|
|
102 accessions_dict[accession_number] = accession_scores;
|
|
4
|
103 is_sabutan = True;
|
|
0
|
104 else: # all-some
|
|
|
105 accessions_list = accessions_list + str(hit) + "\n";
|
|
4
|
106 if is_sabutan:
|
|
12
|
107 sorted_accessions = sorted(accessions_dict, key=lambda i: float(accessions_dict[i]["score"]), reverse=True);
|
|
4
|
108 for acc in sorted_accessions:
|
|
14
|
109 accessions_list = accessions_list + str(acc) + "\t" + str(accessions_dict[acc]["fraction"]) + "\t" + str(accessions_dict[acc]["score"]) + "\n";
|
|
0
|
110 with open(output_file_path, 'w') as out:
|
|
|
111 out.write(accessions_list.strip());
|
|
|
112 return sys.exit(OK_EXIT_CODE);
|
|
|
113 else:
|
|
|
114 return raiseException( ERR_EXIT_CODE, "Unable to query the remote server. Please try again in a while.", output_dir_path, str(options.errorfile) );
|
|
|
115
|
|
|
116 def query( options, args ):
|
|
|
117 output_dir_path = options.outputdir;
|
|
|
118 multiple_data = {};
|
|
|
119 comma_sep_file_paths = options.files;
|
|
|
120 #print("files: "+str(comma_sep_file_paths)+" - "+str(type(comma_sep_file_paths)));
|
|
|
121 # check if options.files contains at least one file path
|
|
|
122 if comma_sep_file_paths is not None:
|
|
|
123 # split file paths
|
|
|
124 file_paths = comma_sep_file_paths.split(",");
|
|
|
125 # split file names
|
|
|
126 comma_sep_file_names = str(options.names);
|
|
|
127 #print("names: "+str(comma_sep_file_names));
|
|
|
128 file_names = comma_sep_file_names.split(",");
|
|
|
129 for idx, file_path in enumerate(file_paths):
|
|
|
130 #file_name = file_names[idx];
|
|
|
131 with open(file_path, 'r') as content_file:
|
|
|
132 for line in content_file:
|
|
|
133 if line.strip() != "":
|
|
|
134 line_split = line.strip().split("\t"); # split on tab
|
|
|
135 if len(line_split) == 2: # 0:id , 1:seq , otherwise skip line
|
|
|
136 seq_id = line_split[0];
|
|
|
137 # fix seq_id using valid chars only
|
|
|
138 seq_id = ''.join(e for e in seq_id if e in VALID_CHARS)
|
|
|
139 seq_text = line_split[1];
|
|
|
140 if seq_id in multiple_data:
|
|
|
141 return raiseException( ERR_EXIT_CODE, "Error: the id '"+seq_id+"' is duplicated", output_dir_path, str(options.errorfile) );
|
|
|
142 multiple_data[seq_id] = seq_text;
|
|
|
143 if len(multiple_data) > 0:
|
|
|
144 return query_request( options, args, multiple_data );
|
|
|
145 #return echo( options, args );
|
|
|
146 else:
|
|
|
147 return raiseException( ERR_EXIT_CODE, "An error has occurred. Please be sure that your input files are valid.", output_dir_path, str(options.errorfile) );
|
|
|
148 else:
|
|
|
149 # try with the sequence in --sequence
|
|
|
150 text_content = options.sequences;
|
|
|
151 #print("sequences: "+text_content);
|
|
|
152 # check if options.sequences contains a list of sequences (one for each row)
|
|
|
153 if text_content is not None:
|
|
|
154 text_content = str(text_content);
|
|
|
155 if text_content.strip():
|
|
|
156 # populate a dictionary with the files containing the sequences to query
|
|
|
157 text_content = text_content.strip().split("__cn__"); # split on new line
|
|
|
158 for line in text_content:
|
|
|
159 if line.strip() != "":
|
|
|
160 line_split = line.strip().split("__tc__"); # split on tab
|
|
|
161 if len(line_split) == 2: # 0:id , 1:seq , otherwise skip line
|
|
|
162 seq_id = line_split[0];
|
|
|
163 # fix seq_id using valid chars only
|
|
|
164 seq_id = ''.join(e for e in seq_id if e in VALID_CHARS)
|
|
|
165 seq_text = line_split[1];
|
|
|
166 if seq_id in multiple_data:
|
|
|
167 return raiseException( ERR_EXIT_CODE, "Error: the id '"+seq_id+"' is duplicated", output_dir_path, str(options.errorfile) );
|
|
|
168 multiple_data[seq_id] = seq_text;
|
|
|
169 if len(multiple_data) > 0:
|
|
|
170 return query_request( options, args, multiple_data );
|
|
|
171 #return echo( options, args );
|
|
|
172 else:
|
|
|
173 return raiseException( ERR_EXIT_CODE, "An error has occurred. Please be sure that your input files are valid.", output_dir_path, str(options.errorfile) );
|
|
|
174 else:
|
|
|
175 return raiseException( ERR_EXIT_CODE, "You have to insert at least one row formatted as a tab delimited (ID, SEQUENCE) couple", output_dir_path, str(options.errorfile) );
|
|
|
176 return ERR_EXIT_CODE;
|
|
|
177
|
|
|
178 def __main__():
|
|
|
179 # Parse the command line options
|
|
|
180 usage = "Usage: query.py --files comma_sep_file_paths --names comma_seq_file_names --sequences sequences_text --search search_mode --exact exact_alg --sthreshold threshold --outputdir output_dir_path";
|
|
|
181 parser = optparse.OptionParser(usage = usage);
|
|
|
182 parser.add_option("-v", "--version", action="store_true", dest="version",
|
|
|
183 default=False, help="display version and exit")
|
|
|
184 parser.add_option("-f", "--files", type="string",
|
|
|
185 action="store", dest="files", help="comma separated files path");
|
|
|
186 parser.add_option("-n", "--names", type="string",
|
|
|
187 action="store", dest="names", help="comma separated names associated to the files specified in --files");
|
|
|
188 parser.add_option("-s", "--sequences", type="string",
|
|
|
189 action="store", dest="sequences", help="contains a list of sequences (one for each row)");
|
|
|
190 parser.add_option("-a", "--fasta", type="string",
|
|
|
191 action="store", dest="fasta", help="contains the content of a fasta file");
|
|
4
|
192 parser.add_option("-x", "--search", type="string", default="rrr",
|
|
0
|
193 action="store", dest="search", help="search mode");
|
|
|
194 parser.add_option("-e", "--exact", type="int", default=0,
|
|
|
195 action="store", dest="exact", help="exact algorithm (required if search is 1 only)");
|
|
4
|
196 parser.add_option("-k", "--tree", type="int", default=0,
|
|
|
197 action="store", dest="treeid", help="the id of the tree that will be queried");
|
|
0
|
198 parser.add_option("-t", "--sthreshold", type="float",
|
|
|
199 action="store", dest="sthreshold", help="threshold applied to the search algrithm");
|
|
10
|
200 parser.add_option("-z", "--sort", type="int", default=1,
|
|
|
201 action="store", dest="sortcontrol", help="boolean required to sort the result");
|
|
0
|
202 parser.add_option("-o", "--outputdir", type="string", default="output",
|
|
|
203 action="store", dest="outputdir", help="output directory (collection) path");
|
|
4
|
204 parser.add_option("-r", "--errorfile", type="string", default="error_txt",
|
|
0
|
205 action="store", dest="errorfile", help="error file name containing error messages");
|
|
|
206
|
|
|
207 # TEST
|
|
|
208 #sequences = 'NM_001169378.2__tc__atttcggatgctttggagggaggaactctagtgctgcattgattggggcgtgtgttaatgatattcccagttcgcatggcgagcatcgattcctggtacgtatgtgggccccttgactcccacttatcgcacttgtcgttcgcaatttgcatgaattccgcttcgtctgaaacgcacttgcgccagacttctccggctggtctgatctggtctgtgatccggtctggtggggcgccagttgcgtttcgagctcatcaccagtcactccgcagtcgcattctgccagaggtctccgatcaagagcgcttctccattcgagattcaaacgcagcgcggtctgacgccgccacatcgagtgaaatccatatcgatggccacattcacacaggacgagatcgacttcctgcgcagccatggcaacgagctgtgtgccaagacctggctgggattgtgggatccgaagcgggctgtgcaccagcaggagcagcgcgaactgatgatggacaagtatgagcggaagcgatactacctggagccggccagtcctcttaagtcgctggccaatgcggtcaacctgaagtcgtctgctccggcgacgaaccacactcagaatggccaccaaaatgggtatgccagcatccatttgacgcctcctgctgcccagcggacctcggccaatggattgcagaaggtggccaactcgtcgagtaactcttctggaaagacctcatcctcgatcagtaggccacactataatcaccagaacaacagccaaaacaacaatcacgatgcctttggcctgggtggcggattgagcagcctgaacagcgccggttccacatccactggagctctttccgacaccagcagttgtgctagcaatggcttcggtgcggactgcgactttgtggctgactttggctcggccaacattttcgacgccacatcggcgcgttccacaggatcgccggcggtgtcgtccgtgtcctcagtgggttccagcaatggctacgccaaggtgcagcccatccgggcagctcatctccagcagcaacagcagttgcagcagcagctgcatcagcagcagctcctcaatggcaatggtcatcagggcactgagaactttgccgacttcgatcacgctcccatctacaatgcagtggctccaccgacttttaacgattggatcagcgactggagcaggcggggcttccacgatcccttcgacgattgcgatgactcgccaccaggtgcccgccctccagcacctgcgccagctcctgctcaagttcccgcagtatcatcaccattgccaaccgtccgagaagaaccagagcttgcgtggaatttttgggaggacgagatgcgaatagaggcgcaggaaaaggagtcccaaactaaacagccggagttgggctactccttttcgattagtactactacgcccctttccccttcgaatcccttcctgccctaccttgtcagtgaggagcagcatcgaaatcatccagagaagccctccttttcgtattcgttgttcagctccatatcaaatagttcgcaagaagatcaggcggatgatcatgagatgaatgttttaaatgccaatttccatgatttctttacgtggagtgctcccttgcagaacggccatacgaccagtccgcccaagggcggaaatgcagcgatggcgcccagtgaggatcgatatgccgctcttaaggatctcgacgagcagctgcgagaactgaaggccagcgaaagcgccacagagacgcccacgcccaccagtggcaatgttcaggccacagatgcctttggtggagccctcaacaacaatccaaatcccttcaagggccagcaacagcagcagctcagcagccatgtggtgaatccattccagcagcagcaacagcagcagcaccagcagaatctctatggccagttgacgctcataccaaatgcctacggcagcagttcccagcagcagatggggcaccatctcctccagcagcagcagcagcaacagcagagcttcttcaacttcaacaacaacgggttcgccatctcgcagggtctgcccaacggctgcggcttcggcagcatgcaacccgctcctgtgatggccaacaatccctttgcagccagcggcgccatgaacaccaacaatccattcttatgagactcaacccgggagaatccgcctcgcgccacctggcagaggcgctgagccagcgaacaaagagcagacgcggaggaaccgaaccgaaattagtccattttactaacaatagcgttaatctatgtatacataatgcacgccggagagcactctttgtgtacatagcccaaatatgtacacccgaaaggctccacgctgacgctagtcctcgcggatggcggaggcggactggggcgttgatatattcttttacatggtaactctactctaacgtttacggatacggatatttgtatttgccgtttgccctagaactctatacttgtactaagcgcccatgaacacttcatccactaacatagctactaatcctcatcctagtggaggatgcagttggtccagacactctgttatttgttttatccatcctcgtacttgtctttgtcccatttagcactttcgttgcggataagaactttgtcagttattgattgtgtggccttaataagattataaaactaaatattataacgtacgactatacatatacggatacagatacagattcagacacagttagtacagatacagatatacatatacgcttttgtacctaatgaattgcttcttgtttccattgctaatcatctgcttttcgtgtgctaattttatacactagtacgtgcgatatcggccgtgcagatagattgctcagctcgcgagtcaagcctcttttggttgcacccacggcagacatttgtacatatactgtctgattgtaagcctcgtgtaatacctccattaacaccactcccccaccacccatccatcgaaccccgaatccatgactcaattcactgctcacatgtccatgcccatgccttaacgtgtcaaacattatcgaagccttaaagttatttaaaactacgaaatttcaataaaaacaaataagaacgctatc';
|
|
|
209 #(options, args) = parser.parse_args(['-x', 'rrr', '-t', 0.5, '-s', sequences, '-o', 'collection_content']);
|
|
|
210
|
|
|
211 (options, args) = parser.parse_args();
|
|
|
212 if options.version:
|
|
|
213 print __version__;
|
|
|
214 else:
|
|
|
215 # create output dir (collection)
|
|
|
216 output_dir_path = options.outputdir;
|
|
|
217 if not os.path.exists(output_dir_path):
|
|
|
218 os.makedirs(output_dir_path);
|
|
|
219
|
|
|
220 return query( options, args );
|
|
|
221
|
|
|
222 if __name__ == "__main__": __main__()
|
