Mercurial > repos > fabio > btman
changeset 16:ba9d0fc8657f draft
Uploaded 20190118
| author | fabio |
|---|---|
| date | Fri, 18 Jan 2019 10:12:19 -0500 |
| parents | 9d2b9e65d73e |
| children | f02c2c58a6f9 |
| files | ._.shed.yml ._create.py ._create.xml ._query.py ._query.xml create.py create.xml macros.xml query.py query.xml |
| diffstat | 10 files changed, 265 insertions(+), 12 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/create.py Fri Jan 18 10:12:19 2019 -0500 @@ -0,0 +1,132 @@ +#!/usr/bin/env python + +# https://github.com/ross/requests-futures +# http://docs.python-requests.org/en/master/user/quickstart/#more-complicated-post-requests + +import sys, os, uuid, optparse, requests, json, time +#from requests_futures.sessions import FuturesSession + +#### NN14 #### +SERVICE_URL = "http://nn14.galaxyproject.org:8080/"; +#service_url = "http://127.0.0.1:8082/"; +CREATE_URL = SERVICE_URL+"tree/create"; +STATUS_URL = SERVICE_URL+"status/<query_id>"; +############## +# query delay in seconds +QUERY_DELAY = 30; +############## + +__version__ = "1.0.0"; +ERR_EXIT_CODE = 1; +OK_EXIT_CODE = 0; + +def raiseException( exitcode, message, errorfilepath ): + with open(errorfilepath, 'w') as out: + out.write(message); + sys.exit(exitcode); + +def create_request( options, args, data ): + outfilepath = options.outfile; + cluster_id_2_query_id = { }; + + for cluster_id in data: + payload = { }; + payload["accessions"] = data[cluster_id]; + # add additional parameters to the payload + payload["qualitycontrol"] = int(options.qualitycontrol); + payload["qualitythreshold"] = float(options.qualitythreshold); + payload["klen"] = int(options.klen); + payload["minabundance"] = int(options.minabundance); + # set the content type to application/json + headers = {'Content-type': 'application/json'}; + # create a session + session = requests.Session(); + # make a synchronous post request to the create route + req = session.post(CREATE_URL, headers=headers, json=payload); + resp_code = req.status_code; + #print(str(req.content)+"\n\n"); + if resp_code == requests.codes.ok: + resp_content = str(req.content); + # convert out to json + json_content = json.loads(resp_content); + # retrieve query id + query_id = json_content['query_id']; + cluster_id_2_query_id[cluster_id] = query_id; + else: + with open(outfilepath, 'a+') as outfile: + outfile.write( "An error has occurred while submitting data to the /tree/create endpoint for the cluster " + cluster_id + "\n\n" ); + + build_flags = [ ] + while len(build_flags) < len(cluster_id_2_query_id): + for idx, cluster_id in enumerate( cluster_id_2_query_id ): + if cluster_id not in build_flags: + query_id = cluster_id_2_query_id[ cluster_id ]; + # create a new session + session = requests.Session(); + # make a synchronous get request to the status route + status_query_url = STATUS_URL.replace("<query_id>", query_id); + status_req = session.get(status_query_url); + status_resp_content = str(status_req.content); + #print(status_resp_content+"\n\n"); + # convert out to json + json_status_content = json.loads(status_resp_content); + # take a look at the state + # state attribute is always available + if json_status_content['state'] == 'SUCCESS': + build_flags.append( cluster_id ); + built_tree_id = json_status_content['results']['tree_id']; + with open(outfilepath, 'a+') as outfile: + outfile.write( "Query ID: " + str(query_id) + "\n" + "Query status: " + str(json_status_content['state']) + "\n" + "Cluster ID: " + cluster_id + "\n" + "Sequence Bloom Tree ID: " + built_tree_id + "\n\n" ); + elif json_status_content['state'] in ['FAILURE', 'REVOKED']: + build_flags.append( cluster_id ); + with open(outfilepath, 'a+') as outfile: + outfile.write( "Query ID: " + str(query_id) + "\n" + "Query status: " + str(json_status_content['state']) + "\n" + "Cluster ID: " + cluster_id + "\n\n" ); + if len(build_flags) < len(cluster_id_2_query_id): + time.sleep(QUERY_DELAY); # in seconds + return sys.exit(OK_EXIT_CODE); + +def create( options, args ): + multiple_data = {}; + experiment_list_file_path = options.explist; + with open(experiment_list_file_path) as explist: + for line in explist: + if line.strip() != "": + line_split = line.strip().split("\t"); # split on tab + if len(line_split) == 2: # 0:accession , 1:cluster_id , otherwise skip line + accession = line_split[0]; + cluster_id = line_split[1]; + if cluster_id in multiple_data: + multiple_data[cluster_id].append( accession ); + else: + multiple_data[cluster_id] = [ accession ]; + if len(multiple_data) > 0: + return create_request( options, args, multiple_data ); + else: + return raiseException( ERR_EXIT_CODE, "An error has occurred. Please be sure that your input file is valid.", options.outfile ); + +def __main__(): + # Parse the command line options + usage = "Usage: create.py --explist experiment_list --qualitycontrol quality_control --qualitythreshold quality_threshold --klen kmer_len --minabundance min_abundance --outfile output_file_path"; + parser = optparse.OptionParser(usage = usage); + parser.add_option("-v", "--version", action="store_true", dest="version", + default=False, help="display version and exit") + parser.add_option("-l", "--explist", type="string", + action="store", dest="explist", help="tabular file with a list of SRA accessions and their cluster label"); + parser.add_option("-q", "--qualitycontrol", type="int", default=0 + action="store", dest="qualitycontrol", help="flag to enable or disable the experiment quality control"); + parser.add_option("-t", "--qualitythreshold", type="float", default=0.0 + action="store", dest="qualitythreshold", help="quality threshold, if quality control is enabled only"); + parser.add_option("-k", "--klen", type="int", default=21, + action="store", dest="klen", help="k-mer length"); + parser.add_option("-m", "--minabundance", type="int", default=2, + action="store", dest="minabundance", help="minimum abundance"); + parser.add_option("-o", "--outfile", type="string", default="outfile_txt", + action="store", dest="outfile", help="output file path"); + + (options, args) = parser.parse_args(); + if options.version: + print __version__; + else: + return create( options, args ); + +if __name__ == "__main__": __main__()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/create.xml Fri Jan 18 10:12:19 2019 -0500 @@ -0,0 +1,104 @@ +<?xml version="1.0"?> +<tool name="BloomTree Manager - Create" id="btman_create" version="1.0.0"> + <description>a Sequence Bloom Tree</description> + <macros> + <import>macros.xml</import> + </macros> + <expand macro="requirements" /> + <command detect_errors="exit_code"> +<![CDATA[ + python '$__tool_directory__/create.py' + + --explist '${explist}' + + --qualitycontrol ${conditional_quality.quality_control} + #if $conditional_quality.quality_control == '0': + --qualitythreshold 0.0 + #elif $conditional_quality.quality_control == '1': + --qualitythreshold ${conditional_quality.quality_threshold} + #end if + + --klen ${kmer_len} + --minabundance ${min_abundance} + + --outfile '${outfile}' +]]> + </command> + <inputs> + <param format="tabular" name="explist" type="data" label="Select a file with the list of experiments" help="This should be a tabular file with two columns. Take a look at the tool documentation for a detailed explanation about the its content." /> + + <conditional name="conditional_quality"> + <param name="quality_control" type="boolean" checked="false" truevalue="1" falsevalue="0" label="Apply a quality control procedure" /> + <when value="1"> + <param name="quality_threshold" size="1" type="float" value="0.8" min="0.0" max="1.0" label="Quality threshold" help="If the number of sequences flagged as poor quality on the total number of sequences in a file is less than this threshold, the sequence file will be excluded." /> + </when> + </conditional> + + <param name="kmer_len" type="integer" value="21" min="0" label="K-mer length" /> + <param name="min_abundance" type="integer" value="2" min="0" label="Bloom filter minimum abundance" help="This value is the minimum abundance cutoff for the creation of the Bloom filters. It is worth noting that the same minimum abundance is used for each Bloom filter." /> + </inputs> + <outputs> + <data format="txt" name="outfile" label="${tool.name} SBT: Result" from_work_dir="btman.create.txt" /> + </outputs> + + <help><![CDATA[ +This tool is part of the BloomTree Manager Framework that allow to create a Sequence Bloom Tree starting +with a set of FASTA or FASTQ files. It allows also to control the quality of the input dataset and +exclude the files that do not reach a specified quality level. + +----- + +**Input file** + +The input file for this tool must contain two columns with their values delimited by a tab. +The first column contains a list of SRA accessions, and the second column contains a unique identifier +for each set of SRA accessions. + +The input file is structured like the example below:: + + SRR805782 blood + SRR837459 blood + SRR837458 blood + SRR837453 blood + SRR837456 blood + ... + SRR791048 breast + SRR553483 breast + SRR553482 breast + SRR791045 breast + ... + SRR950876 brain + SRR786621 brain + +It is worth noting that for each cluster of accessions, every accession should be unique. +It is indeed possible to repeat an accession in multiple clusters. + +The tool will create a Sequence Bloom Tree for each cluster of accessions. + +----- + +**Output** + +The tool returns a single text file only. It contains the a tree identifier, one for +each cluster of accessions specified in the input file, that can be used with the +Query tool of the BloomTree Manager Suite to search for the presence of a set of +specific transcripts. + +Take a look at the Query tool documentation for a detailed description about how +to query a Sequence Bloom Tree. + +----- + +.. class:: infomark + +**Notes** + +This Galaxy tool has been developed by Fabio Cumbo. + +Please visit this GithHub_repository_ for more information about the BloomTree Manager + +.. _GithHub_repository: https://github.com/fabio-cumbo/bloomtree-manager + ]]></help> + + <expand macro="citations" /> +</tool>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/macros.xml Fri Jan 18 10:12:19 2019 -0500 @@ -0,0 +1,14 @@ +<macros> + <xml name="requirements"> + <requirements> + <requirement type="package" version="2.7.10">python</requirement> + <requirement type="package" version="2.18.4">requests</requirement> + </requirements> + </xml> + + <xml name="citations"> + <citations> + <citation type="doi">10.1101/090464</citation> + </citations> + </xml> +</macros> \ No newline at end of file
--- a/query.py Tue Apr 24 13:29:16 2018 -0400 +++ b/query.py Fri Jan 18 10:12:19 2019 -0500 @@ -42,7 +42,6 @@ # create a session session = requests.Session(); # make a synchronous post request to the query route - QUERY_URL.replace("<tree_id>", str(options.treeid)); req = session.post(QUERY_URL.replace("<tree_id>", str(options.treeid)), headers=headers, json=payload); resp_code = req.status_code; #print(str(req.content)+"\n\n"); @@ -193,7 +192,7 @@ action="store", dest="search", help="search mode"); parser.add_option("-e", "--exact", type="int", default=0, action="store", dest="exact", help="exact algorithm (required if search is 1 only)"); - parser.add_option("-k", "--tree", type="int", default=0, + parser.add_option("-k", "--tree", type="string", default=0, action="store", dest="treeid", help="the id of the tree that will be queried"); parser.add_option("-t", "--sthreshold", type="float", action="store", dest="sthreshold", help="threshold applied to the search algrithm");
--- a/query.xml Tue Apr 24 13:29:16 2018 -0400 +++ b/query.xml Fri Jan 18 10:12:19 2019 -0500 @@ -1,15 +1,15 @@ <?xml version="1.0"?> <tool name="BloomTree Manager - Query" id="btman_query" version="1.0.0"> - <description>the Sequence Bloom Tree</description> - <requirements> - <requirement type="package" version="2.7.10">python</requirement> - <requirement type="package" version="2.18.4">requests</requirement> - </requirements> + <description>a Sequence Bloom Tree</description> + <macros> + <import>macros.xml</import> + </macros> + <expand macro="requirements" /> <command detect_errors="exit_code"> <![CDATA[ python '$__tool_directory__/query.py' - --tree 1 + --tree '${treeid}' --search 'rrr' --sthreshold ${sthreshold} --sort ${sortcontrol} @@ -42,8 +42,11 @@ <param name="sequences" type="text" area="True" size="5x25" label="Manually insert sequences" optional="false" help="Insert a list of (ID, TRANSCRIPT) couples in a tab delimited format, one for each line. The content of this text box will represent a query to the Sequence Bloom Tree that will return a collection containing a file for each ID. The content of these files as result of the tool will be a list of accession numbers." /> </when> </conditional> + <param name="sthreshold" size="3" type="float" value="0.7" min="0.0" max="1.0" label="Search threshold" help="This threshold controls the specificity. Lower values will produce more hits to the query. Higher values are more stringent and will produce fewer hits." /> <param name="sortcontrol" type="boolean" checked="true" truevalue="1" falsevalue="0" label="Sort the result by the number of hits per transcript." /> + + <param name="treeid" size="30" type="text" value="" label="Sequence Bloom Tree identifier" help="Set this field according to the result of the Create tool of the BloomTree Manager Suite." /> </inputs> <outputs> <collection name="output_collect" type="list" label="BloomTree Manager - Query result collection"> @@ -52,7 +55,7 @@ </outputs> <help><![CDATA[ -This Query tool is part of the BloomTree Manager Framework that allow to rapidly identify all publicly available +This tool is part of the BloomTree Manager Framework that allow to rapidly identify all sequenced samples which express a transcript of interest. ---- @@ -68,6 +71,9 @@ The ID can contain alphanumeric characters in addition to spaces, dots, dashes, and round and square brackets. Any additional character will be trimmed out. +The Sequence Bloom Tree identifier must be also specified. It is a string that identify an existing Sequence +Bloom Tree, which should be built with the Create tool of the BloomTree Manager Suite. + The output of the tool is a collection that contains a file for each ID with a list of accession numbers representing the samples that express one particular transcript. @@ -84,7 +90,5 @@ .. _GithHub_repository: https://github.com/fabio-cumbo/bloomtree-manager ]]></help> - <citations> - <citation type="doi">10.1101/090464</citation> - </citations> + <expand macro="citations" /> </tool>