# HG changeset patch
# User fabio
# Date 1547824339 18000
# Node ID ba9d0fc8657fedd077aa4604060d51d4f477adb4
# Parent  9d2b9e65d73e1cc0e1cd36daddf7e164c01d1cfe
Uploaded 20190118

diff -r 9d2b9e65d73e -r ba9d0fc8657f ._.shed.yml
Binary file ._.shed.yml has changed
diff -r 9d2b9e65d73e -r ba9d0fc8657f ._create.py
Binary file ._create.py has changed
diff -r 9d2b9e65d73e -r ba9d0fc8657f ._create.xml
Binary file ._create.xml has changed
diff -r 9d2b9e65d73e -r ba9d0fc8657f ._query.py
Binary file ._query.py has changed
diff -r 9d2b9e65d73e -r ba9d0fc8657f ._query.xml
Binary file ._query.xml has changed
diff -r 9d2b9e65d73e -r ba9d0fc8657f create.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/create.py	Fri Jan 18 10:12:19 2019 -0500
@@ -0,0 +1,132 @@
+#!/usr/bin/env python
+
+# https://github.com/ross/requests-futures
+# http://docs.python-requests.org/en/master/user/quickstart/#more-complicated-post-requests
+
+import sys, os, uuid, optparse, requests, json, time
+#from requests_futures.sessions import FuturesSession
+
+#### NN14 ####
+SERVICE_URL = "http://nn14.galaxyproject.org:8080/";
+#service_url = "http://127.0.0.1:8082/";
+CREATE_URL = SERVICE_URL+"tree/create";
+STATUS_URL = SERVICE_URL+"status/<query_id>";
+##############
+# query delay in seconds
+QUERY_DELAY = 30;
+##############
+
+__version__ = "1.0.0";
+ERR_EXIT_CODE = 1;
+OK_EXIT_CODE = 0;
+
+def raiseException( exitcode, message, errorfilepath ):
+    with open(errorfilepath, 'w') as out:
+        out.write(message);
+    sys.exit(exitcode);
+
+def create_request( options, args, data ):
+    outfilepath = options.outfile;
+    cluster_id_2_query_id = { };
+
+    for cluster_id in data:
+        payload = { };
+        payload["accessions"] = data[cluster_id];
+        # add additional parameters to the payload
+        payload["qualitycontrol"] = int(options.qualitycontrol);
+        payload["qualitythreshold"] = float(options.qualitythreshold);
+        payload["klen"] = int(options.klen);
+        payload["minabundance"] = int(options.minabundance);
+        # set the content type to application/json
+        headers = {'Content-type': 'application/json'};
+        # create a session
+        session = requests.Session();
+        # make a synchronous post request to the create route
+        req = session.post(CREATE_URL, headers=headers, json=payload);
+        resp_code = req.status_code;
+        #print(str(req.content)+"\n\n");
+        if resp_code == requests.codes.ok:
+            resp_content = str(req.content);
+            # convert out to json
+            json_content = json.loads(resp_content);
+            # retrieve query id
+            query_id = json_content['query_id'];
+            cluster_id_2_query_id[cluster_id] = query_id;
+        else:
+            with open(outfilepath, 'a+') as outfile:
+                outfile.write( "An error has occurred while submitting data to the /tree/create endpoint for the cluster " + cluster_id + "\n\n" );
+
+    build_flags = [ ]
+    while len(build_flags) < len(cluster_id_2_query_id):
+        for idx, cluster_id in enumerate( cluster_id_2_query_id ):
+            if cluster_id not in build_flags:
+                query_id = cluster_id_2_query_id[ cluster_id ];
+                # create a new session
+                session = requests.Session();
+                # make a synchronous get request to the status route
+                status_query_url = STATUS_URL.replace("<query_id>", query_id);
+                status_req = session.get(status_query_url);
+                status_resp_content = str(status_req.content);
+                #print(status_resp_content+"\n\n");
+                # convert out to json
+                json_status_content = json.loads(status_resp_content);
+                # take a look at the state
+                # state attribute is always available
+                if json_status_content['state'] == 'SUCCESS':
+                    build_flags.append( cluster_id );
+                    built_tree_id = json_status_content['results']['tree_id'];
+                    with open(outfilepath, 'a+') as outfile:
+                        outfile.write( "Query ID: " + str(query_id) + "\n" + "Query status: " + str(json_status_content['state']) + "\n" + "Cluster ID: " + cluster_id + "\n" + "Sequence Bloom Tree ID: " + built_tree_id + "\n\n" );
+                elif json_status_content['state'] in ['FAILURE', 'REVOKED']:
+                    build_flags.append( cluster_id );
+                    with open(outfilepath, 'a+') as outfile:
+                        outfile.write( "Query ID: " + str(query_id) + "\n" + "Query status: " + str(json_status_content['state']) + "\n" + "Cluster ID: " + cluster_id + "\n\n" );
+        if len(build_flags) < len(cluster_id_2_query_id):
+            time.sleep(QUERY_DELAY); # in seconds
+    return sys.exit(OK_EXIT_CODE);
+
+def create( options, args ):
+    multiple_data = {};
+    experiment_list_file_path = options.explist;
+    with open(experiment_list_file_path) as explist:
+        for line in explist:
+            if line.strip() != "":
+                line_split = line.strip().split("\t"); # split on tab
+                if len(line_split) == 2: # 0:accession , 1:cluster_id , otherwise skip line
+                    accession = line_split[0];
+                    cluster_id = line_split[1];
+                    if cluster_id in multiple_data:
+                        multiple_data[cluster_id].append( accession );
+                    else:
+                        multiple_data[cluster_id] = [ accession ];
+    if len(multiple_data) > 0:
+        return create_request( options, args, multiple_data );
+    else:
+        return raiseException( ERR_EXIT_CODE, "An error has occurred. Please be sure that your input file is valid.", options.outfile );
+
+def __main__():
+    # Parse the command line options
+    usage = "Usage: create.py --explist experiment_list --qualitycontrol quality_control --qualitythreshold quality_threshold --klen kmer_len --minabundance min_abundance --outfile output_file_path";
+    parser = optparse.OptionParser(usage = usage);
+    parser.add_option("-v", "--version", action="store_true", dest="version",
+                    default=False, help="display version and exit")
+    parser.add_option("-l", "--explist", type="string",
+                    action="store", dest="explist", help="tabular file with a list of SRA accessions and their cluster label");
+    parser.add_option("-q", "--qualitycontrol", type="int", default=0
+                    action="store", dest="qualitycontrol", help="flag to enable or disable the experiment quality control");
+    parser.add_option("-t", "--qualitythreshold", type="float", default=0.0
+                    action="store", dest="qualitythreshold", help="quality threshold, if quality control is enabled only");
+    parser.add_option("-k", "--klen", type="int", default=21,
+                    action="store", dest="klen", help="k-mer length");
+    parser.add_option("-m", "--minabundance", type="int", default=2,
+                    action="store", dest="minabundance", help="minimum abundance");
+    parser.add_option("-o", "--outfile", type="string", default="outfile_txt",
+                    action="store", dest="outfile", help="output file path");
+
+    (options, args) = parser.parse_args();
+    if options.version:
+        print __version__;
+    else:
+        return create( options, args );
+
+if __name__ == "__main__": __main__()
diff -r 9d2b9e65d73e -r ba9d0fc8657f create.xml
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/create.xml	Fri Jan 18 10:12:19 2019 -0500
@@ -0,0 +1,104 @@
+<?xml version="1.0"?>
+<tool name="BloomTree Manager - Create" id="btman_create" version="1.0.0">
+    <description>a Sequence Bloom Tree</description>
+    <macros>
+        <import>macros.xml</import>
+    </macros>
+    <expand macro="requirements" />
+    <command detect_errors="exit_code">
+<![CDATA[
+    python '$__tool_directory__/create.py'
+    
+    --explist '${explist}'
+
+    --qualitycontrol ${conditional_quality.quality_control}
+    #if $conditional_quality.quality_control == '0':
+        --qualitythreshold 0.0
+    #elif $conditional_quality.quality_control == '1':
+        --qualitythreshold ${conditional_quality.quality_threshold}
+    #end if
+
+    --klen ${kmer_len}
+    --minabundance ${min_abundance}
+
+    --outfile '${outfile}'
+]]>
+    </command>
+    <inputs>
+        <param format="tabular" name="explist" type="data" label="Select a file with the list of experiments" help="This should be a tabular file with two columns. Take a look at the tool documentation for a detailed explanation about the its content." />
+
+        <conditional name="conditional_quality">
+            <param name="quality_control" type="boolean" checked="false" truevalue="1" falsevalue="0" label="Apply a quality control procedure" />
+            <when value="1">
+                <param name="quality_threshold" size="1" type="float" value="0.8" min="0.0" max="1.0" label="Quality threshold" help="If the number of sequences flagged as poor quality on the total number of sequences in a file is less than this threshold, the sequence file will be excluded." />
+            </when>
+        </conditional>
+
+        <param name="kmer_len" type="integer" value="21" min="0" label="K-mer length" />
+        <param name="min_abundance" type="integer" value="2" min="0" label="Bloom filter minimum abundance" help="This value is the minimum abundance cutoff for the creation of the Bloom filters. It is worth noting that the same minimum abundance is used for each Bloom filter." />
+    </inputs>
+    <outputs>
+        <data format="txt" name="outfile" label="${tool.name} SBT: Result" from_work_dir="btman.create.txt" />
+    </outputs>
+
+    <help><![CDATA[
+This tool is part of the BloomTree Manager Framework that allow to create a Sequence Bloom Tree starting 
+with a set of FASTA or FASTQ files. It allows also to control the quality of the input dataset and 
+exclude the files that do not reach a specified quality level.
+
+-----
+
+**Input file**
+
+The input file for this tool must contain two columns with their values delimited by a tab.
+The first column contains a list of SRA accessions, and the second column contains a unique identifier
+for each set of SRA accessions.
+
+The input file is structured like the example below::
+    
+    SRR805782  blood
+    SRR837459  blood
+    SRR837458  blood
+    SRR837453  blood
+    SRR837456  blood
+    ...
+    SRR791048  breast
+    SRR553483  breast
+    SRR553482  breast
+    SRR791045  breast
+    ...
+    SRR950876  brain
+    SRR786621  brain
+
+It is worth noting that for each cluster of accessions, every accession should be unique.
+It is indeed possible to repeat an accession in multiple clusters.
+
+The tool will create a Sequence Bloom Tree for each cluster of accessions.
+
+-----
+
+**Output**
+
+The tool returns a single text file only. It contains the a tree identifier, one for
+each cluster of accessions specified in the input file, that can be used with the
+Query tool of the BloomTree Manager Suite to search for the presence of a set of
+specific transcripts.
+
+Take a look at the Query tool documentation for a detailed description about how
+to query a Sequence Bloom Tree.
+
+-----
+
+.. class:: infomark
+
+**Notes**
+
+This Galaxy tool has been developed by Fabio Cumbo.
+
+Please visit this GithHub_repository_ for more information about the BloomTree Manager
+
+.. _GithHub_repository: https://github.com/fabio-cumbo/bloomtree-manager
+    ]]></help>
+
+    <expand macro="citations" />
+</tool>
diff -r 9d2b9e65d73e -r ba9d0fc8657f macros.xml
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/macros.xml	Fri Jan 18 10:12:19 2019 -0500
@@ -0,0 +1,14 @@
+<macros>
+    <xml name="requirements">
+        <requirements>
+            <requirement type="package" version="2.7.10">python</requirement>
+            <requirement type="package" version="2.18.4">requests</requirement>
+        </requirements>
+    </xml>
+    
+    <xml name="citations">
+        <citations>
+            <citation type="doi">10.1101/090464</citation>
+        </citations>
+    </xml>
+</macros>
\ No newline at end of file
diff -r 9d2b9e65d73e -r ba9d0fc8657f query.py
--- a/query.py	Tue Apr 24 13:29:16 2018 -0400
+++ b/query.py	Fri Jan 18 10:12:19 2019 -0500
@@ -42,7 +42,6 @@
     # create a session
     session = requests.Session();
     # make a synchronous post request to the query route
-    QUERY_URL.replace("<tree_id>", str(options.treeid));
     req = session.post(QUERY_URL.replace("<tree_id>", str(options.treeid)), headers=headers, json=payload);
     resp_code = req.status_code;
     #print(str(req.content)+"\n\n");
@@ -193,7 +192,7 @@
                     action="store", dest="search", help="search mode");
     parser.add_option("-e", "--exact", type="int", default=0,
                     action="store", dest="exact", help="exact algorithm (required if search is 1 only)");
-    parser.add_option("-k", "--tree", type="int", default=0,
+    parser.add_option("-k", "--tree", type="string", default=0,
                     action="store", dest="treeid", help="the id of the tree that will be queried");
     parser.add_option("-t", "--sthreshold", type="float",
                     action="store", dest="sthreshold", help="threshold applied to the search algrithm");
diff -r 9d2b9e65d73e -r ba9d0fc8657f query.xml
--- a/query.xml	Tue Apr 24 13:29:16 2018 -0400
+++ b/query.xml	Fri Jan 18 10:12:19 2019 -0500
@@ -1,15 +1,15 @@
 <?xml version="1.0"?>
 <tool name="BloomTree Manager - Query" id="btman_query" version="1.0.0">
-    <description>the Sequence Bloom Tree</description>
-    <requirements>
-        <requirement type="package" version="2.7.10">python</requirement>
-        <requirement type="package" version="2.18.4">requests</requirement>
-    </requirements>
+    <description>a Sequence Bloom Tree</description>
+    <macros>
+        <import>macros.xml</import>
+    </macros>
+    <expand macro="requirements" />
     <command detect_errors="exit_code">
 <![CDATA[
     python '$__tool_directory__/query.py'
     
-    --tree 1
+    --tree '${treeid}'
     --search 'rrr'
     --sthreshold ${sthreshold}
     --sort ${sortcontrol}
@@ -42,8 +42,11 @@
                 <param name="sequences" type="text" area="True" size="5x25" label="Manually insert sequences" optional="false" help="Insert a list of (ID, TRANSCRIPT) couples in a tab delimited format, one for each line. The content of this text box will represent a query to the Sequence Bloom Tree that will return a collection containing a file for each ID. The content of these files as result of the tool will be a list of accession numbers." />
             </when>
         </conditional>            
+        
         <param name="sthreshold" size="3" type="float" value="0.7" min="0.0" max="1.0" label="Search threshold" help="This threshold controls the specificity. Lower values will produce more hits to the query. Higher values are more stringent and will produce fewer hits." />
         <param name="sortcontrol" type="boolean" checked="true" truevalue="1" falsevalue="0" label="Sort the result by the number of hits per transcript." />
+
+        <param name="treeid" size="30" type="text" value="" label="Sequence Bloom Tree identifier" help="Set this field according to the result of the Create tool of the BloomTree Manager Suite." />
     </inputs>
     <outputs>
         <collection name="output_collect" type="list" label="BloomTree Manager - Query result collection">
@@ -52,7 +55,7 @@
     </outputs>
 
     <help><![CDATA[
-This Query tool is part of the BloomTree Manager Framework that allow to rapidly identify all publicly available 
+This tool is part of the BloomTree Manager Framework that allow to rapidly identify all 
 sequenced samples which express a transcript of interest.
 
 ----
@@ -68,6 +71,9 @@
 The ID can contain alphanumeric characters in addition to spaces, dots, dashes, and round and square brackets.
 Any additional character will be trimmed out.
 
+The Sequence Bloom Tree identifier must be also specified. It is a string that identify an existing Sequence
+Bloom Tree, which should be built with the Create tool of the BloomTree Manager Suite.
+
 The output of the tool is a collection that contains a file for each ID with a list of
 accession numbers representing the samples that express one particular transcript.
 
@@ -84,7 +90,5 @@
 .. _GithHub_repository: https://github.com/fabio-cumbo/bloomtree-manager
     ]]></help>
 
-    <citations>
-        <citation type="doi">10.1101/090464</citation>
-    </citations>
+    <expand macro="citations" />
 </tool>