--- a/gemini_annotate.xml	Fri Jan 11 17:41:39 2019 -0500
+++ b/gemini_annotate.xml	Fri Jan 18 19:34:41 2019 -0500
@@ -1,8 +1,16 @@
-<tool id="gemini_@BINARY@" name="GEMINI @BINARY@" version="@VERSION@">
-    <description>adding your own custom annotations</description>
+<tool id="gemini_@BINARY@" name="GEMINI @BINARY@" version="@VERSION@+galaxy1">
+    <description>the variants in an existing GEMINI database with additional information</description>
     <macros>
         <import>gemini_macros.xml</import>
         <token name="@BINARY@">annotate</token>
+        <xml name="add_as">
+            <param argument="-c" name="column_name" type="text" value=""
+            label="Database column name to use for recording annotations"
+            help="A column with the name provided here will be added to the variants table of the GEMINI database to store the annotations">
+                <validator type="empty_field" />
+                <validator type="expression" message="Only alphanumeric characters and the underscore can be used in column names">value.replace('_', '').isalnum()</validator>
+            </param>
+        </xml>
     </macros>
     <expand macro="requirements" />
     <expand macro="stdio" />
@@ -18,73 +26,86 @@
 
     gemini @BINARY@
         -f '$tabixed_file'
-        -c '$column_name'
         -a $a.a_selector
         #if $a.a_selector == 'extract':
-            -t $a.column_type
-            -e '$a.column_extracts'
-            -o $a.operation
+            #set $types = []
+            #set $elements = []
+            #set $column_names = []
+            #set $operations = []
+            #for $action in $a.actions:
+                $types.append(str($action.column_type))
+                $elements.append(str($action.element_to_extract).strip())
+                $column_names.append(str($action.column_name).strip())
+                $operations.append(str($action.operation))
+            #end for
+
+            -t #echo ",".join($types)
+            -e '#echo ",".join($elements)#'
+            -o #echo ",".join($operations)
+            -c '#echo ",".join($column_names)#'
+        #else:
+            -c '${a.column_name}'
         #end if
-        $region_only
+        #if str($annotate_source.ext) == "vcf":
+            $region_only
+        #end if
         '$outfile'
 ]]>
-
     </command>
     <inputs>
         <expand macro="infile" />
-        <param name="annotate_source" type="data" format="vcf,bed" label="File containing the annotations in BED/VCF format" help="(-f)"/>
-
-        <param name="column_name" type="text" value=""
-            label="The name of the column to be added to the variant table" 
-            help="If the input file is a VCF, then this is the name of the info field to pull. (-c)">
-            <sanitizer invalid_char=" ">
-                <valid initial="string.letters,string.digits">
-                    <add value="_" />
-                </valid>
-            </sanitizer>
-        </param>
+        <param argument="-f" name="annotate_source" type="data" format="vcf,bed"
+        label="Dataset to use as the annotation source"
+        help="The tool can use the information from a BED or VCF dataset to annotate the database variants."/>
+        <param argument="--region-only" name="region_only" type="boolean" checked="true" truevalue="" falsevalue="--region-only"
+        label="Strict variant-identity matching of database and annotation records (VCF format only)"
+        help="The default is to consider VCF-formatted annotations only if a variant in the GEMINI database and a record in the annotation source describe the exact same nucleotide change at the same position in the genome. You can disable this option to make use of any annotation that overlaps with the position of a database variant. This setting is ignored for annotation sources in BED format, for which matching is always based on overlapping positions only." />
         <conditional name="a">
-            <param name="a_selector" type="select" label="How should the annotation file be used?" help="(-a)">
-                <option value="boolean">Did a variant overlap a region or not? (boolean)</option>
-                <option value="count">How many regions did a variant overlap? (count)</option>
-                <option value="extract" selected="True">Extract specific values from a BED/VCF file. (extract)</option>
+            <param argument="-a" name="a_selector" type="select"
+            label="Type of information to add to the database variants"
+            help="">
+                <option value="boolean">Binary indicator (1=found, 0=not found) of whether the variant had any match in the annotation source (boolean)</option>
+                <option value="count">Count of the number of matches found in the annotation source for the database variant (count)</option>
+                <option value="extract" selected="True">Specific values extracted from matching records in the annotation source (extract)</option>
             </param>
             <when value="extract">
-
-                <param name="column_extracts" label="Column to extract information from for list annotations. For BED files, this is the column number. For VCF files, this is the name of the INFO field."
-                    type="text" force_select="true" help="(-e)"/>
-
-
-                <param name="column_type" type="select" label="What data type(s) should be used to represent the new values in the database?"
-                    help="(-t)">
-                    <option value="float">Decimal precision number (float)</option>
-                    <option value="integer">Integer number (integer)</option>
-                    <option value="text">Text columns such as “valid”, “yes” (text)</option>
-                </param>
-
-                <param name="operation" type="select" label="Operation to apply to the extract column values ..."
-                    help="in the event that a variant overlaps multiple annotations in your annotation file. (-o)">
-                    <option value="mean">Compute the average of the (numeric) values</option>
-                    <option value="sum">Compute the sum of the (numeric) values</option>
-                    <option value="median">Compute the median of the (numeric) values</option>
-                    <option value="min">Compute the minimum of the (numeric) values</option>
-                    <option value="max">Compute the maximum of the (numeric) values</option>
-                    <option value="mode">Compute the maximum of the (numeric) values</option>
-                    <option value="first">Use the value from the first record in the annotation file</option>
-                    <option value="last">Use the value from the last record in the annotation file</option>
-                    <option value="list">Create a comma-separated list of the observed (text) values</option>
-                    <option value="uniq_list">Create a comma-separated list of non-redundant observed (text) values</option>
-                </param>
-
+                <repeat name="actions" title="Annotation extraction recipe" default="1" min="1">
+                    <param argument="-e" name="element_to_extract" type="text" value=""
+                    label="Elements to extract from the annotation source"
+                    help="For an annotation source in BED format, specify the number of the column from which the annotations should be read. For a VCF source, name an INFO field element.">
+                        <validator type="expression" message="This field cannot be empty">value.strip()</validator>
+                    </param>
+                    <expand macro="add_as" />
+                    <param argument="-t" name="column_type" type="select" display="radio"
+                    label="What type of data are you trying to extract?"
+                    help="Your selection will determine the data type used to store the new annotations in the database.">
+                        <option value="float">Numbers with decimal precision</option>
+                        <option value="integer">Integer numbers</option>
+                        <option value="text">Text (text)</option>
+                    </param>
+                    <param argument="-o" name="operation" type="select"
+                    label="If multiple annotations are found for the same variant, store ..."
+                    help="Note: If indicated (in parentheses) an option is only applicable to annotations of a specific type.">
+                        <option value="first">the first annotation found</option>
+                        <option value="last">the last annotation found</option>
+                        <option value="list">a comma-separated list of the (text) values</option>
+                        <option value="uniq_list">a comma-separated list of non-redundant (text) values</option>
+                        <option value="min">the smallest of the (numeric) values</option>
+                        <option value="max">the largest of the (numeric) values</option>
+                        <option value="mode">the most frequent of the (numeric) values</option>
+                        <option value="mean">the mean of the (numeric) values</option>
+                        <option value="median">the median of the (numeric) values</option>
+                        <option value="sum">the sum of the (numeric) values</option>
+                    </param>
+                </repeat>
             </when>
-            <when value="boolean"/>
-            <when value="count"/>
+            <when value="boolean">
+                <expand macro="add_as" />
+            </when>
+            <when value="count">
+                <expand macro="add_as" />
+            </when>
         </conditional>
-        <param name="region_only" argument="--region-only" type="boolean" checked="false"
-            truevalue="--region-only" falsevalue=""
-            label="If set, only region coordinates will be considered when annotating variants."
-            help="The default is to annotate using region coordinates as well as REF and ALT
-                variant values. This option is only valid if annotation is a VCF file"/>
     </inputs>
     <outputs>
         <data name="outfile" format="gemini.sqlite" />
@@ -93,125 +114,71 @@
         <test>
             <param name="infile" value="gemini_load_result1.db" ftype="gemini.sqlite" />
             <param name="annotate_source" value="anno.bed" />
-            <param name="a_selector" value="count" />
-            <param name="column_name" value="anno5" />
+            <param name="region_only" value="false" />
+            <conditional name="a">
+                <param name="a_selector" value="count" />
+                <param name="column_name" value="anno5" />
+            </conditional>
+            <assert_command>
+                <not_has_text text="--region-only" />
+            </assert_command>
             <output name="outfile" file="gemini_annotate_result.db" ftype="gemini.sqlite" compare="sim_size" delta="1000"/>
         </test>
+        <test>
+            <param name="infile" value="gemini_load_result1.db" ftype="gemini.sqlite" />
+            <param name="annotate_source" value="anno.bed" />
+            <conditional name="a">
+                <param name="a_selector" value="extract" />
+                <repeat name="actions">
+                    <param name="element_to_extract" value="1" />
+                    <param name="column_name" value="anno5" />
+                    <param name="column_type" value="text" />
+                    <param name="operation" value="first" />
+                </repeat>
+            </conditional>
+            <assert_command>
+                <has_text_matching expression="-a +extract" />
+                <has_text_matching expression="-e +1" />
+                <has_text_matching expression="-c +anno5" />
+                <has_text_matching expression="-t +text" />
+                <has_text_matching expression="-o +first" />
+            </assert_command>
+        </test>
+        <test>
+            <param name="infile" value="gemini_load_result1.db" ftype="gemini.sqlite" />
+            <param name="annotate_source" value="anno.bed" />
+            <conditional name="a">
+                <param name="a_selector" value="extract" />
+                <repeat name="actions">
+                    <param name="element_to_extract" value="1" />
+                    <param name="column_name" value="anno5" />
+                    <param name="column_type" value="text" />
+                    <param name="operation" value="first" />
+                </repeat>
+                <repeat name="actions">
+                    <param name="element_to_extract" value="2" />
+                    <param name="column_name" value="anno6" />
+                    <param name="column_type" value="integer" />
+                    <param name="operation" value="first" />
+                </repeat>
+            </conditional>
+            <assert_command>
+                <has_text_matching expression="-a +extract" />
+                <has_text_matching expression="-e +1,2" />
+                <has_text_matching expression="-c +anno5,anno6" />
+                <has_text_matching expression="-t +text,integer" />
+                <has_text_matching expression="-o +first,first" />
+            </assert_command>
+        </test>
     </tests>
     <help><![CDATA[
 **What it does**
 
-It is inevitable that researchers will want to enhance the GEMINI framework with their own, custom annotations. GEMINI provides a sub-command called annotate for exactly this purpose.
-
-**Details**
-
-It is inevitable that researchers will want to enhance the GEMINI framework with their own, custom annotations. GEMINI provides a sub-command called annotate for exactly this purpose. As long as you provide a tabix‘ed annotation file in BED or VCF format, the annotate tool will, for each variant in the variants table, screen for overlaps in your annotation file and update a one or more new column in the variants table that you may specify on the command line. This is best illustrated by a following **example**.
-
-**Input files**
-
-Let’s assume you have already created a GEMINI database of a **VCF file** using the *load module*.
-
-Now, let’s imagine you have an annotated file in **BED format** (important.bed) that describes regions of the genome that are particularly relevant to your lab’s research. You would like to annotate in the GEMINI database which variants overlap these crucial regions. We want to store this knowledge in a new column in the variants table called important_variant that tracks whether a given variant overlapped (1) or did not overlap (0) intervals in your annotation file.
-
-  *To do this, you must first TABIX your BED file*
-
-**-a boolean - Did a variant overlap a region or not?**
-
-Now, you can use this *TABIX*’ed file to annotate which variants overlap your important regions. In the example below, the results will be stored in a new column called “important”. The **-t boolean** option says that you just want to track whether (1) or not (0) the variant overlapped one or more of your regions.
-
-Since a new columns has been created in the database, we can now directly query the new column. In the example results below, the first and third variants overlapped a crucial region while the second did not::
-
-    chr22   100    101    1   1
-    chr22   200    201    2   0
-    chr22   300    500    3   1
-
-**-a count - How many regions did a variant overlap?**
-
-Instead of a simple yes or no, we can use the **-t count** option to count how many important regions a variant overlapped. It turns out that the 3rd variant actually overlapped two important regions::
-
-    chr22   100    101    1   1
-    chr22   200    201    2   0
-    chr22   300    500    3   2
-
-**-a extract - Extract specific values from a BED file**
-
-Lastly, we may also extract values from specific fields in a BED file (or from the INFO field in a VCF) and populate one or more new columns in the database based on overlaps with the annotation file and the values of the fields therein. To do this, we use the **-a extract** option.
-
-This is best described with an example. To set this up, let’s imagine that we have a VCF file from a different experiment and we want to annotate the variants in our GEMINI database with the allele frequency and depth tags from the INFO fields for the same variants in this other VCF file.
-
-Now that we have a proper *TABIX*’ed VCF file, we can use the **-a extract** option to populate new columns in the GEMINI database. In order to do so, we must specify:
-
- 1) its type (e.g., text, int, float,) (**-t**)
- 2) the field in the INFO column of the VCF file that we should use to extract data with which to populate the new column (**-e**)
- 3) what operation should be used to summarize the data in the event of multiple overlaps in the annotation file (**-o**)
- 4) (optionally) the name of the column we want to add (**-c**), if this is not specified, it will use the value from **-e**.
-
-For example, let’s imagine we want to create a new column called “other_allele_freq” (**-c**) using the AF field in our VCF file to populate it.
-
-This create a new column in my.db called other_allele_freq and this new column will be a FLOAT (**-t float**). In the event of multiple records in the VCF file overlapping a variant in the database, the average (**-o mean**) of the allele frequencies values from the VCF file will be used.
-
-At this point, one can query the database based on the values of the new other_allele_freq column (using **GEMINI query**).
-
-**-t TYPE - Specifying the column type(s) when using -a extract**
-
-The annotate tool will create three different types of columns via the **-t** option:
-
- 1) Floating point columns for annotations with decimal precision as above (-t float)
- 2) Integer columns for integral annotations (-t integer)
- 3) Text columns for string columns such as “valid”, “yes”, etc. (-t text)
-
-  *The -t option is only valid when using the -a extract option.*
-
-**-o OPERATION - Specifying the summary operations when using -a extract**
-
-In the event of multiple overlaps between a variant and records in the annotation file, the annotate tool can summarize the values observed with multiple options:
-
-  - -o mean       Compute the average of the values. They must be numeric.
-  - -o median     Compute the median of the values. They must be numeric.
-  - -o min        Compute the minimum of the values. They must be numeric.
-  - -o max        Compute the maximum of the values. They must be numeric.
-  - -o mode       Compute the maximum of the values. They must be numeric.
-  - -o first      Use the value from the first record in the annotation file.
-  - -o last       Use the value from the last record in the annotation file.
-  - -o list       Create a comma-separated list of the observed values.
-  - -o uniq_list  Create a comma-separated list of the distinct observed values.
-  - -o sum        Compute the sum of the values. They must be numeric.
-
-The -o option is only valid when using the -a extract option.
-
-**Annotating with VCF**
-
-Most of the examples to this point have pulled a column from a tabix indexed bed file. It is likewise possible to pull from the INFO field of a tabix index VCF. The syntax is identical but the **-e** operation will specify the names of fields in the INFO column to pull. By default, those names will be used, but that can still be specified with the **-c column**.
-
-To put a DP column in the db, set:
-
-  -o list, -e DP, -t integer
-
-... and name it 'depth', set:
-
-  -o list, -e DP, -c depth, -t integer
-
-
-Missing values are allowed since we expect that in some cases an annotation VCF will not have all INFO fields specified for all variants.
-
-*We recommend decomposing and normalizing variants before annotating. See Step 1. split, left-align, and trim variants for a detailed explanation of how to do this. To do that see the GEMINI* preprocessing_ *website.*
-
-**Extracting and populating multiple columns at once**
-
-One can also extract and populate multiple columns at once by providing comma-separated lists (no spaces) of column names (**-c**), types (**-t**), numbers (**-e**), and summary operations (**-o**). For example, recall that in the VCF example above, we created a *TABIX*’ed BED file containg the allele frequency and depth values from the INFO field as the 4th and 5th columns in the BED, respectively.
-
-Instead of running the annotate tool twice (once for each column), we can run the tool once and load both columns in the same run. For example with settings:
-
-  - -a extract
-  - -c other_allele_freq,other_depth
-  - -t float,integer
-  - -e 4,5
-  - -o mean,max
-
-We can then use each of the new columns to filter variants with a *GEMINI query*:
-
-.. _preprocessing: https://gemini.readthedocs.org/en/latest/content/preprocessing.html#preprocess
-
+Given an existing GEMINI database and an annotation source in BED or VCF
+format, the annotate tool will, for each variant in the variants table of the
+database, screen for overlapping regions defined in the annotation source and
+update one or more new columns of the variant record in the database based on
+the result and the annotation found.
     ]]></help>
     <expand macro="citations"/>
 </tool>