--- a/ena_upload.xml	Sat Nov 28 09:45:44 2020 +0000
+++ b/ena_upload.xml	Fri Apr 30 12:09:25 2021 +0000
@@ -1,24 +1,31 @@
-<tool id="ena_upload" name="ENA Upload tool" version="0.3" profile="20.01" license="MIT">
+<tool id="ena_upload" name="ENA Upload tool" version="0.3.1" profile="20.01" license="MIT">
     <macros>
-        <token name="@VERSION@">0.2.4</token>
+        <token name="@VERSION@">0.2.7</token>
         <import>samples_macros.xml</import>
     </macros>
     <requirements>
         <requirement type="package" version="@VERSION@">ena-upload-cli</requirement>
         <requirement type="package" version="1.2.0">xlrd</requirement>
     </requirements>
+    <stdio>
+        <regex match="Oops" source="stderr" level="fatal"/>
+        <regex match="different file names between command line and RUN table" source="stderr" level="fatal"/>
+    </stdio>
     <command detect_errors="exit_code"><![CDATA[
 mkdir ./submission_files; 
+#set $studies_table_path = './submission_files/studies.tsv'
+#set $samples_table_path =   './submission_files/samples.tsv'
+#set $experiments_table_path = './submission_files/experiments.tsv'
+#set $runs_table_path =  './submission_files/runs.tsv'
+        
+#set $studies_table_path_updated = './submission_files/studies_updated.tsv'
+#set $samples_table_path_updated =   './submission_files/samples_updated.tsv'
+#set $experiments_table_path_updated = './submission_files/experiments_updated.tsv'
+#set $runs_table_path_updated =  './submission_files/runs_updated.tsv'
 
 #set working_dir = os.getcwd()
-#set $dry_run_option = "False"
-#set viral_submission = "False"
 #if $action_options.input_format_conditional.input_format == "build_tables":
   python '$__tool_directory__/extract_tables.py' --action $action_options.action --out_dir ./submission_files --studies $studies_json;
-  #set $studies_table_path = './submission_files/studies.tsv'
-  #set $samples_table_path =   './submission_files/samples.tsv'
-  #set $experiments_table_path = './submission_files/experiments.tsv'
-  #set $runs_table_path =  './submission_files/runs.tsv'
 #end if
 
 #if $action_options.input_format_conditional.input_format == "excel_tables":
@@ -26,35 +33,30 @@
     #if $action_options.input_format_conditional.viral_submission == "true":
         --vir 
     #end if
-    --action '$action_options.action' --form '$action_options.input_format_conditional.xlsx_file' --out_dir ./submission_files ;
-    #set $studies_table_path = './submission_files/studies.tsv'
-    #set $samples_table_path =   './submission_files/samples.tsv'
-    #set $experiments_table_path = './submission_files/experiments.tsv'
-    #set $runs_table_path =  './submission_files/runs.tsv'
-    #if $action_options.input_format_conditional.dry_run == "true":
-      #set $dry_run_option = "True"
-    #end if
+    --action '$action_options.action' --form '$action_options.input_format_conditional.xlsx_file' --out_dir ./submission_files --verbose > '$output';
 #end if
 
 #if $action_options.input_format_conditional.input_format != "user_generated_tables":
-    cp $studies_table_path $studies_table_out;
-    cp $samples_table_path $samples_table_out;
-    cp $experiments_table_path $experiments_table_out;
-    cp $runs_table_path $runs_table_out;
-    #if $action_options.input_format_conditional.dry_run == "true":
-      #set $dry_run_option = "True"
-    #end if
+    cp $studies_table_path $studies_table_out &&
+    cp $samples_table_path $samples_table_out &&
+    cp $experiments_table_path $experiments_table_out &&
+    cp $runs_table_path $runs_table_out &&
+#else:
+    ln -s '$action_options.input_format_conditional.experiments_users_table' $experiments_table_path &&
+    ln -s '$action_options.input_format_conditional.studies_users_table' $studies_table_path &&
+    ln -s '$action_options.input_format_conditional.runs_users_table' $runs_table_path &&
+    ln -s '$action_options.input_format_conditional.samples_users_table' $samples_table_path &&
 #end if
 
 
-#if $dry_run_option == "False" and $action_options.test_submit == "False":
+#if $action_options.test_submit_parameters.dry_run == "false" and $action_options.test_submit == "False":
     webin_id=`grep 'username' $credentials`;
     if [ "\$webin_id" = "" ]; then
       ## No credentials in user defined preferences    
       ## Fallback to global defined credentials (if exist)   
       #import os
       #if os.path.isfile(os.environ.get('GALAXY_ENA_SECRETS', '')):
-          credentials_path=\${ENA_SECRETS};     
+          credentials_path=\${GALAXY_ENA_SECRETS};     
           webin_id=`grep 'username' \$GALAXY_ENA_SECRETS`;
           if [ "\$webin_id" = "" ]; then
               echo "No global credentials defined. Check your GALAXY_ENA_SECRETS file or set your credentials via: User -> Preferences -> Manage Information";
@@ -91,15 +93,41 @@
       #end for
     #end for
 #else:
-    #for $file in $action_options.input_format_conditional.data:
-        #set $safename_reads_file = re.sub('[^\w\-_\.]', '_', $file.element_identifier)
-        ln -s '$file' $safename_reads_file &&
-        $files_to_upload.append(str($safename_reads_file))
-    #end for
+    #if $action_options.input_format_conditional.run_input_format_conditional.run_input_format == 'paired_list':
+        #for $pair in $action_options.input_format_conditional.run_input_format_conditional.paired_end_collection:
+            #set $safename_reads_file = re.sub('[^\w\-_\.]', '_', $pair.name)
+            #if $pair.forward.is_of_type('fastq.gz', 'fastqsanger.gz'):
+                #set $safename_fwd_reads_file = $safename_reads_file + '_1.fastq.gz'
+            #elif $pair.forward.is_of_type('fastqsanger.bz2', 'fastq.bz2'):
+                #set $safename_fwd_reads_file = $safename_reads_file + '_1.fastq.bz2'
+            #else:
+                #set $safename_fwd_reads_file = $safename_reads_file + '_1.fastq'
+            #end if
+            #if $pair.reverse.is_of_type('fastq.gz', 'fastqsanger.gz'):
+                #set $safename_rev_reads_file = $safename_reads_file + '_2.fastq.gz'
+            #elif $pair.reverse.is_of_type('fastqsanger.bz2', 'fastq.bz2'):
+                #set $safename_rev_reads_file = $safename_reads_file + '_2.fastq.bz2'
+            #else:
+                #set $safename_rev_reads_file = $safename_reads_file + '_2.fastq'
+            #end if
+
+            ln -s '$pair.forward' $safename_fwd_reads_file &&
+            $files_to_upload.append(str($safename_fwd_reads_file))
+            ln -s '$pair.reverse' $safename_rev_reads_file &&
+            $files_to_upload.append(str($safename_rev_reads_file))
+        #end for
+    #end if
+    #if $action_options.input_format_conditional.run_input_format_conditional.run_input_format == 'multiple_selection_list':
+        #for $file in $action_options.input_format_conditional.run_input_format_conditional.data:
+            #set $safename_reads_file = re.sub('[^\w\-_\.]', '_', $file.element_identifier)
+            ln -s '$file' $safename_reads_file &&
+            $files_to_upload.append(str($safename_reads_file))
+        #end for
+    #end if
 #end if
 
 
-#if $dry_run_option == "False":
+#if $action_options.test_submit_parameters.dry_run == "false":
 ena-upload-cli
     --tool 'ena-upload-cli v@VERSION@ @ Galaxy'
     --action '$action_options.action'
@@ -109,19 +137,15 @@
     #for $dataset in $files_to_upload:
         '$dataset'
     #end for
+--experiment '$experiments_table_path'
+--study '$studies_table_path'
+--run '$runs_table_path'
+--sample '$samples_table_path'
 #if $action_options.input_format_conditional.input_format == "user_generated_tables":
-    --experiment '$action_options.input_format_conditional.experiments_users_table'
-    --study '$action_options.input_format_conditional.studies_users_table'
-    --run '$action_options.input_format_conditional.runs_users_table'
-    --sample '$action_options.input_format_conditional.samples_users_table'
     #if "$action_options.input_format_conditional.viral_submission" == "true":
         --vir
     #end if
 #else:
-    --experiment '$experiments_table_path'
-    --study '$studies_table_path'
-    --run '$runs_table_path'
-    --sample '$samples_table_path'
     #if $action_options.input_format_conditional.input_format == "build_tables":
         #if $action_options.input_format_conditional.conditional_viral_metadata.viral_sample == "true":
           --vir
@@ -133,12 +157,19 @@
     #end if
 #end if
 
-#if $action_options.submit_dev == "true":
+#if $action_options.test_submit_parameters.submit_dev == "true":
     -d
 #end if
-  > '$output'
+    >> '$output';
+    echo -e 'center_name\t$action_options.center' >> '$output';
+    echo -e 'action_option\t$action_options.action' >> '$output';
+    #if $action_options.input_format_conditional.input_format != "user_generated_tables":
+        cp $studies_table_path_updated $studies_table_out 2>/dev/null;
+        cp $samples_table_path_updated $samples_table_out 2>/dev/null;
+        cp $experiments_table_path_updated $experiments_table_out 2>/dev/null;
+        cp $runs_table_path_updated $runs_table_out 2>/dev/null;
+    #end if
 #else:
-    echo ""
     exit 0;
 #end if
 
@@ -170,9 +201,9 @@
               #set $safename_reads_file = re.sub('[^\w\-_\.]', '_', $file.element_identifier)
               $run_files.append(str($safename_reads_file))
             #end for
-            $runs.append($run_files)
+            $runs.append((str($run.run_base_name),$run_files))
         #end for
-
+    
 $experiments.append({'title':str($experiment.experiment_title),'experiment_design':str($experiment.experiment_design),'library_strategy':str($experiment.library_strategy),'library_source':str($experiment.library_source),'library_selection':str($experiment.library_selection),'library_layout':str($experiment.library_layout),'insert_size':str($experiment.insert_size),'library_construction_protocol':str($experiment.library_construction_protocol),'platform':str($experiment.platform),'instrument_model':str($experiment.instrument_model),'runs':$runs})
       #end for
       #if $action_options.input_format_conditional.conditional_viral_metadata.viral_sample == "true":
@@ -196,17 +227,18 @@
                 <option value="modify">Modify metadata</option>
             </param>
             <when value="add">
-                <param name="submit_dev" type="boolean" label="Submit to test ENA server?" help="By selecting yes the reads will be submitted " />
+                <expand macro="test_submit_section"/>    
                 <param name="test_submit" type="hidden" value="False" />
                 <expand macro="table_inputs_macro" />
             </when>
             <when value="modify">
+                <expand macro="test_submit_section"/>    
                 <expand macro="table_inputs_macro" />
             </when>
         </conditional>
     </inputs>
     <outputs>
-        <data name="output" format="data" label="${tool.name} on ${on_string}: Upload summary"/>
+        <data name="output" format="txt" label="ENA submission receipt"/>
         <data name="studies_table_out" format="tabular" label="Studies table">
             <filter> action_options['input_format_conditional']['input_format'] == "build_tables" or action_options['input_format_conditional']['input_format'] == "excel_tables"</filter>
         </data>
@@ -225,13 +257,18 @@
         <test>
             <conditional name="action_options">
                 <param name="action" value="add"/>
-                <param name="submit_dev" value="False"/>
+                <section name="test_submit_parameters">
+                    <param name="submit_dev" value="false" />
+                    <param name="dry_run" value="true" />
+                </section>
                 <conditional name="input_format_conditional">
                     <param name="input_format" value="excel_tables"/>
                     <param name="viral_submission" value="True"/>
-                    <param name="dry_run" value="True"/>
                     <param name="xlsx_file" value="metadata_test_viral.xlsx"/>
-                    <param name="data" value="sample.fq"/>
+                    <conditional name="run_input_format_conditional">
+                        <param name="run_input_format" value="multiple_selection_list"/>
+                        <param name="data" value="sample.fq"/>
+                    </conditional>
                 </conditional>
             </conditional>
             <param name="center" value="Some research center"/>
@@ -271,13 +308,18 @@
         <test>
             <conditional name="action_options">
                 <param name="action" value="add"/>
-                <param name="submit_dev" value="False"/>
+                <section name="test_submit_parameters">
+                    <param name="submit_dev" value="false" />
+                    <param name="dry_run" value="true" />
+                </section>
                 <conditional name="input_format_conditional">
                     <param name="input_format" value="excel_tables"/>
                     <param name="viral_submission" value="False"/>
-                    <param name="dry_run" value="True"/>
                     <param name="xlsx_file" value="metadata_test_nonviral.xlsx"/>
-                    <param name="data" value="sample.fq"/>
+                    <conditional name="run_input_format_conditional">
+                        <param name="run_input_format" value="multiple_selection_list"/>
+                        <param name="data" value="sample.fq"/>
+                    </conditional>
                 </conditional>
             </conditional>
             <param name="center" value="Some research center"/>
@@ -311,14 +353,50 @@
                 </assert_contents>
             </output>
         </test>
+        <!--Test failure on excel input of NON-VIRAL samples with runs PAIRED collection -->
+        <test expect_failure="true">
+            <conditional name="action_options">
+                <param name="action" value="add"/>
+                <section name="test_submit_parameters">
+                    <param name="submit_dev" value="false" />
+                    <param name="dry_run" value="false" />
+                </section>
+                <conditional name="input_format_conditional">
+                    <param name="input_format" value="excel_tables"/>
+                    <param name="viral_submission" value="False"/>
+                    <param name="xlsx_file" value="metadata_test_nonviral.xlsx"/>
+                    <conditional name="run_input_format_conditional">
+                        <param name="run_input_format" value="paired_list"/>
+                        <param name="paired_end_collection">
+                            <collection type="list:paired">
+                                <element name="paired_run_name">
+                                    <collection type="paired">
+                                        <element name="forward" value="1.fastqsanger.gz" ftype="fastqsanger.gz" />
+                                        <element name="reverse" value="2.fastqsanger.gz" ftype="fastqsanger.gz" />
+                                    </collection>
+                                </element>
+                            </collection>
+                        </param>
+                    </conditional>
+                </conditional>
+            </conditional>
+            <param name="center" value="Some research center"/>
+            <assert_command>
+                <has_text_matching expression="ena-upload-cli"/>
+                <has_text_matching expression="--data 'paired_run_name_1.fastq.gz' 'paired_run_name_2.fastq.gz'"/>
+                <has_text_matching expression="--action 'add' --center 'Some research center'"/>
+            </assert_command>
+        </test>
         <!--Test build tables from user input fields NON-VIRAL samples-->
         <test>
             <conditional name="action_options">
                 <param name="action" value="add"/>
-                <param name="submit_dev" value="False"/>
+                <section name="test_submit_parameters">
+                    <param name="submit_dev" value="false" />
+                    <param name="dry_run" value="true" />
+                </section>
                 <conditional name="input_format_conditional">
                     <param name="input_format" value="build_tables"/>
-                    <param name="dry_run" value="True"/>
                     <conditional name="conditional_viral_metadata">
                         <param name="viral_sample" value="False"/>
                         <repeat name="rep_study">
@@ -383,12 +461,14 @@
         <test expect_failure="true">
             <conditional name="action_options">
                 <param name="action" value="add"/>
-                <param name="submit_dev" value="True"/>
+                <section name="test_submit_parameters">
+                    <param name="submit_dev" value="true" />
+                    <param name="dry_run" value="false" />
+                </section>
                 <conditional name="input_format_conditional">
                     <param name="input_format" value="build_tables"/>
-                    <param name="dry_run" value="False"/>
                     <conditional name="conditional_viral_metadata">
-                        <param name="viral_sample" value="False"/>
+                        <param name="viral_sample" value="false"/>
                         <repeat name="rep_study">
                             <param name="study_title" value="Test study title"/>
                             <param name="study_abstract" value="Test study abstract"/>
@@ -428,13 +508,15 @@
         <test expect_failure="true">
             <conditional name="action_options">
                 <param name="action" value="add"/>
-                <param name="submit_dev" value="True"/>
+                <section name="test_submit_parameters">
+                    <param name="submit_dev" value="true" />
+                    <param name="dry_run" value="false" />
+                </section>
                 <param name="test_submit" value="True"/>
                 <conditional name="input_format_conditional">
                     <param name="input_format" value="build_tables"/>
-                    <param name="dry_run" value="False"/>
                     <conditional name="conditional_viral_metadata">
-                        <param name="viral_sample" value="False"/>
+                        <param name="viral_sample" value="false"/>
                         <repeat name="rep_study">
                             <param name="study_title" value="Test study title"/>
                             <param name="study_abstract" value="Test study abstract"/>
@@ -457,6 +539,7 @@
                                     <param name="platform" value="ILLUMINA"/>
                                     <param name="instrument_model" value="Illumina HiSeq 4000"/>
                                     <repeat name="rep_runs">
+                                        <param name="run_base_name" value="run_from_hospital_X"/>
                                         <param name="upload_files" value="1.fastqsanger.gz,sample.fq" ftype="fastqsanger"/>
                                     </repeat>
                                 </repeat>
@@ -472,18 +555,20 @@
                 <has_text_matching expression="--action 'add' --center 'Some research center'"/>
             </assert_command>
             <assert_stderr>
-                <has_text_matching expression="ENA_upload: error: Oops, the file test_fake_path does not exist"/>
+                <has_text_matching expression="Oops, the file test_fake_path does not exist"/>
             </assert_stderr>
         </test>
         <!--test viral submission - User input metadata-->
         <test expect_failure="true">
             <conditional name="action_options">
                 <param name="action" value="add"/>
-                <param name="submit_dev" value="False"/>
+                <section name="test_submit_parameters">
+                    <param name="submit_dev" value="false" />
+                    <param name="dry_run" value="false" />
+                </section>
                 <param name="test_submit" value="True"/>
                 <conditional name="input_format_conditional">
                     <param name="input_format" value="build_tables"/>
-                    <param name="dry_run" value="False"/>
                     <conditional name="conditional_viral_metadata">
                         <param name="viral_sample" value="True"/>
                         <repeat name="rep_study">
@@ -518,6 +603,7 @@
                                     <param name="platform" value="ILLUMINA"/>
                                     <param name="instrument_model" value="Illumina HiSeq 4000"/>
                                     <repeat name="rep_runs">
+                                        <param name="run_base_name" value="run_from_hospital_X"/>
                                         <param name="upload_files" value="1.fastqsanger.gz,sample.fq" ftype="fastqsanger"/>
                                     </repeat>
                                 </repeat>
@@ -534,7 +620,7 @@
                 <has_text_matching expression="--vir"/>
             </assert_command>
             <assert_stderr>
-                <has_text_matching expression="ENA_upload: error: Oops, the file test_fake_path does not exist"/>
+                <has_text_matching expression="Oops, the file test_fake_path does not exist"/>
             </assert_stderr>
         </test>
     </tests>