Mercurial > repos > galaxyp > openms_fileconverter

comparison generate-foo.sh @ 15:2a93a25f0599 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload for repository https://github.com/galaxyproteomics/tools-galaxyp/tree/master/tools/openms commit 3d1e5f37fd16524a415f707772eeb7ead848c5e3
author galaxyp
date Thu, 01 Dec 2022 19:09:16 +0000
parents 62e19948b50f
children
comparison
equal deleted inserted replaced
14:9b7977080f6d 15:2a93a25f0599
6 >&2 echo "generate tests for $id" 6 >&2 echo "generate tests for $id"
7 echo '<xml name="autotest_'"$id"'">' 7 echo '<xml name="autotest_'"$id"'">'
8 8
9 # get the tests from the CMakeLists.txt 9 # get the tests from the CMakeLists.txt
10 # 1st remove some tests 10 # 1st remove some tests
11 # - OpenSwathMzMLFileCacher with -convert_back argumen https://github.com/OpenMS/OpenMS/issues/4399 11 # - OpenSwathMzMLFileCacher with -convert_back argument https://github.com/OpenMS/OpenMS/issues/4399
12 # - IDRipper PATH gets empty causing problems. TODO But overall the option needs to be handled differentlt 12 # - IDRipper PATH gets empty causing problems. TODO But overall the option needs to be handled differentlt
13 # - several tools with duplicated input (leads to conflict when linking) 13 # - several tools with duplicated input (leads to conflict when linking)
14 # - TOFCalibration inputs we extension (also in prepare_test_data) https://github.com/OpenMS/OpenMS/pull/4525
15 # - MaRaCluster with -consensus_out (parameter blacklister: https://github.com/OpenMS/OpenMS/issues/4456) 14 # - MaRaCluster with -consensus_out (parameter blacklister: https://github.com/OpenMS/OpenMS/issues/4456)
16 # - FileMerger with mixed dta dta2d input (ftype can not be specified in the test, dta can not be sniffed) 15 # - FileMerger with mixed dta dta2d input (ftype can not be specified in the test, dta can not be sniffed)
17 # - some input files are originally in a subdir (degenerated cases/), but not in test-data 16 # - some input files are originally in a subdir (degenerated cases/), but not in test-data
18 # - SeedListGenerator: https://github.com/OpenMS/OpenMS/issues/4404
19 # - OpenSwathAnalyzer 9/10: cachedMzML (not supported yet) 17 # - OpenSwathAnalyzer 9/10: cachedMzML (not supported yet)
20 # - FeatureFinderIdentification name clash of two tests https://github.com/OpenMS/OpenMS/pull/5002 18 # - SiriusAdapter_4 depends on online service which may timeout .. so keep disabled https://github.com/OpenMS/OpenMS/pull/5010
21 # - TODO SiriusAdapter https://github.com/OpenMS/OpenMS/pull/5010 19 # - SiriusAdapter_10 should work in >2.8 https://github.com/OpenMS/OpenMS/issues/5869
22 CMAKE=$(cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake | 20 CMAKE=$(cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake |
23 sed 's@${DATA_DIR_SHARE}/@@g' | 21 sed 's@${DATA_DIR_SHARE}/@@g' |
24 grep -v 'OpenSwathMzMLFileCacher .*-convert_back' | 22 grep -v 'OpenSwathMzMLFileCacher .*-convert_back' |
25 sed 's/${TMP_RIP_PATH}/""/' | 23 sed 's/${TMP_RIP_PATH}/""/' |
26 grep -v "MaRaClusterAdapter.*-consensus_out"| 24 grep -v "MaRaClusterAdapter.*-consensus_out"|
27 grep -v "FileMerger_1_input1.dta2d.*FileMerger_1_input2.dta " | 25 grep -v "FileMerger_1_input1.dta2d.*FileMerger_1_input2.dta " |
28 sed 's@degenerate_cases/@@g' | 26 sed 's@degenerate_cases/@@g' |
29 grep -v 'TOPP_SeedListGenerator_3"' |
30 egrep -v 'TOPP_OpenSwathAnalyzer_test_3"|TOPP_OpenSwathAnalyzer_test_4"' | 27 egrep -v 'TOPP_OpenSwathAnalyzer_test_3"|TOPP_OpenSwathAnalyzer_test_4"' |
31 egrep -v '"TOPP_FeatureFinderIdentification_4"' | 28 sed 's/\("TOPP_SiriusAdapter_4".*\)-sirius:database all\(.*\)/\1-sirius:database pubchem\2/' |
32 sed 's/\("TOPP_SiriusAdapter_4".*\)-sirius:database all\(.*\)/\1-sirius:database pubchem\2/') 29 grep -v '"TOPP_SiriusAdapter_10"')
33
34
35 # grep -v 'FileFilter.*-spectra:select_polarity ""' |
36 # grep -v 'MassTraceExtractor_2.ini ' |
37 # grep -v "FileMerger_6_input2.mzML.*FileMerger_6_input2.mzML" |
38 # grep -v "IDMerger_1_input1.idXML.*IDMerger_1_input1.idXML" |
39 # grep -v "degenerated_empty.idXML.*degenerated_empty.idXML" |
40 # grep -v "FeatureLinkerUnlabeledKD_1_output.consensusXML.*FeatureLinkerUnlabeledKD_1_output.consensusXML" |
41 # grep -v "FeatureLinkerUnlabeledQT_1_output.consensusXML.*FeatureLinkerUnlabeledQT_1_output.consensusXML" |
42 30
43 # 1st part is a dirty hack to join lines containing a single function call, e.g. 31 # 1st part is a dirty hack to join lines containing a single function call, e.g.
44 # addtest(.... 32 # addtest(....
45 # ....) 33 # ....)
46 echo "$CMAKE" | sed 's/#.*//; s/^\s*//; s/\s*$//' | grep -v "^#" | grep -v "^$" | awk '{printf("%s@NEWLINE@", $0)}' | sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' | 34 echo "$CMAKE" | sed 's/#.*//; s/^\s*//; s/\s*$//' | grep -v "^#" | grep -v "^$" | awk '{printf("%s@NEWLINE@", $0)}' | sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' |
48 do 36 do
49 line=$(echo "$line" | sed 's/add_test("\([^"]\+\)"/\1/; s/)$//; s/\${TOPP_BIN_PATH}\///g;s/\${DATA_DIR_TOPP}\///g; s#THIRDPARTY/##g') 37 line=$(echo "$line" | sed 's/add_test("\([^"]\+\)"/\1/; s/)$//; s/\${TOPP_BIN_PATH}\///g;s/\${DATA_DIR_TOPP}\///g; s#THIRDPARTY/##g')
50 # >&2 echo $line 38 # >&2 echo $line
51 test_id=$(echo "$line" | cut -d" " -f 1) 39 test_id=$(echo "$line" | cut -d" " -f 1)
52 tool_id=$(echo "$line" | cut -d" " -f 2) 40 tool_id=$(echo "$line" | cut -d" " -f 2)
41 # >&2 echo "test_id $test_id"
53 if [[ $test_id =~ _out_?[0-9]? ]]; then 42 if [[ $test_id =~ _out_?[0-9]? ]]; then
54 >&2 echo " skip $test_id $line" 43 >&2 echo " skip $test_id $line"
55 continue 44 continue
56 fi 45 fi
57 if [[ ${id,,} != ${tool_id,,} ]]; then 46 if [[ ${id,,} != ${tool_id,,} ]]; then
65 continue 54 continue
66 fi 55 fi
67 tes=" <test>\n" 56 tes=" <test>\n"
68 line=$(fix_tmp_files "$line") 57 line=$(fix_tmp_files "$line")
69 line=$(unique_files "$line") 58 line=$(unique_files "$line")
70 # >&2 echo $line 59 # >&2 echo LINE $line
71 #if there is an ini file then we use this to generate the test 60 #if there is an ini file then we use this to generate the test
72 #otherwise the ctd file is used 61 #otherwise the ctd file is used
73 #other command line parameters are inserted later into this xml 62 #other command line parameters are inserted later into this xml
74 if grep -lq "\-ini" <<<"$line"; then 63 if grep -lq "\-ini" <<<"$line"; then
75 ini=$(echo $line | sed 's/.*-ini \([^ ]\+\).*/\1/') 64 ini=$(echo $line | sed 's/.*-ini \([^ ]\+\).*/\1/')
76 ini="test-data/$ini" 65 ini="test-data/$ini"
77 else 66 else
78 ini="ctd/$tool_id.ctd" 67 ini="ctd/$tool_id.ctd"
79 fi 68 fi
69 # >&2 echo "========================================================"
70 # >&2 echo "USING ini $ini"
80 cli=$(echo $line |cut -d" " -f3- | sed 's/-ini [^ ]\+//') 71 cli=$(echo $line |cut -d" " -f3- | sed 's/-ini [^ ]\+//')
81 72
82 ctdtmp=$(mktemp) 73 ctdtmp=$(mktemp)
83 #echo python3 fill_ctd_clargs.py --ctd $ini $cli
84 # using eval: otherwise for some reason quoted values are not used properly ('A B' -> ["'A", "B'"]) 74 # using eval: otherwise for some reason quoted values are not used properly ('A B' -> ["'A", "B'"])
85 # >&2 echo "python3 fill_ctd_clargs.py --ctd $ini $cli" 75 # >&2 echo "python3 fill_ctd_clargs.py --ini_file $ini $cli"
86 eval "python3 fill_ctd_clargs.py --ctd $ini $cli" > "$ctdtmp" 76 eval "python3 fill_ctd_clargs.py --ini_file $ini $cli" > "$ctdtmp"
87 # echo $ctdtmp 77 # >&2 echo $ctdtmp
88 # >&2 cat $ctdtmp 78 # >&2 cat $ctdtmp
89 testtmp=$(mktemp) 79 testtmp=$(mktemp)
90 python3 $CTDCONVERTER/convert.py galaxy -i $ctdtmp -o $testtmp -s tools_blacklist.txt -f "$FILETYPES" -m macros.xml -t tool.conf -p hardcoded_params.json --tool-version $VERSION --test-only --test-unsniffable csv tsv txt dta dta2d edta mrm splib > /dev/null 80 # >&2 echo CTDConverter galaxy -i $ctdtmp -o $testtmp -s aux/tools_blacklist.txt -f "$FILETYPES" -m macros.xml -t tool.conf -p aux/hardcoded_params.json --tool-version $VERSION --test-only --test-unsniffable csv tsv txt dta dta2d edta mrm splib --test-condition "compare=sim_size" "delta_frac=0.7"
81 CTDConverter galaxy -i $ctdtmp -o $testtmp -s aux/tools_blacklist.txt -f "$FILETYPES" -m macros.xml -t tool.conf -p aux/hardcoded_params.json --tool-version $VERSION --test-only --test-unsniffable csv tsv txt dta dta2d edta mrm splib --test-condition "compare=sim_size" "delta_frac=0.7" > /dev/null
82 echo "<!-- $test_id -->"
91 cat $testtmp | grep -v '<output.*file=""' # | grep -v 'CHEMISTRY/' 83 cat $testtmp | grep -v '<output.*file=""' # | grep -v 'CHEMISTRY/'
92 rm $ctdtmp $testtmp 84
85 rm "$ctdtmp" "$testtmp"
93 86
94 #> /dev/null 87 #> /dev/null
95 88
96 #rm $testtmp 89 #rm $testtmp
97 done 90 done
128 #problem: the extension of the tmp files is unusable for test generation. 121 #problem: the extension of the tmp files is unusable for test generation.
129 #unfortunately the extensions used in the DIFF lines are not always usable for the CLI 122 #unfortunately the extensions used in the DIFF lines are not always usable for the CLI
130 #(e.g. for prepare_test_data, e.g. CLI expects csv but test file is txt) 123 #(e.g. for prepare_test_data, e.g. CLI expects csv but test file is txt)
131 #this function replaces the tmp file by the expected file. 124 #this function replaces the tmp file by the expected file.
132 function fix_tmp_files { 125 function fix_tmp_files {
133 # >&2 echo "FIX $line" 126 # >&2 echo "FIX $line"
134 ret="" 127 ret=""
135 for a in $@; do 128 for a in $@; do
136 if [[ ! $a =~ .tmp$ ]]; then 129 # >&2 echo " a "$a
130 if [[ ! $a =~ .tmp$ ]] && [[ ! $a =~ _tmp_ ]]; then
137 ret="$ret $a" 131 ret="$ret $a"
138 continue 132 continue
139 fi 133 fi
140 # >&2 echo " a "$a 134 diff_line=$(cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake | awk '{printf("%s@NEWLINE@", $0)}' | sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' | grep '\${DIFF}.*'"$a")
141 g=$(cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake | awk '{printf("%s@NEWLINE@", $0)}' | sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' | grep '\${DIFF}.*'"$a") 135 # >&2 echo " diff_line "$diff_line
142 # >&2 echo " g "$g 136 in1=$(sed 's/.*-in1 \([^ ]\+\).*/\1/' <<<$diff_line)
143 in1=$(sed 's/.*-in1 \([^ ]\+\).*/\1/' <<<$g)
144 # >&2 echo " in1 "$in1 137 # >&2 echo " in1 "$in1
145 if [[ "$a" != "$in1" ]]; then 138 if [[ "$a" != "$in1" ]]; then
146 ret="$ret $a" 139 ret="$ret $a"
147 continue 140 continue
148 fi 141 fi
149 in2=$(sed 's/.*-in2 \([^ ]\+\).*/\1/' <<<$g) 142 in2=$(sed 's/.*-in2 \([^ ]\+\).*/\1/' <<<$diff_line)
150 in2=$(basename $in2 | sed 's/)$//') 143 in2=$(basename $in2 | sed 's/)$//')
151 # >&2 echo " in2 "$in2 144 # >&2 echo " in2 "$in2
152 if [[ -f "test-data/$in2" ]]; then 145 if [[ -f "test-data/$in2" ]]; then
153 ln -fs "$in1" "test-data/$in2" 146 ln -fs "$in1" "test-data/$in2"
154 ret="$ret $in2" 147 ret="$ret $in2"
174 >&2 echo "not linking equal $in1 $in2" 167 >&2 echo "not linking equal $in1 $in2"
175 continue 168 continue
176 fi 169 fi
177 ln -f -s $in1 test-data/$in2 170 ln -f -s $in1 test-data/$in2
178 done 171 done
179 for i in test-data/*.tmp 172
180 do 173 find test-data/ -name "*.tmp" -print0 |
174 while IFS= read -r -d '' i; do
181 if [ ! -e test-data/$(basename $i .tmp) ]; then 175 if [ ! -e test-data/$(basename $i .tmp) ]; then
182 ln -s $(basename $i) test-data/$(basename $i .tmp) 176 ln -s $(basename $i) test-data/$(basename $i .tmp)
183 #ln -s $(basename $i) test-data/$(basename $i .tmp)
184 else 177 else
185 ln -fs $(basename $i) test-data/$(basename $i .tmp) 178 ln -fs $(basename $i) test-data/$(basename $i .tmp)
186 fi 179 fi
187 done 180 done
188 } 181 }
192 # parse data preparation calls from OpenMS sources for a tool with a given id 185 # parse data preparation calls from OpenMS sources for a tool with a given id
193 function prepare_test_data { 186 function prepare_test_data {
194 # id=$1 187 # id=$1
195 # | egrep -i "$id\_.*[0-9]+(_prepare\"|_convert)?" 188 # | egrep -i "$id\_.*[0-9]+(_prepare\"|_convert)?"
196 189
197 # TODO SiriusAdapter https://github.com/OpenMS/OpenMS/pull/5010 190 # TODO SiriusAdapter depends on online service which may timeout .. so keep disabled https://github.com/OpenMS/OpenMS/pull/5010
198 cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake | sed 's/#.*$//'| sed 's/^\s*//; s/\s*$//' | grep -v "^$" | awk '{printf("%s@NEWLINE@", $0)}' | sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' | 191 cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake | sed 's/#.*$//'| sed 's/^\s*//; s/\s*$//' | grep -v "^$" | awk '{printf("%s@NEWLINE@", $0)}' | sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' |
199 sed 's/degenerate_cases\///' | 192 sed 's/degenerate_cases\///' |
200 egrep -v "WRITEINI|WRITECTD|INVALIDVALUE|DIFF" | 193 egrep -v "WRITEINI|WRITECTD|INVALIDVALUE|DIFF" |
201 grep add_test | 194 grep add_test |
202 egrep "TOPP|UTILS" | 195 egrep "TOPP|UTILS" |
203 sed 's@${DATA_DIR_SHARE}/@@g;'| 196 sed 's@${DATA_DIR_SHARE}/@@g;'|
204 sed 's@${TMP_RIP_PATH}@dummy2.tmp@g'| 197 sed 's@${TMP_RIP_PATH}@./@g'|
205 sed 's@TOFCalibration_ref_masses @TOFCalibration_ref_masses.txt @g; s@TOFCalibration_const @TOFCalibration_const.csv @'| 198 sed 's@TOFCalibration_ref_masses @TOFCalibration_ref_masses.txt @g; s@TOFCalibration_const @TOFCalibration_const.csv @'|
206 sed 's/\("TOPP_SiriusAdapter_4".*\)-sirius:database all\(.*\)/\1-sirius:database pubchem\2/' | 199 sed 's/\("TOPP_SiriusAdapter_4".*\)-sirius:database all\(.*\)/\1-sirius:database pubchem\2/' |
207 while read line 200 while read line
208 do 201 do
209 test_id=$(echo "$line" | sed 's/add_test(//; s/"//g; s/)[^)]*$//; s/\${TOPP_BIN_PATH}\///g;s/\${DATA_DIR_TOPP}\///g; s#THIRDPARTY/##g' | cut -d" " -f1) 202 test_id=$(echo "$line" | sed 's/add_test(//; s/"//g; s/)[^)]*$//; s/\${TOPP_BIN_PATH}\///g;s/\${DATA_DIR_TOPP}\///g; s#THIRDPARTY/##g' | cut -d" " -f1)