Mercurial > repos > galaxyp > openms_fuzzydiff

comparison test-data.sh @ 16:5369624840b9 draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload for repository https://github.com/galaxyproteomics/tools-galaxyp/tree/master/tools/openms commit 5c080b1e2b99f1c88f4557e9fec8c45c9d23b906
author galaxyp
date Fri, 14 Jun 2024 21:28:23 +0000
parents c940abd8d622
children
comparison
equal deleted inserted replaced
15:c940abd8d622 16:5369624840b9
1 #!/usr/bin/env bash 1 #!/usr/bin/env bash
2 2
3 VERSION=2.8 3 # set -x
4
5 VERSION=3.1
4 FILETYPES="aux/filetypes.txt" 6 FILETYPES="aux/filetypes.txt"
5 CONDAPKG="https://anaconda.org/bioconda/openms/2.8.0/download/linux-64/openms-2.8.0-h7ca0330_0.tar.bz2" 7 CONDAPKG="https://anaconda.org/bioconda/openms/3.1.0/download/linux-64/openms-3.1.0-h8964181_1.tar.bz2"
6
7 # import the magic
8 . ./generate-foo.sh
9 8
10 # install conda 9 # install conda
11 if [ -z "$tmp" ]; then 10 if [ -z "$tmp" ]; then
12 tmp=$(mktemp -d) 11 tmp=$(mktemp -d)
13 created="yes" 12 created="yes"
43 ## - the git clone of OpenMS (for generating the tests) 42 ## - the git clone of OpenMS (for generating the tests)
44 ############################################################################### 43 ###############################################################################
45 44
46 echo "Clone OpenMS $VERSION sources" 45 echo "Clone OpenMS $VERSION sources"
47 if [[ ! -d $OPENMSGIT ]]; then 46 if [[ ! -d $OPENMSGIT ]]; then
48 # TODO >2.8 reenable original release branch .. also in else branch 47 if [[ "$created" == "yes" ]]; then
49 # the plus branch contains commits from https://github.com/OpenMS/OpenMS/pull/5920 and https://github.com/OpenMS/OpenMS/pull/5917 48 GIT_DIR=$(mktemp -d --dry-run)
50 # git clone -b release/$VERSION.0 https://github.com/OpenMS/OpenMS.git $OPENMSGIT 49 GIT_EXTRA_OPTS="--separate-git-dir=$GIT_DIR"
51 git clone -b release/$VERSION.0-plus https://github.com/bernt-matthias/OpenMS.git $OPENMSGIT 50 fi
51 git clone -b release/$VERSION.0 --depth 1 --recurse-submodules=THIRDPARTY --shallow-submodules $GIT_EXTRA_OPTS https://github.com/OpenMS/OpenMS.git $OPENMSGIT
52 ## save some space by just keeping the needed binaries
53 find $OPENMSGIT/THIRDPARTY/ -type f -not \( -name maracluster -o -name spectrast \) -delete
54 find $OPENMSGIT/THIRDPARTY/ -empty -type d -delete
55 if [[ "$created" == "yes" ]]; then
56 rm -rf $GIT_DIR
57 fi
58 else
52 cd $OPENMSGIT 59 cd $OPENMSGIT
53 git submodule init 60 git pull origin release/$VERSION.0
54 git submodule update
55 cd -
56 else
57 cd $OPENMSGIT
58 git pull origin release/$VERSION.0-plus
59 cd - 61 cd -
60 fi 62 fi
61 63
62 echo "Create OpenMS $VERSION conda env" 64 echo "Create OpenMS $VERSION conda env"
63 # TODO currently add lxml (needed by CTDConverter) 65 # TODO currently add lxml (needed by CTDConverter)
64 # TODO for some reason a to recent openjdk is used 66 # TODO for some reason a to recent openjdk is used
65 if conda env list | grep "$OPENMSENV"; then 67 if conda env list | grep "$OPENMSENV"; then
66 true 68 true
67 else 69 else
68 conda create -y --quiet --override-channels --channel iuc --channel conda-forge --channel bioconda --channel defaults -n $OPENMSENV openms=$VERSION openms-thirdparty=$VERSION omssa=2.1.9 ctdopts=1.5 lxml 70 conda create -y --quiet --solver libmamba --override-channels --strict-channel-priority --channel conda-forge --channel bioconda -n $OPENMSENV openms=$VERSION openms-thirdparty=$VERSION ctdopts=1.5 lxml
69 # chmod -R u-w $OPENMSENV 71 # chmod -R u-w $OPENMSENV
70 fi 72 fi
71 ############################################################################### 73 ###############################################################################
72 ## get the 74 ## get the
73 ## - conda package (for easy access and listing of the OpenMS binaries), 75 ## - conda package (for easy access and listing of the OpenMS binaries),
99 python -m pip install . --no-deps 101 python -m pip install . --no-deps
100 cd - 102 cd -
101 conda deactivate 103 conda deactivate
102 104
103 105
104 # ############################################################################### 106 # # ###############################################################################
105 # ## copy all the test data files to test-data 107 # # ## copy all the test data files to test-data
106 # ## most of it (outputs) will be overwritten later, but its needed for 108 # # ## most of it (outputs) will be overwritten later, but its needed for
107 # ## prepare_test_data 109 # # ## prepare_test_data
108 # ############################################################################### 110 # # ###############################################################################
109 echo "Get test data" 111 echo "Get test data"
110 find test-data -type f,l,d ! -name "*fa" ! -name "*loc" ! -name "test-data" -delete 112 find test-data -type f,l,d ! -name "*fa" ! -name "*loc" ! -name "test-data" ! -name MetaboliteSpectralDB.mzML -delete
111 113
112 cp $(find $OPENMSGIT/src/tests/topp/ -type f | grep -Ev "third_party_tests.cmake|CMakeLists.txt|check_ini") test-data/ 114 cp $(find $OPENMSGIT/src/tests/topp/ -type f | grep -Ev "third_party_tests.cmake|CMakeLists.txt|check_ini") test-data/
113 cp -r $OPENMSGIT/share/OpenMS/MAPPING/ test-data/ 115 cp -r $OPENMSGIT/share/OpenMS/MAPPING/ test-data/
114 cp -r $OPENMSGIT/share/OpenMS/CHEMISTRY test-data/ 116 cp -r $OPENMSGIT/share/OpenMS/CHEMISTRY test-data/
115 cp -r $OPENMSGIT/share/OpenMS/examples/ test-data/ 117 cp -r $OPENMSGIT/share/OpenMS/examples/ test-data/
116 if [ ! -f test-data/MetaboliteSpectralDB.mzML ]; then 118 if [ ! -f test-data/MetaboliteSpectralDB.mzML ]; then
117 wget -nc https://abibuilder.cs.uni-tuebingen.de/archive/openms/Tutorials/Data/latest/Example_Data/Metabolomics/databases/MetaboliteSpectralDB.mzML 119 wget -nc https://raw.githubusercontent.com/sneumann/OpenMS/master/share/OpenMS/CHEMISTRY/MetaboliteSpectralDB.mzML
120 # wget -nc https://abibuilder.cs.uni-tuebingen.de/archive/openms/Tutorials/Data/latest/Example_Data/Metabolomics/databases/MetaboliteSpectralDB.mzML
118 mv MetaboliteSpectralDB.mzML test-data/ 121 mv MetaboliteSpectralDB.mzML test-data/
119 fi 122 fi
120 ln -fs TOFCalibration_ref_masses test-data/TOFCalibration_ref_masses.txt 123 ln -fs TOFCalibration_ref_masses test-data/TOFCalibration_ref_masses.txt
121 ln -fs TOFCalibration_const test-data/TOFCalibration_const.csv 124 ln -fs TOFCalibration_const test-data/TOFCalibration_const.csv
122 125
123 if [ ! -d test-data/pepnovo_models/ ]; then 126 # if [ ! -d test-data/pepnovo_models/ ]; then
124 mkdir -p /tmp/pepnovo 127 # mkdir -p /tmp/pepnovo
125 wget -nc http://proteomics.ucsd.edu/Software/PepNovo/PepNovo.20120423.zip 128 # wget -nc http://proteomics.ucsd.edu/Software/PepNovo/PepNovo.20120423.zip
126 unzip PepNovo.20120423.zip -d /tmp/pepnovo/ 129 # unzip PepNovo.20120423.zip -d /tmp/pepnovo/
127 mv /tmp/pepnovo/Models test-data/pepnovo_models/ 130 # mv /tmp/pepnovo/Models test-data/pepnovo_models/
128 rm PepNovo.20120423.zip 131 # rm PepNovo.20120423.zip
129 rm -rf /tmp/pepnovo 132 # rm -rf /tmp/pepnovo
130 fi 133 # fi
131 ############################################################################### 134 ###############################################################################
132 ## generate ctd files using the binaries in the conda package 135 ## generate ctd files using the binaries in the conda package
133 ############################################################################### 136 ###############################################################################
134 echo "Create CTD files" 137 echo "Create CTD files"
135 conda activate $OPENMSENV 138 conda activate $OPENMSENV
168 done 171 done
169 172
170 ############################################################################### 173 ###############################################################################
171 ## create script to create results for the tests and run it 174 ## create script to create results for the tests and run it
172 ############################################################################### 175 ###############################################################################
176 # parse data preparation calls from OpenMS sources for a tool with a given id
177 function prepare_test_data {
178 # id=$1
179 # | egrep -i "$id\_.*[0-9]+(_prepare\"|_convert)?"
180
181 OLD_OSW_PARAM=$(cat $OPENMSGIT/src/tests/topp/CMakeLists.txt |sed 's/#.*$//'| sed 's/^\s*//; s/\s*$//' |awk '{printf("%s@NEWLINE@", $0)}' | sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' | grep OLD_OSW_PARAM | head -n 1 | sed 's/^[^"]\+//; s/)$//; s/"//g')
182 # TODO SiriusAdapter depends on online service which may timeout .. so keep disabled https://github.com/OpenMS/OpenMS/pull/5010
183 cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake |
184 sed "s/\${OLD_OSW_PARAM}/$OLD_OSW_PARAM/" |
185 grep -v "\.ini\.json" |
186 sed 's/.ini.json /ini /' |
187 sed 's/#.*$//'|
188 sed 's/^\s*//; s/\s*$//' |
189 grep -v "^$" |
190 awk '{printf("%s@NEWLINE@", $0)}' |
191 sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' |
192 sed 's/degenerate_cases\///' |
193 egrep -v "WRITEINI|WRITECTD|INVALIDVALUE|DIFF" |
194 grep add_test |
195 egrep "TOPP|UTILS" |
196 sed 's@${DATA_DIR_SHARE}/@@g;'|
197 sed 's@${TMP_RIP_PATH}@./@g'|
198 sed 's@TOFCalibration_ref_masses @TOFCalibration_ref_masses.txt @g; s@TOFCalibration_const @TOFCalibration_const.csv @'|
199 sed 's/\("TOPP_SiriusAdapter_4".*\)-sirius:database all\(.*\)/\1-sirius:database pubchem\2/' |
200 while read line
201 do
202 test_id=$(echo "$line" | sed 's/add_test(//; s/"//g; s/)[^)]*$//; s/\${TOPP_BIN_PATH}\///g;s/\${DATA_DIR_TOPP}\///g; s#THIRDPARTY/##g' | cut -d" " -f1)
203
204 if grep -lq "$test_id"'\".* PROPERTIES WILL_FAIL 1' $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake; then
205 >&2 echo " skip failing "$test_id
206 continue
207 fi
208
209 line=$(echo "$line" | sed 's/add_test("//; s/)[^)]*$//; s/\${TOPP_BIN_PATH}\///g;s/\${DATA_DIR_TOPP}\///g; s#THIRDPARTY/##g' | cut -d" " -f2-)
210 # line="$(fix_tmp_files $line)"
211 echo 'echo executing "'$test_id'"'
212 echo "$line > $test_id.stdout 2> $test_id.stderr"
213 echo "if [[ \"\$?\" -ne \"0\" ]]; then >&2 echo '$test_id failed'; >&2 echo -e \"stderr:\n\$(cat $test_id.stderr | sed 's/^/ /')\"; echo -e \"stdout:\n\$(cat $test_id.stdout)\";fi"
214 done
215 }
216
173 echo "Create test shell script" 217 echo "Create test shell script"
174 218
175 echo -n "" > prepare_test_data.sh 219 echo -n "" > prepare_test_data.sh
176 echo 'export COMET_BINARY="comet"' >> prepare_test_data.sh 220 echo 'export COMET_BINARY="comet"' >> prepare_test_data.sh
177 echo 'export CRUX_BINARY="crux"' >> prepare_test_data.sh 221 echo 'export CRUX_BINARY="crux"' >> prepare_test_data.sh
178 echo 'export FIDOCHOOSEPARAMS_BINARY="FidoChooseParameters"' >> prepare_test_data.sh 222 echo 'export FIDOCHOOSEPARAMS_BINARY="FidoChooseParameters"' >> prepare_test_data.sh
179 echo 'export FIDO_BINARY="Fido"' >> prepare_test_data.sh 223 echo 'export FIDO_BINARY="Fido"' >> prepare_test_data.sh
180 echo 'export LUCIPHOR_BINARY="$(dirname $(realpath $(which luciphor2)))/luciphor2.jar"' >> prepare_test_data.sh 224 echo 'export LUCIPHOR_BINARY="$(dirname $(realpath $(which luciphor2)))/luciphor2.jar"' >> prepare_test_data.sh
181 225
182 echo 'export MARACLUSTER_BINARY="'"$OPENMSGIT"'/THIRDPARTY/Linux/64bit/MaRaCluster/maracluster"'>> prepare_test_data.sh 226 echo 'export MARACLUSTER_BINARY="'"$OPENMSGIT"'/THIRDPARTY/Linux/64bit/MaRaCluster/maracluster"'>> prepare_test_data.sh
183 echo 'export MSFRAGGER_BINARY="/home/berntm/Downloads/MSFragger-20171106/MSFragger-20171106.jar"'>> prepare_test_data.sh 227 echo 'export MSFRAGGER_BINARY="/home/berntm/Downloads/MSFragger-3.5/MSFragger-3.5.jar"'>> prepare_test_data.sh
184 echo 'export MSGFPLUS_BINARY="$(msgf_plus -get_jar_path)"' >> prepare_test_data.sh 228 echo 'export MSGFPLUS_BINARY="$(msgf_plus -get_jar_path)"' >> prepare_test_data.sh
185 echo 'export MYRIMATCH_BINARY="myrimatch"'>> prepare_test_data.sh 229 echo 'export MYRIMATCH_BINARY="myrimatch"'>> prepare_test_data.sh
186 echo 'export NOVOR_BINARY="/home/berntm/Downloads/novor/lib/novor.jar"' >> prepare_test_data.sh 230 echo 'export NOVOR_BINARY="/home/berntm/Downloads/novor/lib/novor.jar"' >> prepare_test_data.sh
187 echo 'export OMSSA_BINARY="$(dirname $(realpath $(which omssacl)))/omssacl"'>> prepare_test_data.sh
188 echo 'export PERCOLATOR_BINARY="percolator"'>> prepare_test_data.sh 231 echo 'export PERCOLATOR_BINARY="percolator"'>> prepare_test_data.sh
189 echo 'export SIRIUS_BINARY="$(which sirius)"' >> prepare_test_data.sh 232 echo 'export SIRIUS_BINARY="$(which sirius)"' >> prepare_test_data.sh
190 echo 'export SPECTRAST_BINARY="'"$OPENMSGIT"'/THIRDPARTY/Linux/64bit/SpectraST/spectrast"' >> prepare_test_data.sh 233 echo 'export SPECTRAST_BINARY="'"$OPENMSGIT"'/THIRDPARTY/Linux/64bit/SpectraST/spectrast"' >> prepare_test_data.sh
191 echo 'export XTANDEM_BINARY="xtandem"' >> prepare_test_data.sh 234 echo 'export XTANDEM_BINARY="xtandem"' >> prepare_test_data.sh
192 echo 'export THERMORAWFILEPARSER_BINARY="ThermoRawFileParser.exe"' >> prepare_test_data.sh 235 echo 'export THERMORAWFILEPARSER_BINARY="ThermoRawFileParser.exe"' >> prepare_test_data.sh
236 echo 'export SAGE_BINARY=sage' >> prepare_test_data.sh
193 237
194 prepare_test_data >> prepare_test_data.sh #tmp_test_data.sh 238 prepare_test_data >> prepare_test_data.sh #tmp_test_data.sh
195
196 ## prepare_test_data > tmp_test_data.sh
197 ## # remove calls not needed for the tools listed in any .list file
198 ## echo LIST $LIST
199 ## if [ ! -z "$LIST" ]; then
200 ## REX=$(echo $LIST | sed 's/ /\n/g' | sed 's@.*/\([^/]\+\).xml$@\1@' | tr '\n' '|' | sed 's/|$//')
201 ## else
202 ## REX=".*"
203 ## fi
204 ## echo REX $REX
205 ## cat tmp_test_data.sh | egrep "($REX)" >> prepare_test_data.sh
206 ## rm tmp_test_data.sh
207 239
208 echo "Execute test shell script" 240 echo "Execute test shell script"
209 chmod u+x prepare_test_data.sh 241 chmod u+x prepare_test_data.sh
210 cd ./test-data || exit 242 cd ./test-data || exit
211 ../prepare_test_data.sh 243 ../prepare_test_data.sh
232 echo "<macros>" > "$autotests" 264 echo "<macros>" > "$autotests"
233 265
234 for i in $(ls ctd/*ctd) 266 for i in $(ls ctd/*ctd)
235 do 267 do
236 b=$(basename "$i" .ctd) 268 b=$(basename "$i" .ctd)
237 get_tests2 "$b" >> "$autotests" 269 ./get_tests.py --id "$b" --cmake "$OPENMSGIT"/src/tests/topp/CMakeLists.txt "$OPENMSGIT"/src/tests/topp/THIRDPARTY/third_party_tests.cmake >> "$autotests"
270 wc -l "$autotests"
238 done 271 done
239 echo "</macros>" >> "$autotests" 272 echo "</macros>" >> "$autotests"
240
241 # echo "Create test data links"
242 # Breaks DecoyDatabase
243 # link_tmp_files
244 273
245 # tests for tools using output_prefix parameters can not be auto generated 274 # tests for tools using output_prefix parameters can not be auto generated
246 # hence we output the tests for manual curation in macros_test.xml 275 # hence we output the tests for manual curation in macros_test.xml
247 # and remove them from the autotests 276 # and remove them from the autotests
248 # -> OpenSwathFileSplitter IDRipper MzMLSplitter SeedListGenerator 277 # -> OpenSwathFileSplitter IDRipper MzMLSplitter SeedListGenerator
253 # Furthermore we remove tests for tools without binaries in conda 282 # Furthermore we remove tests for tools without binaries in conda
254 # -> MSFragger MaRaClusterAdapter NovorAdapter 283 # -> MSFragger MaRaClusterAdapter NovorAdapter
255 # 284 #
256 # not able to specify composite test data 285 # not able to specify composite test data
257 # -> SpectraSTSearchAdapter 286 # -> SpectraSTSearchAdapter
287 echo "Discard some tests"
258 if [[ ! -z "$1" ]]; then 288 if [[ ! -z "$1" ]]; then
259 echo "" > macros_discarded_auto.xml 289 echo "" > macros_discarded_auto.xml
260 for i in OpenSwathFileSplitter IDRipper MzMLSplitter SeedListGenerator MSFraggerAdapter MaRaClusterAdapter NovorAdapter SpectraSTSearchAdapter 290 for i in OpenSwathFileSplitter IDRipper MzMLSplitter SeedListGenerator MSFraggerAdapter MaRaClusterAdapter NovorAdapter SpectraSTSearchAdapter
261 do 291 do
262 echo "<xml name=\"manutest_$i\">" >> macros_discarded_auto.xml 292 echo "<xml name=\"manutest_$i\">" >> macros_discarded_auto.xml
270 conda deactivate 300 conda deactivate
271 301
272 ## remove broken symlinks in test-data 302 ## remove broken symlinks in test-data
273 find test-data/ -xtype l -delete 303 find test-data/ -xtype l -delete
274 304
275 if [ ! -z "$created" ]; then 305 if [[ "$created" == "yes" ]]; then
276 echo "Removing temporary directory" 306 echo "Removing temporary directory"
277 rm -rf "$tmp" 307 rm -rf "$tmp"
278 fi 308 fi