Mercurial > repos > galaxyp > openms_sageadapter

comparison test-data.sh @ 0:3ebae1b284b9 draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload for repository https://github.com/galaxyproteomics/tools-galaxyp/tree/master/tools/openms commit 5c080b1e2b99f1c88f4557e9fec8c45c9d23b906
author galaxyp
date Fri, 14 Jun 2024 21:28:32 +0000
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:3ebae1b284b9
1 #!/usr/bin/env bash
2
3 # set -x
4
5 VERSION=3.1
6 FILETYPES="aux/filetypes.txt"
7 CONDAPKG="https://anaconda.org/bioconda/openms/3.1.0/download/linux-64/openms-3.1.0-h8964181_1.tar.bz2"
8
9 # install conda
10 if [ -z "$tmp" ]; then
11 tmp=$(mktemp -d)
12 created="yes"
13 fi
14
15 export OPENMSGIT="$tmp/OpenMS$VERSION.0-git"
16 export OPENMSPKG="$tmp/OpenMS$VERSION-pkg/"
17 export OPENMSENV="OpenMS$VERSION-env"
18
19 if [ -z "$CTDCONVERTER" ]; then
20 export CTDCONVERTER="$tmp/CTDConverter"
21 fi
22
23 if [[ -z "$1" ]]; then
24 autotests="/dev/null"
25 else
26 autotests="$1"
27 fi
28
29 if type conda > /dev/null; then
30 true
31 else
32 wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
33 bash Miniconda3-latest-Linux-x86_64.sh -b -p "$tmp/miniconda"
34 source "$tmp/miniconda/bin/activate"
35 fi
36 eval "$(conda shell.bash hook)"
37
38
39 ###############################################################################
40 ## get
41 ## - conda environment (for executing the binaries) and
42 ## - the git clone of OpenMS (for generating the tests)
43 ###############################################################################
44
45 echo "Clone OpenMS $VERSION sources"
46 if [[ ! -d $OPENMSGIT ]]; then
47 if [[ "$created" == "yes" ]]; then
48 GIT_DIR=$(mktemp -d --dry-run)
49 GIT_EXTRA_OPTS="--separate-git-dir=$GIT_DIR"
50 fi
51 git clone -b release/$VERSION.0 --depth 1 --recurse-submodules=THIRDPARTY --shallow-submodules $GIT_EXTRA_OPTS https://github.com/OpenMS/OpenMS.git $OPENMSGIT
52 ## save some space by just keeping the needed binaries
53 find $OPENMSGIT/THIRDPARTY/ -type f -not \( -name maracluster -o -name spectrast \) -delete
54 find $OPENMSGIT/THIRDPARTY/ -empty -type d -delete
55 if [[ "$created" == "yes" ]]; then
56 rm -rf $GIT_DIR
57 fi
58 else
59 cd $OPENMSGIT
60 git pull origin release/$VERSION.0
61 cd -
62 fi
63
64 echo "Create OpenMS $VERSION conda env"
65 # TODO currently add lxml (needed by CTDConverter)
66 # TODO for some reason a to recent openjdk is used
67 if conda env list | grep "$OPENMSENV"; then
68 true
69 else
70 conda create -y --quiet --solver libmamba --override-channels --strict-channel-priority --channel conda-forge --channel bioconda -n $OPENMSENV openms=$VERSION openms-thirdparty=$VERSION ctdopts=1.5 lxml
71 # chmod -R u-w $OPENMSENV
72 fi
73 ###############################################################################
74 ## get the
75 ## - conda package (for easy access and listing of the OpenMS binaries),
76 ###############################################################################
77 echo "Download OpenMS $VERSION package $CONDAPKG"
78
79 if [[ ! -d $OPENMSPKG ]]; then
80 mkdir $OPENMSPKG
81 wget -q -P $OPENMSPKG/ "$CONDAPKG"
82 tar -xf $OPENMSPKG/"$(basename $CONDAPKG)" -C $OPENMSPKG/
83 rm $OPENMSPKG/"$(basename $CONDAPKG)"
84 fi
85
86 ###############################################################################
87 ## Get python libaries for CTD -> Galaxy conversion
88 ## TODO fix to main repo OR conda packkage if PRs are merged
89 ###############################################################################
90 echo "Clone CTDConverter"
91 if [[ ! -d $CTDCONVERTER ]]; then
92 #git clone https://github.com/WorkflowConversion/CTDConverter.git CTDConverter
93 git clone -b topic/fix-selects2 https://github.com/bernt-matthias/CTDConverter.git $CTDCONVERTER
94 else
95 cd $CTDCONVERTER
96 git pull origin topic/fix-selects2
97 cd -
98 fi
99 conda activate $OPENMSENV
100 cd $CTDCONVERTER
101 python -m pip install . --no-deps
102 cd -
103 conda deactivate
104
105
106 # # ###############################################################################
107 # # ## copy all the test data files to test-data
108 # # ## most of it (outputs) will be overwritten later, but its needed for
109 # # ## prepare_test_data
110 # # ###############################################################################
111 echo "Get test data"
112 find test-data -type f,l,d ! -name "*fa" ! -name "*loc" ! -name "test-data" ! -name MetaboliteSpectralDB.mzML -delete
113
114 cp $(find $OPENMSGIT/src/tests/topp/ -type f | grep -Ev "third_party_tests.cmake|CMakeLists.txt|check_ini") test-data/
115 cp -r $OPENMSGIT/share/OpenMS/MAPPING/ test-data/
116 cp -r $OPENMSGIT/share/OpenMS/CHEMISTRY test-data/
117 cp -r $OPENMSGIT/share/OpenMS/examples/ test-data/
118 if [ ! -f test-data/MetaboliteSpectralDB.mzML ]; then
119 wget -nc https://raw.githubusercontent.com/sneumann/OpenMS/master/share/OpenMS/CHEMISTRY/MetaboliteSpectralDB.mzML
120 # wget -nc https://abibuilder.cs.uni-tuebingen.de/archive/openms/Tutorials/Data/latest/Example_Data/Metabolomics/databases/MetaboliteSpectralDB.mzML
121 mv MetaboliteSpectralDB.mzML test-data/
122 fi
123 ln -fs TOFCalibration_ref_masses test-data/TOFCalibration_ref_masses.txt
124 ln -fs TOFCalibration_const test-data/TOFCalibration_const.csv
125
126 # if [ ! -d test-data/pepnovo_models/ ]; then
127 # mkdir -p /tmp/pepnovo
128 # wget -nc http://proteomics.ucsd.edu/Software/PepNovo/PepNovo.20120423.zip
129 # unzip PepNovo.20120423.zip -d /tmp/pepnovo/
130 # mv /tmp/pepnovo/Models test-data/pepnovo_models/
131 # rm PepNovo.20120423.zip
132 # rm -rf /tmp/pepnovo
133 # fi
134 ###############################################################################
135 ## generate ctd files using the binaries in the conda package
136 ###############################################################################
137 echo "Create CTD files"
138 conda activate $OPENMSENV
139 rm -rf ctd
140 mkdir -p ctd
141
142 for i in $OPENMSPKG/bin/*
143 do
144 b=$(basename $i)
145 echo $b
146 $b -write_ctd ctd/
147 sed -i -e 's/²/^2/' ctd/$b.ctd
148 done
149 ###############################################################################
150 ## fix ini files: OpenMS test data contains ini files with outdated ini files.
151 ## e.g. variables might be in different nodes, outdated variables present, new
152 ## variables missing, ...
153 ## OpenMS tools fix this on the fly (so its no problem for the OpenMS tests)
154 ## but it is for the generation of the tests
155 ## see https://github.com/OpenMS/OpenMS/issues/4462
156 ###############################################################################
157 echo "Update test INI files"
158 for ini in test-data/*ini
159 do
160 tool=$(cat $ini | grep 'NODE name="' | head -n 1 | sed 's/.*name="\([^"]\+\)".*/\1/')
161 bin=$(which $tool)
162 if [[ -z $bin ]]; then
163 >&2 echo "missing binary to convert $ini"
164 continue
165 fi
166 cp $ini $ini.backup
167 $bin -ini $ini -write_ini $ini > $ini.stdout 2> $ini.stderr
168 if [[ "$?" -ne "0" ]]; then
169 >&2 echo "could not convert $ini"
170 fi
171 done
172
173 ###############################################################################
174 ## create script to create results for the tests and run it
175 ###############################################################################
176 # parse data preparation calls from OpenMS sources for a tool with a given id
177 function prepare_test_data {
178 # id=$1
179 # | egrep -i "$id\_.*[0-9]+(_prepare\"|_convert)?"
180
181 OLD_OSW_PARAM=$(cat $OPENMSGIT/src/tests/topp/CMakeLists.txt |sed 's/#.*$//'| sed 's/^\s*//; s/\s*$//' |awk '{printf("%s@NEWLINE@", $0)}' | sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' | grep OLD_OSW_PARAM | head -n 1 | sed 's/^[^"]\+//; s/)$//; s/"//g')
182 # TODO SiriusAdapter depends on online service which may timeout .. so keep disabled https://github.com/OpenMS/OpenMS/pull/5010
183 cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake |
184 sed "s/\${OLD_OSW_PARAM}/$OLD_OSW_PARAM/" |
185 grep -v "\.ini\.json" |
186 sed 's/.ini.json /ini /' |
187 sed 's/#.*$//'|
188 sed 's/^\s*//; s/\s*$//' |
189 grep -v "^$" |
190 awk '{printf("%s@NEWLINE@", $0)}' |
191 sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' |
192 sed 's/degenerate_cases\///' |
193 egrep -v "WRITEINI|WRITECTD|INVALIDVALUE|DIFF" |
194 grep add_test |
195 egrep "TOPP|UTILS" |
196 sed 's@${DATA_DIR_SHARE}/@@g;'|
197 sed 's@${TMP_RIP_PATH}@./@g'|
198 sed 's@TOFCalibration_ref_masses @TOFCalibration_ref_masses.txt @g; s@TOFCalibration_const @TOFCalibration_const.csv @'|
199 sed 's/\("TOPP_SiriusAdapter_4".*\)-sirius:database all\(.*\)/\1-sirius:database pubchem\2/' |
200 while read line
201 do
202 test_id=$(echo "$line" | sed 's/add_test(//; s/"//g; s/)[^)]*$//; s/\${TOPP_BIN_PATH}\///g;s/\${DATA_DIR_TOPP}\///g; s#THIRDPARTY/##g' | cut -d" " -f1)
203
204 if grep -lq "$test_id"'\".* PROPERTIES WILL_FAIL 1' $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake; then
205 >&2 echo " skip failing "$test_id
206 continue
207 fi
208
209 line=$(echo "$line" | sed 's/add_test("//; s/)[^)]*$//; s/\${TOPP_BIN_PATH}\///g;s/\${DATA_DIR_TOPP}\///g; s#THIRDPARTY/##g' | cut -d" " -f2-)
210 # line="$(fix_tmp_files $line)"
211 echo 'echo executing "'$test_id'"'
212 echo "$line > $test_id.stdout 2> $test_id.stderr"
213 echo "if [[ \"\$?\" -ne \"0\" ]]; then >&2 echo '$test_id failed'; >&2 echo -e \"stderr:\n\$(cat $test_id.stderr | sed 's/^/ /')\"; echo -e \"stdout:\n\$(cat $test_id.stdout)\";fi"
214 done
215 }
216
217 echo "Create test shell script"
218
219 echo -n "" > prepare_test_data.sh
220 echo 'export COMET_BINARY="comet"' >> prepare_test_data.sh
221 echo 'export CRUX_BINARY="crux"' >> prepare_test_data.sh
222 echo 'export FIDOCHOOSEPARAMS_BINARY="FidoChooseParameters"' >> prepare_test_data.sh
223 echo 'export FIDO_BINARY="Fido"' >> prepare_test_data.sh
224 echo 'export LUCIPHOR_BINARY="$(dirname $(realpath $(which luciphor2)))/luciphor2.jar"' >> prepare_test_data.sh
225
226 echo 'export MARACLUSTER_BINARY="'"$OPENMSGIT"'/THIRDPARTY/Linux/64bit/MaRaCluster/maracluster"'>> prepare_test_data.sh
227 echo 'export MSFRAGGER_BINARY="/home/berntm/Downloads/MSFragger-3.5/MSFragger-3.5.jar"'>> prepare_test_data.sh
228 echo 'export MSGFPLUS_BINARY="$(msgf_plus -get_jar_path)"' >> prepare_test_data.sh
229 echo 'export MYRIMATCH_BINARY="myrimatch"'>> prepare_test_data.sh
230 echo 'export NOVOR_BINARY="/home/berntm/Downloads/novor/lib/novor.jar"' >> prepare_test_data.sh
231 echo 'export PERCOLATOR_BINARY="percolator"'>> prepare_test_data.sh
232 echo 'export SIRIUS_BINARY="$(which sirius)"' >> prepare_test_data.sh
233 echo 'export SPECTRAST_BINARY="'"$OPENMSGIT"'/THIRDPARTY/Linux/64bit/SpectraST/spectrast"' >> prepare_test_data.sh
234 echo 'export XTANDEM_BINARY="xtandem"' >> prepare_test_data.sh
235 echo 'export THERMORAWFILEPARSER_BINARY="ThermoRawFileParser.exe"' >> prepare_test_data.sh
236 echo 'export SAGE_BINARY=sage' >> prepare_test_data.sh
237
238 prepare_test_data >> prepare_test_data.sh #tmp_test_data.sh
239
240 echo "Execute test shell script"
241 chmod u+x prepare_test_data.sh
242 cd ./test-data || exit
243 ../prepare_test_data.sh
244 cd - || exit
245
246
247 # ###############################################################################
248 # ## create/update test data for the manually generated tests
249 # ## - run convert once with the manual tests only and
250 # ## - update test-data (needs to run 2x)
251 # ###############################################################################
252 echo "Execute test shell script for manually curated tests"
253 chmod u+x prepare_test_data_manual.sh
254 cd ./test-data || exit
255 ../prepare_test_data_manual.sh
256 cd - || exit
257
258
259 ###############################################################################
260 ## auto generate tests
261 ###############################################################################
262
263 echo "Write test macros to $autotests"
264 echo "<macros>" > "$autotests"
265
266 for i in $(ls ctd/*ctd)
267 do
268 b=$(basename "$i" .ctd)
269 ./get_tests.py --id "$b" --cmake "$OPENMSGIT"/src/tests/topp/CMakeLists.txt "$OPENMSGIT"/src/tests/topp/THIRDPARTY/third_party_tests.cmake >> "$autotests"
270 wc -l "$autotests"
271 done
272 echo "</macros>" >> "$autotests"
273
274 # tests for tools using output_prefix parameters can not be auto generated
275 # hence we output the tests for manual curation in macros_test.xml
276 # and remove them from the autotests
277 # -> OpenSwathFileSplitter IDRipper MzMLSplitter SeedListGenerator
278 # TODO reevaluate in >2.8
279 # - https://github.com/OpenMS/OpenMS/pull/5873
280 # - https://github.com/OpenMS/OpenMS/pull/5912
281 #
282 # Furthermore we remove tests for tools without binaries in conda
283 # -> MSFragger MaRaClusterAdapter NovorAdapter
284 #
285 # not able to specify composite test data
286 # -> SpectraSTSearchAdapter
287 echo "Discard some tests"
288 if [[ ! -z "$1" ]]; then
289 echo "" > macros_discarded_auto.xml
290 for i in OpenSwathFileSplitter IDRipper MzMLSplitter SeedListGenerator MSFraggerAdapter MaRaClusterAdapter NovorAdapter SpectraSTSearchAdapter
291 do
292 echo "<xml name=\"manutest_$i\">" >> macros_discarded_auto.xml
293 xmlstarlet sel -t -c "/macros/xml[@name='autotest_$i']/test" macros_autotest.xml >> macros_discarded_auto.xml
294 echo "</xml>" >> macros_discarded_auto.xml
295 xmlstarlet ed -d "/macros/xml[@name='autotest_$i']/test" macros_autotest.xml > tmp
296 mv tmp macros_autotest.xml
297 done
298 >&2 echo "discarded autogenerated macros for curation in macros_discarded_auto.xml"
299 fi
300 conda deactivate
301
302 ## remove broken symlinks in test-data
303 find test-data/ -xtype l -delete
304
305 if [[ "$created" == "yes" ]]; then
306 echo "Removing temporary directory"
307 rm -rf "$tmp"
308 fi