Mercurial > repos > galaxyp > openms_triqlerconverter

comparison test-data.sh @ 0:fb0ebc6f97f3 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload for repository https://github.com/galaxyproteomics/tools-galaxyp/tree/master/tools/openms commit 3d1e5f37fd16524a415f707772eeb7ead848c5e3
author galaxyp
date Thu, 01 Dec 2022 19:22:27 +0000
parents
children f9675543622a
comparison
equal deleted inserted replaced
-1:000000000000 0:fb0ebc6f97f3
1 #!/usr/bin/env bash
2
3 VERSION=2.8
4 FILETYPES="aux/filetypes.txt"
5 CONDAPKG="https://anaconda.org/bioconda/openms/2.8.0/download/linux-64/openms-2.8.0-h7ca0330_0.tar.bz2"
6
7 # import the magic
8 . ./generate-foo.sh
9
10 # install conda
11 if [ -z "$tmp" ]; then
12 tmp=$(mktemp -d)
13 created="yes"
14 fi
15
16 export OPENMSGIT="$tmp/OpenMS$VERSION.0-git"
17 export OPENMSPKG="$tmp/OpenMS$VERSION-pkg/"
18 export OPENMSENV="OpenMS$VERSION-env"
19
20 if [ -z "$CTDCONVERTER" ]; then
21 export CTDCONVERTER="$tmp/CTDConverter"
22 fi
23
24 if [[ -z "$1" ]]; then
25 autotests="/dev/null"
26 else
27 autotests="$1"
28 fi
29
30 if type conda > /dev/null; then
31 true
32 else
33 wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
34 bash Miniconda3-latest-Linux-x86_64.sh -b -p "$tmp/miniconda"
35 source "$tmp/miniconda/bin/activate"
36 fi
37 eval "$(conda shell.bash hook)"
38
39
40 ###############################################################################
41 ## get
42 ## - conda environment (for executing the binaries) and
43 ## - the git clone of OpenMS (for generating the tests)
44 ###############################################################################
45
46 echo "Clone OpenMS $VERSION sources"
47 if [[ ! -d $OPENMSGIT ]]; then
48 # TODO >2.8 reenable original release branch .. also in else branch
49 # the plus branch contains commits from https://github.com/OpenMS/OpenMS/pull/5920 and https://github.com/OpenMS/OpenMS/pull/5917
50 # git clone -b release/$VERSION.0 https://github.com/OpenMS/OpenMS.git $OPENMSGIT
51 git clone -b release/$VERSION.0-plus https://github.com/bernt-matthias/OpenMS.git $OPENMSGIT
52 cd $OPENMSGIT
53 git submodule init
54 git submodule update
55 cd -
56 else
57 cd $OPENMSGIT
58 git pull origin release/$VERSION.0-plus
59 cd -
60 fi
61
62 echo "Create OpenMS $VERSION conda env"
63 # TODO currently add lxml (needed by CTDConverter)
64 # TODO for some reason a to recent openjdk is used
65 if conda env list | grep "$OPENMSENV"; then
66 true
67 else
68 conda create -y --quiet --override-channels --channel iuc --channel conda-forge --channel bioconda --channel defaults -n $OPENMSENV openms=$VERSION openms-thirdparty=$VERSION omssa=2.1.9 ctdopts=1.5 lxml
69 # chmod -R u-w $OPENMSENV
70 fi
71 ###############################################################################
72 ## get the
73 ## - conda package (for easy access and listing of the OpenMS binaries),
74 ###############################################################################
75 echo "Download OpenMS $VERSION package $CONDAPKG"
76
77 if [[ ! -d $OPENMSPKG ]]; then
78 mkdir $OPENMSPKG
79 wget -q -P $OPENMSPKG/ "$CONDAPKG"
80 tar -xf $OPENMSPKG/"$(basename $CONDAPKG)" -C $OPENMSPKG/
81 rm $OPENMSPKG/"$(basename $CONDAPKG)"
82 fi
83
84 ###############################################################################
85 ## Get python libaries for CTD -> Galaxy conversion
86 ## TODO fix to main repo OR conda packkage if PRs are merged
87 ###############################################################################
88 echo "Clone CTDConverter"
89 if [[ ! -d $CTDCONVERTER ]]; then
90 #git clone https://github.com/WorkflowConversion/CTDConverter.git CTDConverter
91 git clone -b topic/fix-selects2 https://github.com/bernt-matthias/CTDConverter.git $CTDCONVERTER
92 else
93 cd $CTDCONVERTER
94 git pull origin topic/fix-selects2
95 cd -
96 fi
97 conda activate $OPENMSENV
98 cd $CTDCONVERTER
99 python -m pip install . --no-deps
100 cd -
101 conda deactivate
102
103
104 # ###############################################################################
105 # ## copy all the test data files to test-data
106 # ## most of it (outputs) will be overwritten later, but its needed for
107 # ## prepare_test_data
108 # ###############################################################################
109 echo "Get test data"
110 find test-data -type f,l,d ! -name "*fa" ! -name "*loc" ! -name "test-data" -delete
111
112 cp $(find $OPENMSGIT/src/tests/topp/ -type f | grep -Ev "third_party_tests.cmake|CMakeLists.txt|check_ini") test-data/
113 cp -r $OPENMSGIT/share/OpenMS/MAPPING/ test-data/
114 cp -r $OPENMSGIT/share/OpenMS/CHEMISTRY test-data/
115 cp -r $OPENMSGIT/share/OpenMS/examples/ test-data/
116 if [ ! -f test-data/MetaboliteSpectralDB.mzML ]; then
117 wget -nc https://abibuilder.cs.uni-tuebingen.de/archive/openms/Tutorials/Data/latest/Example_Data/Metabolomics/databases/MetaboliteSpectralDB.mzML
118 mv MetaboliteSpectralDB.mzML test-data/
119 fi
120 ln -fs TOFCalibration_ref_masses test-data/TOFCalibration_ref_masses.txt
121 ln -fs TOFCalibration_const test-data/TOFCalibration_const.csv
122
123 if [ ! -d test-data/pepnovo_models/ ]; then
124 mkdir -p /tmp/pepnovo
125 wget -nc http://proteomics.ucsd.edu/Software/PepNovo/PepNovo.20120423.zip
126 unzip PepNovo.20120423.zip -d /tmp/pepnovo/
127 mv /tmp/pepnovo/Models test-data/pepnovo_models/
128 rm PepNovo.20120423.zip
129 rm -rf /tmp/pepnovo
130 fi
131 ###############################################################################
132 ## generate ctd files using the binaries in the conda package
133 ###############################################################################
134 echo "Create CTD files"
135 conda activate $OPENMSENV
136 rm -rf ctd
137 mkdir -p ctd
138
139 for i in $OPENMSPKG/bin/*
140 do
141 b=$(basename $i)
142 echo $b
143 $b -write_ctd ctd/
144 sed -i -e 's/²/^2/' ctd/$b.ctd
145 done
146 ###############################################################################
147 ## fix ini files: OpenMS test data contains ini files with outdated ini files.
148 ## e.g. variables might be in different nodes, outdated variables present, new
149 ## variables missing, ...
150 ## OpenMS tools fix this on the fly (so its no problem for the OpenMS tests)
151 ## but it is for the generation of the tests
152 ## see https://github.com/OpenMS/OpenMS/issues/4462
153 ###############################################################################
154 echo "Update test INI files"
155 for ini in test-data/*ini
156 do
157 tool=$(cat $ini | grep 'NODE name="' | head -n 1 | sed 's/.*name="\([^"]\+\)".*/\1/')
158 bin=$(which $tool)
159 if [[ -z $bin ]]; then
160 >&2 echo "missing binary to convert $ini"
161 continue
162 fi
163 cp $ini $ini.backup
164 $bin -ini $ini -write_ini $ini > $ini.stdout 2> $ini.stderr
165 if [[ "$?" -ne "0" ]]; then
166 >&2 echo "could not convert $ini"
167 fi
168 done
169
170 ###############################################################################
171 ## create script to create results for the tests and run it
172 ###############################################################################
173 echo "Create test shell script"
174
175 echo -n "" > prepare_test_data.sh
176 echo 'export COMET_BINARY="comet"' >> prepare_test_data.sh
177 echo 'export CRUX_BINARY="crux"' >> prepare_test_data.sh
178 echo 'export FIDOCHOOSEPARAMS_BINARY="FidoChooseParameters"' >> prepare_test_data.sh
179 echo 'export FIDO_BINARY="Fido"' >> prepare_test_data.sh
180 echo 'export LUCIPHOR_BINARY="$(dirname $(realpath $(which luciphor2)))/luciphor2.jar"' >> prepare_test_data.sh
181
182 echo 'export MARACLUSTER_BINARY="'"$OPENMSGIT"'/THIRDPARTY/Linux/64bit/MaRaCluster/maracluster"'>> prepare_test_data.sh
183 echo 'export MSFRAGGER_BINARY="/home/berntm/Downloads/MSFragger-20171106/MSFragger-20171106.jar"'>> prepare_test_data.sh
184 echo 'export MSGFPLUS_BINARY="$(msgf_plus -get_jar_path)"' >> prepare_test_data.sh
185 echo 'export MYRIMATCH_BINARY="myrimatch"'>> prepare_test_data.sh
186 echo 'export NOVOR_BINARY="/home/berntm/Downloads/novor/lib/novor.jar"' >> prepare_test_data.sh
187 echo 'export OMSSA_BINARY="$(dirname $(realpath $(which omssacl)))/omssacl"'>> prepare_test_data.sh
188 echo 'export PERCOLATOR_BINARY="percolator"'>> prepare_test_data.sh
189 echo 'export SIRIUS_BINARY="$(which sirius)"' >> prepare_test_data.sh
190 echo 'export SPECTRAST_BINARY="'"$OPENMSGIT"'/THIRDPARTY/Linux/64bit/SpectraST/spectrast"' >> prepare_test_data.sh
191 echo 'export XTANDEM_BINARY="xtandem"' >> prepare_test_data.sh
192 echo 'export THERMORAWFILEPARSER_BINARY="ThermoRawFileParser.exe"' >> prepare_test_data.sh
193
194 prepare_test_data >> prepare_test_data.sh #tmp_test_data.sh
195
196 ## prepare_test_data > tmp_test_data.sh
197 ## # remove calls not needed for the tools listed in any .list file
198 ## echo LIST $LIST
199 ## if [ ! -z "$LIST" ]; then
200 ## REX=$(echo $LIST | sed 's/ /\n/g' | sed 's@.*/\([^/]\+\).xml$@\1@' | tr '\n' '|' | sed 's/|$//')
201 ## else
202 ## REX=".*"
203 ## fi
204 ## echo REX $REX
205 ## cat tmp_test_data.sh | egrep "($REX)" >> prepare_test_data.sh
206 ## rm tmp_test_data.sh
207
208 echo "Execute test shell script"
209 chmod u+x prepare_test_data.sh
210 cd ./test-data || exit
211 ../prepare_test_data.sh
212 cd - || exit
213
214
215 # ###############################################################################
216 # ## create/update test data for the manually generated tests
217 # ## - run convert once with the manual tests only and
218 # ## - update test-data (needs to run 2x)
219 # ###############################################################################
220 echo "Execute test shell script for manually curated tests"
221 chmod u+x prepare_test_data_manual.sh
222 cd ./test-data || exit
223 ../prepare_test_data_manual.sh
224 cd - || exit
225
226
227 ###############################################################################
228 ## auto generate tests
229 ###############################################################################
230
231 echo "Write test macros to $autotests"
232 echo "<macros>" > "$autotests"
233
234 for i in $(ls ctd/*ctd)
235 do
236 b=$(basename "$i" .ctd)
237 get_tests2 "$b" >> "$autotests"
238 done
239 echo "</macros>" >> "$autotests"
240
241 # echo "Create test data links"
242 # Breaks DecoyDatabase
243 # link_tmp_files
244
245 # tests for tools using output_prefix parameters can not be auto generated
246 # hence we output the tests for manual curation in macros_test.xml
247 # and remove them from the autotests
248 # -> OpenSwathFileSplitter IDRipper MzMLSplitter SeedListGenerator
249 # TODO reevaluate in >2.8
250 # - https://github.com/OpenMS/OpenMS/pull/5873
251 # - https://github.com/OpenMS/OpenMS/pull/5912
252 #
253 # Furthermore we remove tests for tools without binaries in conda
254 # -> MSFragger MaRaClusterAdapter NovorAdapter
255 #
256 # not able to specify composite test data
257 # -> SpectraSTSearchAdapter
258 if [[ ! -z "$1" ]]; then
259 echo "" > macros_discarded_auto.xml
260 for i in OpenSwathFileSplitter IDRipper MzMLSplitter SeedListGenerator MSFraggerAdapter MaRaClusterAdapter NovorAdapter SpectraSTSearchAdapter
261 do
262 echo "<xml name=\"manutest_$i\">" >> macros_discarded_auto.xml
263 xmlstarlet sel -t -c "/macros/xml[@name='autotest_$i']/test" macros_autotest.xml >> macros_discarded_auto.xml
264 echo "</xml>" >> macros_discarded_auto.xml
265 xmlstarlet ed -d "/macros/xml[@name='autotest_$i']/test" macros_autotest.xml > tmp
266 mv tmp macros_autotest.xml
267 done
268 >&2 echo "discarded autogenerated macros for curation in macros_discarded_auto.xml"
269 fi
270 conda deactivate
271
272 ## remove broken symlinks in test-data
273 find test-data/ -xtype l -delete
274
275 if [ ! -z "$created" ]; then
276 echo "Removing temporary directory"
277 rm -rf "$tmp"
278 fi