Mercurial > repos > galaxyp > openms_databasesuitability

comparison test-data.sh @ 0:d994c72a91fc draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload for repository https://github.com/galaxyproteomics/tools-galaxyp/tree/master/tools/openms commit 55a2aeba8bfd8a6910630721de9857dcdfe05d3c"
author galaxyp
date Tue, 13 Oct 2020 19:54:23 +0000
parents
children 4890f50ae1a2
comparison
equal deleted inserted replaced
-1:000000000000 0:d994c72a91fc
1 #!/usr/bin/env bash
2
3 VERSION=2.6
4 FILETYPES="filetypes.txt"
5 CONDAPKG="https://anaconda.org/bioconda/openms/2.6.0/download/linux-64/openms-2.6.0-h4afb90d_0.tar.bz2"
6
7 # import the magic
8 . ./generate-foo.sh
9
10 # install conda
11 if [ -z "$tmp" ]; then
12 tmp=$(mktemp -d)
13 created="yes"
14 fi
15
16 export OPENMSGIT="$tmp/OpenMS$VERSION.0-git"
17 export OPENMSPKG="$tmp/OpenMS$VERSION-pkg/"
18 export OPENMSENV="$tmp/OpenMS$VERSION-env"
19 export CTDCONVERTER="$tmp/CTDConverter"
20
21 if [[ -z "$1" ]]; then
22 autotests="/dev/null"
23 else
24 autotests="$1"
25 fi
26
27 if type conda > /dev/null; then
28 true
29 else
30 wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
31 bash Miniconda3-latest-Linux-x86_64.sh -b -p "$tmp/miniconda"
32 source "$tmp/miniconda/bin/activate"
33 fi
34 eval "$(conda shell.bash hook)"
35
36
37 ###############################################################################
38 ## get
39 ## - conda environment (for executing the binaries) and
40 ## - the git clone of OpenMS (for generating the tests)
41 ###############################################################################
42
43 echo "Clone OpenMS $VERSION sources"
44 if [[ ! -d $OPENMSGIT ]]; then
45 git clone -b release/$VERSION.0 https://github.com/OpenMS/OpenMS.git $OPENMSGIT
46 cd $OPENMSGIT
47 git submodule init
48 git submodule update
49 cd -
50 else
51 cd $OPENMSGIT
52 git pull origin release/$VERSION.0
53 cd -
54 fi
55
56 echo "Create OpenMS $VERSION conda env"
57 # TODO currently add lxml (needed by CTDConverter)
58 # TODO for some reason a to recent openjdk is used
59 if conda env list | grep "$OPENMSENV"; then
60 true
61 else
62 conda create -y --quiet --override-channels --channel iuc --channel conda-forge --channel bioconda --channel defaults -p $OPENMSENV openms=$VERSION openms-thirdparty=$VERSION ctdopts=1.4 lxml
63 # chmod -R u-w $OPENMSENV
64 fi
65 ###############################################################################
66 ## get the
67 ## - conda package (for easy access and listing of the OpenMS binaries),
68 ###############################################################################
69 echo "Download OpenMS $VERSION package $CONDAPKG"
70
71 if [[ ! -d $OPENMSPKG ]]; then
72 mkdir $OPENMSPKG
73 wget -q -P $OPENMSPKG/ "$CONDAPKG"
74 tar -xf $OPENMSPKG/"$(basename $CONDAPKG)" -C $OPENMSPKG/
75 rm $OPENMSPKG/"$(basename $CONDAPKG)"
76 fi
77
78 ###############################################################################
79 ## Get python libaries for CTD -> Galaxy conversion
80 ## TODO fix to main repo OR conda packkage if PRs are merged
81 ###############################################################################
82 echo "Clone CTDConverter"
83 if [[ ! -d $CTDCONVERTER ]]; then
84 #git clone https://github.com/WorkflowConversion/CTDConverter.git CTDConverter
85 git clone -b topic/cdata https://github.com/bernt-matthias/CTDConverter.git $CTDCONVERTER
86 else
87 cd $CTDCONVERTER
88 git pull origin topic/cdata
89 cd -
90 fi
91
92 ###############################################################################
93 ## copy all the test data files to test-data
94 ## most of it (outputs) will be overwritten later, but its needed for
95 ## prepare_test_data
96 ###############################################################################
97 echo "Get test data"
98 find test-data -type f,l,d ! -name "*fa" ! -name "*loc" -delete
99
100 cp $(find $OPENMSGIT/src/tests/topp/ -type f | grep -Ev "third_party_tests.cmake|CMakeLists.txt|check_ini") test-data/
101 cp -r $OPENMSGIT/share/OpenMS/MAPPING/ test-data/
102 cp -r $OPENMSGIT/share/OpenMS/CHEMISTRY test-data/
103 cp -r $OPENMSGIT/share/OpenMS/examples/ test-data/
104 if [[ ! -f test-data/MetaboliteSpectralDB.mzML ]]; then
105 wget -nc https://abibuilder.informatik.uni-tuebingen.de/archive/openms/Tutorials/Data/latest/Example_Data/Metabolomics/databases/MetaboliteSpectralDB.mzML
106 mv MetaboliteSpectralDB.mzML test-data/
107 fi
108 ln -fs TOFCalibration_ref_masses test-data/TOFCalibration_ref_masses.txt
109 ln -fs TOFCalibration_const test-data/TOFCalibration_const.csv
110
111 if [ ! -d test-data/pepnovo_models/ ]; then
112 mkdir -p /tmp/pepnovo
113 wget -nc http://proteomics.ucsd.edu/Software/PepNovo/PepNovo.20120423.zip
114 unzip PepNovo.20120423.zip -d /tmp/pepnovo/
115 mv /tmp/pepnovo/Models test-data/pepnovo_models/
116 rm PepNovo.20120423.zip
117 rm -rf /tmp/pepnovo
118 fi
119 ###############################################################################
120 ## generate ctd files using the binaries in the conda package
121 ###############################################################################
122 echo "Create CTD files"
123 conda activate $OPENMSENV
124 rm -rf ctd
125 mkdir -p ctd
126
127 # TODO because of https://github.com/OpenMS/OpenMS/issues/4641
128 # this needs to be done from within test-data
129 cd test-data
130 for i in $OPENMSPKG/bin/*
131 do
132 b=$(basename $i)
133 echo $b
134 $b -write_ctd ../ctd/
135 sed -i -e 's/²/^2/' ../ctd/$b.ctd
136 done
137 cd -
138 ###############################################################################
139 ## fix ini files: OpenMS test data contains ini files with outdated ini files.
140 ## e.g. variables might be in different nodes, outdated variables present, new
141 ## variables missing, ...
142 ## OpenMS tools fix this on the fly (so its no problem for the OpenMS tests)
143 ## but it is for the generation of the tests
144 ## see https://github.com/OpenMS/OpenMS/issues/4462
145 ###############################################################################
146 echo "Update test INI files"
147 for ini in test-data/*ini
148 do
149 tool=$(cat $ini | grep 'NODE name="' | head -n 1 | sed 's/.*name="\([^"]\+\)".*/\1/')
150 bin=$(which $tool)
151 if [[ -z $bin ]]; then
152 >&2 echo "missing binary to convert $ini"
153 continue
154 fi
155 cp $ini $ini.backup
156 $bin -ini $ini -write_ini $ini > $ini.stdout 2> $ini.stderr
157 if [[ "$?" -ne "0" ]]; then
158 >&2 echo "could not convert $ini"
159 fi
160 done
161
162 ###############################################################################
163 ## create script to create results for the tests and run it
164 ###############################################################################
165 echo "Create test shell script"
166
167 echo -n "" > prepare_test_data.sh
168 echo 'export COMET_BINARY="comet"' >> prepare_test_data.sh
169 echo 'export CRUX_BINARY="crux"' >> prepare_test_data.sh
170 echo 'export FIDOCHOOSEPARAMS_BINARY="FidoChooseParameters"' >> prepare_test_data.sh
171 echo 'export FIDO_BINARY="Fido"' >> prepare_test_data.sh
172 echo 'export LUCIPHOR_BINARY="$(dirname $(realpath $(which luciphor2)))/luciphor2.jar"' >> prepare_test_data.sh
173
174 echo 'export MARACLUSTER_BINARY="'"$OPENMSGIT"'/THIRDPARTY/Linux/64bit/MaRaCluster/maracluster"'>> prepare_test_data.sh
175 echo 'export MSFRAGGER_BINARY="/home/berntm/Downloads/MSFragger-20171106/MSFragger-20171106.jar"'>> prepare_test_data.sh
176 echo 'export MSGFPLUS_BINARY="$(msgf_plus -get_jar_path)"' >> prepare_test_data.sh
177 echo 'export MYRIMATCH_BINARY="myrimatch"'>> prepare_test_data.sh
178 echo 'export NOVOR_BINARY="/home/berntm/Downloads/novor/lib/novor.jar"' >> prepare_test_data.sh
179 echo 'export OMSSA_BINARY="$(dirname $(realpath $(which omssacl)))/omssacl"'>> prepare_test_data.sh
180 echo 'export PERCOLATOR_BINARY="percolator"'>> prepare_test_data.sh
181 echo 'export SIRIUS_BINARY="$(which sirius)"' >> prepare_test_data.sh
182 echo 'export SPECTRAST_BINARY="'"$OPENMSGIT"'/THIRDPARTY/Linux/64bit/SpectraST/spectrast"' >> prepare_test_data.sh
183 echo 'export XTANDEM_BINARY="xtandem"' >> prepare_test_data.sh
184 echo 'export THERMORAWFILEPARSER_BINARY="ThermoRawFileParser.exe"' >> prepare_test_data.sh
185
186 prepare_test_data >> prepare_test_data.sh #tmp_test_data.sh
187
188 # prepare_test_data > tmp_test_data.sh
189 # # remove calls not needed for the tools listed in any .list file
190 # echo LIST $LIST
191 # if [ ! -z "$LIST" ]; then
192 # REX=$(echo $LIST | sed 's/ /\n/g' | sed 's@.*/\([^/]\+\).xml$@\1@' | tr '\n' '|' | sed 's/|$//')
193 # else
194 # REX=".*"
195 # fi
196 # echo REX $REX
197 # cat tmp_test_data.sh | egrep "($REX)" >> prepare_test_data.sh
198 # rm tmp_test_data.sh
199
200 echo "Execute test shell script"
201 chmod u+x prepare_test_data.sh
202 cd ./test-data || exit
203 ../prepare_test_data.sh
204 cd - || exit
205
206
207 ###############################################################################
208 ## create/update test data for the manually generated tests
209 ## - run convert once with the manual tests only and
210 ## - update test-data (needs to run 2x)
211 ###############################################################################
212 echo "Execute test shell script for manually curated tests"
213 chmod u+x prepare_test_data_manual.sh
214
215 cd ./test-data || exit
216 ../prepare_test_data_manual.sh
217 cd - || exit
218
219
220 ###############################################################################
221 ## auto generate tests
222 ###############################################################################
223 echo "Write test macros to $autotests"
224 echo "<macros>" > "$autotests"
225 for i in $(ls *xml |grep -v macros)
226 do
227 b=$(basename "$i" .xml)
228 get_tests2 "$b" >> "$autotests"
229 done
230 echo "</macros>" >> "$autotests"
231
232 echo "Create test data links"
233 link_tmp_files
234
235 # tests for tools using output_prefix parameters can not be auto generated
236 # hence we output the tests for manual curation in macros_test.xml
237 # and remove them from the autotests
238 # -> OpenSwathFileSplitter IDRipper MzMLSplitter
239 #
240 # Furthermore we remove tests for tools without binaries in conda
241 # -> MSFragger MaRaClusterAdapter NovorAdapter
242 #
243 # not able to specify composite test data
244 # -> SpectraSTSearchAdapter
245 if [[ ! -z "$1" ]]; then
246 echo "" > macros_discarded_auto.xml
247 for i in OpenSwathFileSplitter IDRipper MzMLSplitter MSFraggerAdapter MaRaClusterAdapter NovorAdapter SpectraSTSearchAdapter
248 do
249 echo "<xml name=\"manutest_$i\">" >> macros_discarded_auto.xml
250 xmlstarlet sel -t -c "/macros/xml[@name='autotest_$i']/test" macros_autotest.xml >> macros_discarded_auto.xml
251 echo "</xml>" >> macros_discarded_auto.xml
252 xmlstarlet ed -d "/macros/xml[@name='autotest_$i']/test" macros_autotest.xml > tmp
253 mv tmp macros_autotest.xml
254 done
255 >&2 echo "discarded autogenerated macros for curation in macros_discarded_auto.xml"
256 fi
257 conda deactivate
258
259 ## remove broken symlinks in test-data
260 find test-data/ -xtype l -delete
261
262 # if [ ! -z "$created" ]; then
263 # echo "Removing temporary directory"
264 # rm -rf "$tmp"
265 # fi