Mercurial > repos > galaxyp > openms_qcexporter

comparison test-data.sh @ 9:7e8a254ef96d draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload for repository https://github.com/galaxyproteomics/tools-galaxyp/tree/master/tools/openms commit ddf41e8bda1ba065f5cdec98e93dee8165ffc1b9"
author galaxyp
date Thu, 03 Sep 2020 16:24:22 +0000
parents
children bd8735ccc6f6
comparison
equal deleted inserted replaced
8:d02e46f6c54a 9:7e8a254ef96d
1 #!/usr/bin/env bash
2
3 VERSION=2.5
4 FILETYPES="filetypes.txt"
5 CONDAPKG="https://anaconda.org/bioconda/openms/2.5.0/download/linux-64/openms-2.5.0-h463af6b_1.tar.bz2"
6
7 # import the magic
8 . ./generate-foo.sh
9
10 # install conda
11 if [ -z "$tmp" ]; then
12 tmp=$(mktemp -d)
13 created="yes"
14 fi
15
16 export OPENMSGIT="$tmp/OpenMS$VERSION.0-git"
17 export OPENMSPKG="$tmp/OpenMS$VERSION-pkg/"
18 export OPENMSENV="$tmp/OpenMS$VERSION-env"
19 export CTDCONVERTER="$tmp/CTDConverter"
20
21
22 if type conda > /dev/null; then
23 true
24 else
25 wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
26 bash Miniconda3-latest-Linux-x86_64.sh -b -p "$tmp/miniconda"
27 source "$tmp/miniconda/bin/activate"
28 fi
29 eval "$(conda shell.bash hook)"
30
31 if [[ -z "$1" ]]; then
32 autotests="/dev/null"
33 else
34 autotests="$1"
35 fi
36
37 ###############################################################################
38 ## get
39 ## - conda environment (for executing the binaries) and
40 ## - the git clone of OpenMS (for generating the tests)
41 ###############################################################################
42
43 echo "Clone OpenMS $VERSION sources"
44 if [[ ! -d $OPENMSGIT ]]; then
45 git clone -b release/$VERSION.0 https://github.com/OpenMS/OpenMS.git $OPENMSGIT
46 cd $OPENMSGIT
47 git submodule init
48 git submodule update
49 cd -
50 else
51 cd $OPENMSGIT
52 git pull origin release/$VERSION.0
53 cd -
54 fi
55
56 echo "Create OpenMS $VERSION conda env"
57 # TODO currently add lxml (needed by CTDConverter)
58 # TODO for some reason a to recent openjdk is used
59 if conda env list | grep "$OPENMSENV"; then
60 true
61 else
62 conda create -y --quiet --override-channels --channel iuc --channel conda-forge --channel bioconda --channel defaults -p $OPENMSENV openms=$VERSION openms-thirdparty=$VERSION openjdk=8.0.192 ctdopts=1.4 lxml
63 # chmod -R u-w $OPENMSENV
64 fi
65 ###############################################################################
66 ## get the
67 ## - conda package (for easy access and listing of the OpenMS binaries),
68 ###############################################################################
69 echo "Download OpenMS $VERSION package $CONDAPKG"
70
71 if [[ ! -d $OPENMSPKG ]]; then
72 mkdir $OPENMSPKG
73 wget -q -P $OPENMSPKG/ "$CONDAPKG"
74 tar -xf $OPENMSPKG/"$(basename $CONDAPKG)" -C $OPENMSPKG/
75 rm $OPENMSPKG/"$(basename $CONDAPKG)"
76 fi
77
78 ###############################################################################
79 ## Get python libaries for CTD -> Galaxy conversion
80 ## TODO fix to main repo OR conda packkage if PRs are merged
81 ###############################################################################
82 echo "Clone CTDConverter"
83 if [[ ! -d $CTDCONVERTER ]]; then
84 #git clone https://github.com/WorkflowConversion/CTDConverter.git CTDConverter
85 git clone -b topic/cdata https://github.com/bernt-matthias/CTDConverter.git $CTDCONVERTER
86 else
87 cd $CTDCONVERTER
88 git pull origin topic/cdata
89 cd -
90 fi
91 export PYTHONPATH=$(pwd)/CTDopts
92
93 ###############################################################################
94 ## copy all the test data files to test-data
95 ## most of it (outputs) will be overwritten later, but its needed for
96 ## prepare_test_data
97 ###############################################################################
98 echo "Get test data"
99 cp $(find $OPENMSGIT/src/tests/topp/ -type f | grep -Ev "third_party_tests.cmake|CMakeLists.txt|check_ini") test-data/
100 cp -r $OPENMSGIT/share/OpenMS/MAPPING/ test-data/
101 cp -r $OPENMSGIT/share/OpenMS/CHEMISTRY test-data/
102 cp -r $OPENMSGIT/share/OpenMS/examples/ test-data/
103 if [[ ! -f test-data/MetaboliteSpectralDB.mzML ]]; then
104 wget -q https://abibuilder.informatik.uni-tuebingen.de/archive/openms/Tutorials/Data/latest/Example_Data/Metabolomics/databases/MetaboliteSpectralDB.mzML && mv MetaboliteSpectralDB.mzML test-data/
105 fi
106 ln -fs TOFCalibration_ref_masses test-data/TOFCalibration_ref_masses.txt
107 ln -fs TOFCalibration_const test-data/TOFCalibration_const.csv
108
109 if [ ! -d test-data/pepnovo_models/ ]; then
110 wget http://proteomics.ucsd.edu/Software/PepNovo/PepNovo.20120423.zip
111 unzip -e PepNovo.20120423.zip -d /tmp/
112 mv /tmp/Models test-data/pepnovo_models/
113 fi
114 ###############################################################################
115 ## generate ctd files using the binaries in the conda package
116 ###############################################################################
117 echo "Create CTD files"
118 conda activate $OPENMSENV
119 mkdir -p ctd
120
121 # TODO because of https://github.com/OpenMS/OpenMS/issues/4641
122 # this needs to be done from within test-data
123 cd test-data
124 for i in $OPENMSPKG/bin/*
125 do
126 b=$(basename $i)
127 echo $b
128 $b -write_ctd ../ctd/
129 sed -i -e 's/²/^2/' ../ctd/$b.ctd
130 done
131 cd -
132 ###############################################################################
133 ## fix ini files: OpenMS test data contains ini files with outdated ini files.
134 ## e.g. variables might be in different nodes, outdated variables present, new
135 ## variables missing, ...
136 ## OpenMS tools fix this on the fly (so its no problem for the OpenMS tests)
137 ## but it is for the generation of the tests
138 ## see https://github.com/OpenMS/OpenMS/issues/4462
139 ###############################################################################
140 echo "Update test INI files"
141 for ini in test-data/*ini
142 do
143 tool=$(cat $ini | grep 'NODE name="' | head -n 1 | sed 's/.*name="\([^"]\+\)".*/\1/')
144 bin=$(which $tool)
145 if [[ -z $bin ]]; then
146 >&2 echo "missing binary to convert $ini"
147 continue
148 fi
149 cp $ini $ini.backup
150 $bin -ini $ini -write_ini $ini > $ini.stdout 2> $ini.stderr
151 if [[ "$?" -ne "0" ]]; then
152 >&2 echo "could not convert $ini"
153 fi
154 done
155
156 ###############################################################################
157 ## create script to create results for the tests and run it
158 ###############################################################################
159 echo "Create test shell script"
160
161 echo -n "" > prepare_test_data.sh
162 echo 'export COMET_BINARY="comet"' >> prepare_test_data.sh
163 echo 'export CRUX_BINARY="crux"' >> prepare_test_data.sh
164 echo 'export FIDOCHOOSEPARAMS_BINARY="FidoChooseParameters"' >> prepare_test_data.sh
165 echo 'export FIDO_BINARY="Fido"' >> prepare_test_data.sh
166 echo 'export LUCIPHOR_BINARY="$(dirname $(realpath $(which luciphor2)))/luciphor2.jar"' >> prepare_test_data.sh
167
168 echo 'export MARACLUSTER_BINARY="'"$OPENMSGIT"'/THIRDPARTY/Linux/64bit/MaRaCluster/maracluster"'>> prepare_test_data.sh
169 echo 'export MSFRAGGER_BINARY="/home/berntm/Downloads/MSFragger-20171106/MSFragger-20171106.jar"'>> prepare_test_data.sh
170 echo 'export MSGFPLUS_BINARY="$(msgf_plus -get_jar_path)"' >> prepare_test_data.sh
171 echo 'export MYRIMATCH_BINARY="myrimatch"'>> prepare_test_data.sh
172 echo 'export NOVOR_BINARY="/home/berntm/Downloads/novor/lib/novor.jar"' >> prepare_test_data.sh
173 echo 'export OMSSA_BINARY="$(dirname $(realpath $(which omssacl)))/omssacl"'>> prepare_test_data.sh
174 echo 'export PERCOLATOR_BINARY="percolator"'>> prepare_test_data.sh
175 echo 'export SIRIUS_BINARY="$(which sirius)"' >> prepare_test_data.sh
176 echo 'export SPECTRAST_BINARY="spectrast"' >> prepare_test_data.sh
177 echo 'export XTANDEM_BINARY="xtandem"' >> prepare_test_data.sh
178 echo 'export THERMORAWFILEPARSER_BINARY="ThermoRawFileParser.exe"' >> prepare_test_data.sh
179
180 prepare_test_data >> prepare_test_data.sh #tmp_test_data.sh
181
182 # prepare_test_data > tmp_test_data.sh
183 # # remove calls not needed for the tools listed in any .list file
184 # echo LIST $LIST
185 # if [ ! -z "$LIST" ]; then
186 # REX=$(echo $LIST | sed 's/ /\n/g' | sed 's@.*/\([^/]\+\).xml$@\1@' | tr '\n' '|' | sed 's/|$//')
187 # else
188 # REX=".*"
189 # fi
190 # echo REX $REX
191 # cat tmp_test_data.sh | egrep "($REX)" >> prepare_test_data.sh
192 # rm tmp_test_data.sh
193
194 echo "Execute test shell script"
195 chmod u+x prepare_test_data.sh
196 cd ./test-data || exit
197 ../prepare_test_data.sh
198 cd - || exit
199
200
201 ###############################################################################
202 ## create/update test data for the manually generated tests
203 ## - run convert once with the manual tests only and
204 ## - update test-data (needs to run 2x)
205 ###############################################################################
206 echo "Execute test shell script for manually curated tests"
207 chmod u+x prepare_test_data_manual.sh
208
209 cd ./test-data || exit
210 ../prepare_test_data_manual.sh
211 cd - || exit
212
213
214 # # # exit
215
216 ###############################################################################
217 ## auto generate tests
218 ###############################################################################
219
220 echo "Write test macros to "$autotests
221 echo "<macros>" > $autotests
222 for i in $(ls *xml |grep -v macros)
223 do
224 b=$(basename "$i" .xml)
225 get_tests2 "$b" >> $autotests
226 done
227 echo "</macros>" >> $autotests
228
229 echo "Create test data links"
230 link_tmp_files
231
232 # tests for tools using output_prefix parameters can not be auto generated
233 # hence we output the tests for manual curation in macros_test.xml
234 # and remove them from the autotests
235 if [[ ! -z "$1" ]]; then
236 echo "" > macros_discarded_auto.xml
237 for i in OpenSwathFileSplitter IDRipper MzMLSplitter
238 do
239 echo "<xml name=\"manutest_$i\">" >> macros_discarded_auto.xml
240 xmlstarlet sel -t -c "/macros/xml[@name='autotest_$i']/test" macros_autotest.xml >> macros_discarded_auto.xml
241 echo "</xml>" >> macros_discarded_auto.xml
242 xmlstarlet ed -d "/macros/xml[@name='autotest_$i']/test" macros_autotest.xml > tmp
243 mv tmp macros_autotest.xml
244 done
245 >&2 "discarded autogenerated macros for curation in macros_discarded_auto.xml"
246 fi
247 conda deactivate
248
249 ## remove broken symlinks in test-data
250 find test-data/ -xtype l -delete
251
252 # if [ ! -z "$created" ]; then
253 # echo "Removing temporary directory"
254 # rm -rf "$tmp"
255 # fi