Mercurial > repos > galaxyp > openms_simplesearchengine

comparison test-data.sh @ 13:da78f97a6dda draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload for repository https://github.com/galaxyproteomics/tools-galaxyp/tree/master/tools/openms commit 3d1e5f37fd16524a415f707772eeb7ead848c5e3
author galaxyp
date Thu, 01 Dec 2022 19:17:17 +0000
parents 1b56031ac321
children a1724438d580
comparison
equal deleted inserted replaced
12:4ef2f946290d 13:da78f97a6dda
1 #!/usr/bin/env bash 1 #!/usr/bin/env bash
2 2
3 VERSION=2.6 3 VERSION=2.8
4 FILETYPES="filetypes.txt" 4 FILETYPES="aux/filetypes.txt"
5 CONDAPKG="https://anaconda.org/bioconda/openms/2.6.0/download/linux-64/openms-2.6.0-h4afb90d_0.tar.bz2" 5 CONDAPKG="https://anaconda.org/bioconda/openms/2.8.0/download/linux-64/openms-2.8.0-h7ca0330_0.tar.bz2"
6 6
7 # import the magic 7 # import the magic
8 . ./generate-foo.sh 8 . ./generate-foo.sh
9 9
10 # install conda 10 # install conda
11 if [ -z "$tmp" ]; then 11 if [ -z "$tmp" ]; then
12 tmp=$(mktemp -d) 12 tmp=$(mktemp -d)
13 created="yes" 13 created="yes"
14 fi 14 fi
15 15
16 export OPENMSGIT="$tmp/OpenMS$VERSION.0-git" 16 export OPENMSGIT="$tmp/OpenMS$VERSION.0-git"
17 export OPENMSPKG="$tmp/OpenMS$VERSION-pkg/" 17 export OPENMSPKG="$tmp/OpenMS$VERSION-pkg/"
18 export OPENMSENV="$tmp/OpenMS$VERSION-env" 18 export OPENMSENV="OpenMS$VERSION-env"
19 export CTDCONVERTER="$tmp/CTDConverter" 19
20 if [ -z "$CTDCONVERTER" ]; then
21 export CTDCONVERTER="$tmp/CTDConverter"
22 fi
20 23
21 if [[ -z "$1" ]]; then 24 if [[ -z "$1" ]]; then
22 autotests="/dev/null" 25 autotests="/dev/null"
23 else 26 else
24 autotests="$1" 27 autotests="$1"
25 fi 28 fi
26 29
27 if type conda > /dev/null; then 30 if type conda > /dev/null; then
28 true 31 true
29 else 32 else
30 wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh 33 wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
31 bash Miniconda3-latest-Linux-x86_64.sh -b -p "$tmp/miniconda" 34 bash Miniconda3-latest-Linux-x86_64.sh -b -p "$tmp/miniconda"
32 source "$tmp/miniconda/bin/activate" 35 source "$tmp/miniconda/bin/activate"
33 fi 36 fi
34 eval "$(conda shell.bash hook)" 37 eval "$(conda shell.bash hook)"
35 38
36 39
37 ############################################################################### 40 ###############################################################################
40 ## - the git clone of OpenMS (for generating the tests) 43 ## - the git clone of OpenMS (for generating the tests)
41 ############################################################################### 44 ###############################################################################
42 45
43 echo "Clone OpenMS $VERSION sources" 46 echo "Clone OpenMS $VERSION sources"
44 if [[ ! -d $OPENMSGIT ]]; then 47 if [[ ! -d $OPENMSGIT ]]; then
45 git clone -b release/$VERSION.0 https://github.com/OpenMS/OpenMS.git $OPENMSGIT 48 # TODO >2.8 reenable original release branch .. also in else branch
46 cd $OPENMSGIT 49 # the plus branch contains commits from https://github.com/OpenMS/OpenMS/pull/5920 and https://github.com/OpenMS/OpenMS/pull/5917
47 git submodule init 50 # git clone -b release/$VERSION.0 https://github.com/OpenMS/OpenMS.git $OPENMSGIT
48 git submodule update 51 git clone -b release/$VERSION.0-plus https://github.com/bernt-matthias/OpenMS.git $OPENMSGIT
49 cd - 52 cd $OPENMSGIT
50 else 53 git submodule init
51 cd $OPENMSGIT 54 git submodule update
52 git pull origin release/$VERSION.0 55 cd -
53 cd - 56 else
57 cd $OPENMSGIT
58 git pull origin release/$VERSION.0-plus
59 cd -
54 fi 60 fi
55 61
56 echo "Create OpenMS $VERSION conda env" 62 echo "Create OpenMS $VERSION conda env"
57 # TODO currently add lxml (needed by CTDConverter) 63 # TODO currently add lxml (needed by CTDConverter)
58 # TODO for some reason a to recent openjdk is used 64 # TODO for some reason a to recent openjdk is used
59 if conda env list | grep "$OPENMSENV"; then 65 if conda env list | grep "$OPENMSENV"; then
60 true 66 true
61 else 67 else
62 conda create -y --quiet --override-channels --channel iuc --channel conda-forge --channel bioconda --channel defaults -p $OPENMSENV openms=$VERSION openms-thirdparty=$VERSION ctdopts=1.4 lxml 68 conda create -y --quiet --override-channels --channel iuc --channel conda-forge --channel bioconda --channel defaults -n $OPENMSENV openms=$VERSION openms-thirdparty=$VERSION omssa=2.1.9 ctdopts=1.5 lxml
63 # chmod -R u-w $OPENMSENV 69 # chmod -R u-w $OPENMSENV
64 fi 70 fi
65 ############################################################################### 71 ###############################################################################
66 ## get the 72 ## get the
67 ## - conda package (for easy access and listing of the OpenMS binaries), 73 ## - conda package (for easy access and listing of the OpenMS binaries),
68 ############################################################################### 74 ###############################################################################
69 echo "Download OpenMS $VERSION package $CONDAPKG" 75 echo "Download OpenMS $VERSION package $CONDAPKG"
70 76
71 if [[ ! -d $OPENMSPKG ]]; then 77 if [[ ! -d $OPENMSPKG ]]; then
72 mkdir $OPENMSPKG 78 mkdir $OPENMSPKG
73 wget -q -P $OPENMSPKG/ "$CONDAPKG" 79 wget -q -P $OPENMSPKG/ "$CONDAPKG"
74 tar -xf $OPENMSPKG/"$(basename $CONDAPKG)" -C $OPENMSPKG/ 80 tar -xf $OPENMSPKG/"$(basename $CONDAPKG)" -C $OPENMSPKG/
75 rm $OPENMSPKG/"$(basename $CONDAPKG)" 81 rm $OPENMSPKG/"$(basename $CONDAPKG)"
76 fi 82 fi
77 83
78 ############################################################################### 84 ###############################################################################
79 ## Get python libaries for CTD -> Galaxy conversion 85 ## Get python libaries for CTD -> Galaxy conversion
80 ## TODO fix to main repo OR conda packkage if PRs are merged 86 ## TODO fix to main repo OR conda packkage if PRs are merged
81 ############################################################################### 87 ###############################################################################
82 echo "Clone CTDConverter" 88 echo "Clone CTDConverter"
83 if [[ ! -d $CTDCONVERTER ]]; then 89 if [[ ! -d $CTDCONVERTER ]]; then
84 #git clone https://github.com/WorkflowConversion/CTDConverter.git CTDConverter 90 #git clone https://github.com/WorkflowConversion/CTDConverter.git CTDConverter
85 git clone -b topic/cdata https://github.com/bernt-matthias/CTDConverter.git $CTDCONVERTER 91 git clone -b topic/fix-selects2 https://github.com/bernt-matthias/CTDConverter.git $CTDCONVERTER
86 else 92 else
87 cd $CTDCONVERTER 93 cd $CTDCONVERTER
88 git pull origin topic/cdata 94 git pull origin topic/fix-selects2
89 cd - 95 cd -
90 fi 96 fi
91 97 conda activate $OPENMSENV
92 ############################################################################### 98 cd $CTDCONVERTER
93 ## copy all the test data files to test-data 99 python -m pip install . --no-deps
94 ## most of it (outputs) will be overwritten later, but its needed for 100 cd -
95 ## prepare_test_data 101 conda deactivate
96 ############################################################################### 102
103
104 # ###############################################################################
105 # ## copy all the test data files to test-data
106 # ## most of it (outputs) will be overwritten later, but its needed for
107 # ## prepare_test_data
108 # ###############################################################################
97 echo "Get test data" 109 echo "Get test data"
98 find test-data -type f,l,d ! -name "*fa" ! -name "*loc" -delete 110 find test-data -type f,l,d ! -name "*fa" ! -name "*loc" ! -name "test-data" -delete
99 111
100 cp $(find $OPENMSGIT/src/tests/topp/ -type f | grep -Ev "third_party_tests.cmake|CMakeLists.txt|check_ini") test-data/ 112 cp $(find $OPENMSGIT/src/tests/topp/ -type f | grep -Ev "third_party_tests.cmake|CMakeLists.txt|check_ini") test-data/
101 cp -r $OPENMSGIT/share/OpenMS/MAPPING/ test-data/ 113 cp -r $OPENMSGIT/share/OpenMS/MAPPING/ test-data/
102 cp -r $OPENMSGIT/share/OpenMS/CHEMISTRY test-data/ 114 cp -r $OPENMSGIT/share/OpenMS/CHEMISTRY test-data/
103 cp -r $OPENMSGIT/share/OpenMS/examples/ test-data/ 115 cp -r $OPENMSGIT/share/OpenMS/examples/ test-data/
104 if [[ ! -f test-data/MetaboliteSpectralDB.mzML ]]; then 116 if [ ! -f test-data/MetaboliteSpectralDB.mzML ]; then
105 wget -nc https://abibuilder.informatik.uni-tuebingen.de/archive/openms/Tutorials/Data/latest/Example_Data/Metabolomics/databases/MetaboliteSpectralDB.mzML 117 wget -nc https://abibuilder.cs.uni-tuebingen.de/archive/openms/Tutorials/Data/latest/Example_Data/Metabolomics/databases/MetaboliteSpectralDB.mzML
106 mv MetaboliteSpectralDB.mzML test-data/ 118 mv MetaboliteSpectralDB.mzML test-data/
107 fi 119 fi
108 ln -fs TOFCalibration_ref_masses test-data/TOFCalibration_ref_masses.txt 120 ln -fs TOFCalibration_ref_masses test-data/TOFCalibration_ref_masses.txt
109 ln -fs TOFCalibration_const test-data/TOFCalibration_const.csv 121 ln -fs TOFCalibration_const test-data/TOFCalibration_const.csv
110 122
111 if [ ! -d test-data/pepnovo_models/ ]; then 123 if [ ! -d test-data/pepnovo_models/ ]; then
112 mkdir -p /tmp/pepnovo 124 mkdir -p /tmp/pepnovo
113 wget -nc http://proteomics.ucsd.edu/Software/PepNovo/PepNovo.20120423.zip 125 wget -nc http://proteomics.ucsd.edu/Software/PepNovo/PepNovo.20120423.zip
114 unzip PepNovo.20120423.zip -d /tmp/pepnovo/ 126 unzip PepNovo.20120423.zip -d /tmp/pepnovo/
115 mv /tmp/pepnovo/Models test-data/pepnovo_models/ 127 mv /tmp/pepnovo/Models test-data/pepnovo_models/
116 rm PepNovo.20120423.zip 128 rm PepNovo.20120423.zip
117 rm -rf /tmp/pepnovo 129 rm -rf /tmp/pepnovo
118 fi 130 fi
119 ############################################################################### 131 ###############################################################################
120 ## generate ctd files using the binaries in the conda package 132 ## generate ctd files using the binaries in the conda package
121 ############################################################################### 133 ###############################################################################
122 echo "Create CTD files" 134 echo "Create CTD files"
123 conda activate $OPENMSENV 135 conda activate $OPENMSENV
124 rm -rf ctd 136 rm -rf ctd
125 mkdir -p ctd 137 mkdir -p ctd
126 138
127 # TODO because of https://github.com/OpenMS/OpenMS/issues/4641
128 # this needs to be done from within test-data
129 cd test-data
130 for i in $OPENMSPKG/bin/* 139 for i in $OPENMSPKG/bin/*
131 do 140 do
132 b=$(basename $i) 141 b=$(basename $i)
133 echo $b 142 echo $b
134 $b -write_ctd ../ctd/ 143 $b -write_ctd ctd/
135 sed -i -e 's/²/^2/' ../ctd/$b.ctd 144 sed -i -e 's/²/^2/' ctd/$b.ctd
136 done 145 done
137 cd -
138 ############################################################################### 146 ###############################################################################
139 ## fix ini files: OpenMS test data contains ini files with outdated ini files. 147 ## fix ini files: OpenMS test data contains ini files with outdated ini files.
140 ## e.g. variables might be in different nodes, outdated variables present, new 148 ## e.g. variables might be in different nodes, outdated variables present, new
141 ## variables missing, ... 149 ## variables missing, ...
142 ## OpenMS tools fix this on the fly (so its no problem for the OpenMS tests) 150 ## OpenMS tools fix this on the fly (so its no problem for the OpenMS tests)
144 ## see https://github.com/OpenMS/OpenMS/issues/4462 152 ## see https://github.com/OpenMS/OpenMS/issues/4462
145 ############################################################################### 153 ###############################################################################
146 echo "Update test INI files" 154 echo "Update test INI files"
147 for ini in test-data/*ini 155 for ini in test-data/*ini
148 do 156 do
149 tool=$(cat $ini | grep 'NODE name="' | head -n 1 | sed 's/.*name="\([^"]\+\)".*/\1/') 157 tool=$(cat $ini | grep 'NODE name="' | head -n 1 | sed 's/.*name="\([^"]\+\)".*/\1/')
150 bin=$(which $tool) 158 bin=$(which $tool)
151 if [[ -z $bin ]]; then 159 if [[ -z $bin ]]; then
152 >&2 echo "missing binary to convert $ini" 160 >&2 echo "missing binary to convert $ini"
153 continue 161 continue
154 fi 162 fi
155 cp $ini $ini.backup 163 cp $ini $ini.backup
156 $bin -ini $ini -write_ini $ini > $ini.stdout 2> $ini.stderr 164 $bin -ini $ini -write_ini $ini > $ini.stdout 2> $ini.stderr
157 if [[ "$?" -ne "0" ]]; then 165 if [[ "$?" -ne "0" ]]; then
158 >&2 echo "could not convert $ini" 166 >&2 echo "could not convert $ini"
159 fi 167 fi
160 done 168 done
161 169
162 ############################################################################### 170 ###############################################################################
163 ## create script to create results for the tests and run it 171 ## create script to create results for the tests and run it
164 ############################################################################### 172 ###############################################################################
183 echo 'export XTANDEM_BINARY="xtandem"' >> prepare_test_data.sh 191 echo 'export XTANDEM_BINARY="xtandem"' >> prepare_test_data.sh
184 echo 'export THERMORAWFILEPARSER_BINARY="ThermoRawFileParser.exe"' >> prepare_test_data.sh 192 echo 'export THERMORAWFILEPARSER_BINARY="ThermoRawFileParser.exe"' >> prepare_test_data.sh
185 193
186 prepare_test_data >> prepare_test_data.sh #tmp_test_data.sh 194 prepare_test_data >> prepare_test_data.sh #tmp_test_data.sh
187 195
188 # prepare_test_data > tmp_test_data.sh 196 ## prepare_test_data > tmp_test_data.sh
189 # # remove calls not needed for the tools listed in any .list file 197 ## # remove calls not needed for the tools listed in any .list file
190 # echo LIST $LIST 198 ## echo LIST $LIST
191 # if [ ! -z "$LIST" ]; then 199 ## if [ ! -z "$LIST" ]; then
192 # REX=$(echo $LIST | sed 's/ /\n/g' | sed 's@.*/\([^/]\+\).xml$@\1@' | tr '\n' '|' | sed 's/|$//') 200 ## REX=$(echo $LIST | sed 's/ /\n/g' | sed 's@.*/\([^/]\+\).xml$@\1@' | tr '\n' '|' | sed 's/|$//')
193 # else 201 ## else
194 # REX=".*" 202 ## REX=".*"
195 # fi 203 ## fi
196 # echo REX $REX 204 ## echo REX $REX
197 # cat tmp_test_data.sh | egrep "($REX)" >> prepare_test_data.sh 205 ## cat tmp_test_data.sh | egrep "($REX)" >> prepare_test_data.sh
198 # rm tmp_test_data.sh 206 ## rm tmp_test_data.sh
199 207
200 echo "Execute test shell script" 208 echo "Execute test shell script"
201 chmod u+x prepare_test_data.sh 209 chmod u+x prepare_test_data.sh
202 cd ./test-data || exit 210 cd ./test-data || exit
203 ../prepare_test_data.sh 211 ../prepare_test_data.sh
204 cd - || exit 212 cd - || exit
205 213
206 214
207 ############################################################################### 215 # ###############################################################################
208 ## create/update test data for the manually generated tests 216 # ## create/update test data for the manually generated tests
209 ## - run convert once with the manual tests only and 217 # ## - run convert once with the manual tests only and
210 ## - update test-data (needs to run 2x) 218 # ## - update test-data (needs to run 2x)
211 ############################################################################### 219 # ###############################################################################
212 echo "Execute test shell script for manually curated tests" 220 echo "Execute test shell script for manually curated tests"
213 chmod u+x prepare_test_data_manual.sh 221 chmod u+x prepare_test_data_manual.sh
214
215 cd ./test-data || exit 222 cd ./test-data || exit
216 ../prepare_test_data_manual.sh 223 ../prepare_test_data_manual.sh
217 cd - || exit 224 cd - || exit
218 225
219 226
220 ############################################################################### 227 ###############################################################################
221 ## auto generate tests 228 ## auto generate tests
222 ############################################################################### 229 ###############################################################################
230
223 echo "Write test macros to $autotests" 231 echo "Write test macros to $autotests"
224 echo "<macros>" > "$autotests" 232 echo "<macros>" > "$autotests"
225 for i in $(ls *xml |grep -v macros) 233
234 for i in $(ls ctd/*ctd)
226 do 235 do
227 b=$(basename "$i" .xml) 236 b=$(basename "$i" .ctd)
228 get_tests2 "$b" >> "$autotests" 237 get_tests2 "$b" >> "$autotests"
229 done 238 done
230 echo "</macros>" >> "$autotests" 239 echo "</macros>" >> "$autotests"
231 240
232 echo "Create test data links" 241 # echo "Create test data links"
233 link_tmp_files 242 # Breaks DecoyDatabase
243 # link_tmp_files
234 244
235 # tests for tools using output_prefix parameters can not be auto generated 245 # tests for tools using output_prefix parameters can not be auto generated
236 # hence we output the tests for manual curation in macros_test.xml 246 # hence we output the tests for manual curation in macros_test.xml
237 # and remove them from the autotests 247 # and remove them from the autotests
238 # -> OpenSwathFileSplitter IDRipper MzMLSplitter 248 # -> OpenSwathFileSplitter IDRipper MzMLSplitter SeedListGenerator
249 # TODO reevaluate in >2.8
250 # - https://github.com/OpenMS/OpenMS/pull/5873
251 # - https://github.com/OpenMS/OpenMS/pull/5912
239 # 252 #
240 # Furthermore we remove tests for tools without binaries in conda 253 # Furthermore we remove tests for tools without binaries in conda
241 # -> MSFragger MaRaClusterAdapter NovorAdapter 254 # -> MSFragger MaRaClusterAdapter NovorAdapter
242 # 255 #
243 # not able to specify composite test data 256 # not able to specify composite test data
244 # -> SpectraSTSearchAdapter 257 # -> SpectraSTSearchAdapter
245 if [[ ! -z "$1" ]]; then 258 if [[ ! -z "$1" ]]; then
246 echo "" > macros_discarded_auto.xml 259 echo "" > macros_discarded_auto.xml
247 for i in OpenSwathFileSplitter IDRipper MzMLSplitter MSFraggerAdapter MaRaClusterAdapter NovorAdapter SpectraSTSearchAdapter 260 for i in OpenSwathFileSplitter IDRipper MzMLSplitter SeedListGenerator MSFraggerAdapter MaRaClusterAdapter NovorAdapter SpectraSTSearchAdapter
248 do 261 do
249 echo "<xml name=\"manutest_$i\">" >> macros_discarded_auto.xml 262 echo "<xml name=\"manutest_$i\">" >> macros_discarded_auto.xml
250 xmlstarlet sel -t -c "/macros/xml[@name='autotest_$i']/test" macros_autotest.xml >> macros_discarded_auto.xml 263 xmlstarlet sel -t -c "/macros/xml[@name='autotest_$i']/test" macros_autotest.xml >> macros_discarded_auto.xml
251 echo "</xml>" >> macros_discarded_auto.xml 264 echo "</xml>" >> macros_discarded_auto.xml
252 xmlstarlet ed -d "/macros/xml[@name='autotest_$i']/test" macros_autotest.xml > tmp 265 xmlstarlet ed -d "/macros/xml[@name='autotest_$i']/test" macros_autotest.xml > tmp
253 mv tmp macros_autotest.xml 266 mv tmp macros_autotest.xml
254 done 267 done
255 >&2 echo "discarded autogenerated macros for curation in macros_discarded_auto.xml" 268 >&2 echo "discarded autogenerated macros for curation in macros_discarded_auto.xml"
256 fi 269 fi
257 conda deactivate 270 conda deactivate
258 271
259 ## remove broken symlinks in test-data 272 ## remove broken symlinks in test-data
260 find test-data/ -xtype l -delete 273 find test-data/ -xtype l -delete
261 274
262 # if [ ! -z "$created" ]; then 275 if [ ! -z "$created" ]; then
263 # echo "Removing temporary directory" 276 echo "Removing temporary directory"
264 # rm -rf "$tmp" 277 rm -rf "$tmp"
265 # fi 278 fi