Mercurial > repos > galaxyp > openms_consensusid
comparison test-data.sh @ 18:f4be248c3d47 draft
planemo upload for repository https://github.com/galaxyproteomics/tools-galaxyp/tree/master/tools/openms commit 3d1e5f37fd16524a415f707772eeb7ead848c5e3
author | galaxyp |
---|---|
date | Thu, 01 Dec 2022 19:03:29 +0000 |
parents | abffdfee3021 |
children | 91da6dd00bbc |
comparison
equal
deleted
inserted
replaced
17:abffdfee3021 | 18:f4be248c3d47 |
---|---|
1 #!/usr/bin/env bash | 1 #!/usr/bin/env bash |
2 | 2 |
3 VERSION=2.6 | 3 VERSION=2.8 |
4 FILETYPES="filetypes.txt" | 4 FILETYPES="aux/filetypes.txt" |
5 CONDAPKG="https://anaconda.org/bioconda/openms/2.6.0/download/linux-64/openms-2.6.0-h4afb90d_0.tar.bz2" | 5 CONDAPKG="https://anaconda.org/bioconda/openms/2.8.0/download/linux-64/openms-2.8.0-h7ca0330_0.tar.bz2" |
6 | 6 |
7 # import the magic | 7 # import the magic |
8 . ./generate-foo.sh | 8 . ./generate-foo.sh |
9 | 9 |
10 # install conda | 10 # install conda |
11 if [ -z "$tmp" ]; then | 11 if [ -z "$tmp" ]; then |
12 tmp=$(mktemp -d) | 12 tmp=$(mktemp -d) |
13 created="yes" | 13 created="yes" |
14 fi | 14 fi |
15 | 15 |
16 export OPENMSGIT="$tmp/OpenMS$VERSION.0-git" | 16 export OPENMSGIT="$tmp/OpenMS$VERSION.0-git" |
17 export OPENMSPKG="$tmp/OpenMS$VERSION-pkg/" | 17 export OPENMSPKG="$tmp/OpenMS$VERSION-pkg/" |
18 export OPENMSENV="OpenMS$VERSION-env" | 18 export OPENMSENV="OpenMS$VERSION-env" |
19 | 19 |
20 if [ -z "$CTDCONVERTER" ]; then | 20 if [ -z "$CTDCONVERTER" ]; then |
21 export CTDCONVERTER="$tmp/CTDConverter" | 21 export CTDCONVERTER="$tmp/CTDConverter" |
22 fi | 22 fi |
23 | 23 |
24 if [[ -z "$1" ]]; then | 24 if [[ -z "$1" ]]; then |
25 autotests="/dev/null" | 25 autotests="/dev/null" |
26 else | 26 else |
27 autotests="$1" | 27 autotests="$1" |
28 fi | 28 fi |
29 | 29 |
30 if type conda > /dev/null; then | 30 if type conda > /dev/null; then |
31 true | 31 true |
32 else | 32 else |
33 wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh | 33 wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh |
34 bash Miniconda3-latest-Linux-x86_64.sh -b -p "$tmp/miniconda" | 34 bash Miniconda3-latest-Linux-x86_64.sh -b -p "$tmp/miniconda" |
35 source "$tmp/miniconda/bin/activate" | 35 source "$tmp/miniconda/bin/activate" |
36 fi | 36 fi |
37 eval "$(conda shell.bash hook)" | 37 eval "$(conda shell.bash hook)" |
38 | 38 |
39 | 39 |
40 ############################################################################### | 40 ############################################################################### |
43 ## - the git clone of OpenMS (for generating the tests) | 43 ## - the git clone of OpenMS (for generating the tests) |
44 ############################################################################### | 44 ############################################################################### |
45 | 45 |
46 echo "Clone OpenMS $VERSION sources" | 46 echo "Clone OpenMS $VERSION sources" |
47 if [[ ! -d $OPENMSGIT ]]; then | 47 if [[ ! -d $OPENMSGIT ]]; then |
48 git clone -b release/$VERSION.0 https://github.com/OpenMS/OpenMS.git $OPENMSGIT | 48 # TODO >2.8 reenable original release branch .. also in else branch |
49 cd $OPENMSGIT | 49 # the plus branch contains commits from https://github.com/OpenMS/OpenMS/pull/5920 and https://github.com/OpenMS/OpenMS/pull/5917 |
50 git submodule init | 50 # git clone -b release/$VERSION.0 https://github.com/OpenMS/OpenMS.git $OPENMSGIT |
51 git submodule update | 51 git clone -b release/$VERSION.0-plus https://github.com/bernt-matthias/OpenMS.git $OPENMSGIT |
52 cd - | 52 cd $OPENMSGIT |
53 else | 53 git submodule init |
54 cd $OPENMSGIT | 54 git submodule update |
55 git pull origin release/$VERSION.0 | 55 cd - |
56 cd - | 56 else |
57 cd $OPENMSGIT | |
58 git pull origin release/$VERSION.0-plus | |
59 cd - | |
57 fi | 60 fi |
58 | 61 |
59 echo "Create OpenMS $VERSION conda env" | 62 echo "Create OpenMS $VERSION conda env" |
60 # TODO currently add lxml (needed by CTDConverter) | 63 # TODO currently add lxml (needed by CTDConverter) |
61 # TODO for some reason a to recent openjdk is used | 64 # TODO for some reason a to recent openjdk is used |
62 if conda env list | grep "$OPENMSENV"; then | 65 if conda env list | grep "$OPENMSENV"; then |
63 true | 66 true |
64 else | 67 else |
65 conda create -y --quiet --override-channels --channel iuc --channel conda-forge --channel bioconda --channel defaults -n $OPENMSENV openms=$VERSION openms-thirdparty=$VERSION ctdopts=1.4 lxml | 68 conda create -y --quiet --override-channels --channel iuc --channel conda-forge --channel bioconda --channel defaults -n $OPENMSENV openms=$VERSION openms-thirdparty=$VERSION omssa=2.1.9 ctdopts=1.5 lxml |
66 # chmod -R u-w $OPENMSENV | 69 # chmod -R u-w $OPENMSENV |
67 fi | 70 fi |
68 ############################################################################### | 71 ############################################################################### |
69 ## get the | 72 ## get the |
70 ## - conda package (for easy access and listing of the OpenMS binaries), | 73 ## - conda package (for easy access and listing of the OpenMS binaries), |
71 ############################################################################### | 74 ############################################################################### |
72 echo "Download OpenMS $VERSION package $CONDAPKG" | 75 echo "Download OpenMS $VERSION package $CONDAPKG" |
73 | 76 |
74 if [[ ! -d $OPENMSPKG ]]; then | 77 if [[ ! -d $OPENMSPKG ]]; then |
75 mkdir $OPENMSPKG | 78 mkdir $OPENMSPKG |
76 wget -q -P $OPENMSPKG/ "$CONDAPKG" | 79 wget -q -P $OPENMSPKG/ "$CONDAPKG" |
77 tar -xf $OPENMSPKG/"$(basename $CONDAPKG)" -C $OPENMSPKG/ | 80 tar -xf $OPENMSPKG/"$(basename $CONDAPKG)" -C $OPENMSPKG/ |
78 rm $OPENMSPKG/"$(basename $CONDAPKG)" | 81 rm $OPENMSPKG/"$(basename $CONDAPKG)" |
79 fi | 82 fi |
80 | 83 |
81 ############################################################################### | 84 ############################################################################### |
82 ## Get python libaries for CTD -> Galaxy conversion | 85 ## Get python libaries for CTD -> Galaxy conversion |
83 ## TODO fix to main repo OR conda packkage if PRs are merged | 86 ## TODO fix to main repo OR conda packkage if PRs are merged |
84 ############################################################################### | 87 ############################################################################### |
85 echo "Clone CTDConverter" | 88 echo "Clone CTDConverter" |
86 if [[ ! -d $CTDCONVERTER ]]; then | 89 if [[ ! -d $CTDCONVERTER ]]; then |
87 #git clone https://github.com/WorkflowConversion/CTDConverter.git CTDConverter | 90 #git clone https://github.com/WorkflowConversion/CTDConverter.git CTDConverter |
88 git clone -b topic/fix-selects https://github.com/bernt-matthias/CTDConverter.git $CTDCONVERTER | 91 git clone -b topic/fix-selects2 https://github.com/bernt-matthias/CTDConverter.git $CTDCONVERTER |
89 else | 92 else |
90 cd $CTDCONVERTER | 93 cd $CTDCONVERTER |
91 git pull origin topic/fix-selects | 94 git pull origin topic/fix-selects2 |
92 cd - | 95 cd - |
93 fi | 96 fi |
94 | 97 conda activate $OPENMSENV |
95 ############################################################################### | 98 cd $CTDCONVERTER |
96 ## copy all the test data files to test-data | 99 python -m pip install . --no-deps |
97 ## most of it (outputs) will be overwritten later, but its needed for | 100 cd - |
98 ## prepare_test_data | 101 conda deactivate |
99 ############################################################################### | 102 |
103 | |
104 # ############################################################################### | |
105 # ## copy all the test data files to test-data | |
106 # ## most of it (outputs) will be overwritten later, but its needed for | |
107 # ## prepare_test_data | |
108 # ############################################################################### | |
100 echo "Get test data" | 109 echo "Get test data" |
101 find test-data -type f,l,d ! -name "*fa" ! -name "*loc" -delete | 110 find test-data -type f,l,d ! -name "*fa" ! -name "*loc" ! -name "test-data" -delete |
102 | 111 |
103 cp $(find $OPENMSGIT/src/tests/topp/ -type f | grep -Ev "third_party_tests.cmake|CMakeLists.txt|check_ini") test-data/ | 112 cp $(find $OPENMSGIT/src/tests/topp/ -type f | grep -Ev "third_party_tests.cmake|CMakeLists.txt|check_ini") test-data/ |
104 cp -r $OPENMSGIT/share/OpenMS/MAPPING/ test-data/ | 113 cp -r $OPENMSGIT/share/OpenMS/MAPPING/ test-data/ |
105 cp -r $OPENMSGIT/share/OpenMS/CHEMISTRY test-data/ | 114 cp -r $OPENMSGIT/share/OpenMS/CHEMISTRY test-data/ |
106 cp -r $OPENMSGIT/share/OpenMS/examples/ test-data/ | 115 cp -r $OPENMSGIT/share/OpenMS/examples/ test-data/ |
107 if [[ ! -f test-data/MetaboliteSpectralDB.mzML ]]; then | 116 if [ ! -f test-data/MetaboliteSpectralDB.mzML ]; then |
108 wget -nc https://abibuilder.informatik.uni-tuebingen.de/archive/openms/Tutorials/Data/latest/Example_Data/Metabolomics/databases/MetaboliteSpectralDB.mzML | 117 wget -nc https://abibuilder.cs.uni-tuebingen.de/archive/openms/Tutorials/Data/latest/Example_Data/Metabolomics/databases/MetaboliteSpectralDB.mzML |
109 mv MetaboliteSpectralDB.mzML test-data/ | 118 mv MetaboliteSpectralDB.mzML test-data/ |
110 fi | 119 fi |
111 ln -fs TOFCalibration_ref_masses test-data/TOFCalibration_ref_masses.txt | 120 ln -fs TOFCalibration_ref_masses test-data/TOFCalibration_ref_masses.txt |
112 ln -fs TOFCalibration_const test-data/TOFCalibration_const.csv | 121 ln -fs TOFCalibration_const test-data/TOFCalibration_const.csv |
113 | 122 |
114 if [ ! -d test-data/pepnovo_models/ ]; then | 123 if [ ! -d test-data/pepnovo_models/ ]; then |
115 mkdir -p /tmp/pepnovo | 124 mkdir -p /tmp/pepnovo |
116 wget -nc http://proteomics.ucsd.edu/Software/PepNovo/PepNovo.20120423.zip | 125 wget -nc http://proteomics.ucsd.edu/Software/PepNovo/PepNovo.20120423.zip |
117 unzip PepNovo.20120423.zip -d /tmp/pepnovo/ | 126 unzip PepNovo.20120423.zip -d /tmp/pepnovo/ |
118 mv /tmp/pepnovo/Models test-data/pepnovo_models/ | 127 mv /tmp/pepnovo/Models test-data/pepnovo_models/ |
119 rm PepNovo.20120423.zip | 128 rm PepNovo.20120423.zip |
120 rm -rf /tmp/pepnovo | 129 rm -rf /tmp/pepnovo |
121 fi | 130 fi |
122 ############################################################################### | 131 ############################################################################### |
123 ## generate ctd files using the binaries in the conda package | 132 ## generate ctd files using the binaries in the conda package |
124 ############################################################################### | 133 ############################################################################### |
125 echo "Create CTD files" | 134 echo "Create CTD files" |
126 conda activate $OPENMSENV | 135 conda activate $OPENMSENV |
127 rm -rf ctd | 136 rm -rf ctd |
128 mkdir -p ctd | 137 mkdir -p ctd |
129 | 138 |
130 # TODO because of https://github.com/OpenMS/OpenMS/issues/4641 | |
131 # this needs to be done from within test-data | |
132 cd test-data | |
133 for i in $OPENMSPKG/bin/* | 139 for i in $OPENMSPKG/bin/* |
134 do | 140 do |
135 b=$(basename $i) | 141 b=$(basename $i) |
136 echo $b | 142 echo $b |
137 $b -write_ctd ../ctd/ | 143 $b -write_ctd ctd/ |
138 sed -i -e 's/²/^2/' ../ctd/$b.ctd | 144 sed -i -e 's/²/^2/' ctd/$b.ctd |
139 done | 145 done |
140 cd - | |
141 ############################################################################### | 146 ############################################################################### |
142 ## fix ini files: OpenMS test data contains ini files with outdated ini files. | 147 ## fix ini files: OpenMS test data contains ini files with outdated ini files. |
143 ## e.g. variables might be in different nodes, outdated variables present, new | 148 ## e.g. variables might be in different nodes, outdated variables present, new |
144 ## variables missing, ... | 149 ## variables missing, ... |
145 ## OpenMS tools fix this on the fly (so its no problem for the OpenMS tests) | 150 ## OpenMS tools fix this on the fly (so its no problem for the OpenMS tests) |
147 ## see https://github.com/OpenMS/OpenMS/issues/4462 | 152 ## see https://github.com/OpenMS/OpenMS/issues/4462 |
148 ############################################################################### | 153 ############################################################################### |
149 echo "Update test INI files" | 154 echo "Update test INI files" |
150 for ini in test-data/*ini | 155 for ini in test-data/*ini |
151 do | 156 do |
152 tool=$(cat $ini | grep 'NODE name="' | head -n 1 | sed 's/.*name="\([^"]\+\)".*/\1/') | 157 tool=$(cat $ini | grep 'NODE name="' | head -n 1 | sed 's/.*name="\([^"]\+\)".*/\1/') |
153 bin=$(which $tool) | 158 bin=$(which $tool) |
154 if [[ -z $bin ]]; then | 159 if [[ -z $bin ]]; then |
155 >&2 echo "missing binary to convert $ini" | 160 >&2 echo "missing binary to convert $ini" |
156 continue | 161 continue |
157 fi | 162 fi |
158 cp $ini $ini.backup | 163 cp $ini $ini.backup |
159 $bin -ini $ini -write_ini $ini > $ini.stdout 2> $ini.stderr | 164 $bin -ini $ini -write_ini $ini > $ini.stdout 2> $ini.stderr |
160 if [[ "$?" -ne "0" ]]; then | 165 if [[ "$?" -ne "0" ]]; then |
161 >&2 echo "could not convert $ini" | 166 >&2 echo "could not convert $ini" |
162 fi | 167 fi |
163 done | 168 done |
164 | 169 |
165 ############################################################################### | 170 ############################################################################### |
166 ## create script to create results for the tests and run it | 171 ## create script to create results for the tests and run it |
167 ############################################################################### | 172 ############################################################################### |
190 | 195 |
191 ## prepare_test_data > tmp_test_data.sh | 196 ## prepare_test_data > tmp_test_data.sh |
192 ## # remove calls not needed for the tools listed in any .list file | 197 ## # remove calls not needed for the tools listed in any .list file |
193 ## echo LIST $LIST | 198 ## echo LIST $LIST |
194 ## if [ ! -z "$LIST" ]; then | 199 ## if [ ! -z "$LIST" ]; then |
195 ## REX=$(echo $LIST | sed 's/ /\n/g' | sed 's@.*/\([^/]\+\).xml$@\1@' | tr '\n' '|' | sed 's/|$//') | 200 ## REX=$(echo $LIST | sed 's/ /\n/g' | sed 's@.*/\([^/]\+\).xml$@\1@' | tr '\n' '|' | sed 's/|$//') |
196 ## else | 201 ## else |
197 ## REX=".*" | 202 ## REX=".*" |
198 ## fi | 203 ## fi |
199 ## echo REX $REX | 204 ## echo REX $REX |
200 ## cat tmp_test_data.sh | egrep "($REX)" >> prepare_test_data.sh | 205 ## cat tmp_test_data.sh | egrep "($REX)" >> prepare_test_data.sh |
201 ## rm tmp_test_data.sh | 206 ## rm tmp_test_data.sh |
202 | 207 |
205 cd ./test-data || exit | 210 cd ./test-data || exit |
206 ../prepare_test_data.sh | 211 ../prepare_test_data.sh |
207 cd - || exit | 212 cd - || exit |
208 | 213 |
209 | 214 |
210 ############################################################################### | 215 # ############################################################################### |
211 ## create/update test data for the manually generated tests | 216 # ## create/update test data for the manually generated tests |
212 ## - run convert once with the manual tests only and | 217 # ## - run convert once with the manual tests only and |
213 ## - update test-data (needs to run 2x) | 218 # ## - update test-data (needs to run 2x) |
214 ############################################################################### | 219 # ############################################################################### |
215 echo "Execute test shell script for manually curated tests" | 220 echo "Execute test shell script for manually curated tests" |
216 chmod u+x prepare_test_data_manual.sh | 221 chmod u+x prepare_test_data_manual.sh |
217 | |
218 cd ./test-data || exit | 222 cd ./test-data || exit |
219 ../prepare_test_data_manual.sh | 223 ../prepare_test_data_manual.sh |
220 cd - || exit | 224 cd - || exit |
221 | 225 |
222 | 226 |
223 ############################################################################### | 227 ############################################################################### |
224 ## auto generate tests | 228 ## auto generate tests |
225 ############################################################################### | 229 ############################################################################### |
230 | |
226 echo "Write test macros to $autotests" | 231 echo "Write test macros to $autotests" |
227 echo "<macros>" > "$autotests" | 232 echo "<macros>" > "$autotests" |
228 for i in $(ls *xml |grep -v macros) | 233 |
234 for i in $(ls ctd/*ctd) | |
229 do | 235 do |
230 b=$(basename "$i" .xml) | 236 b=$(basename "$i" .ctd) |
231 get_tests2 "$b" >> "$autotests" | 237 get_tests2 "$b" >> "$autotests" |
232 done | 238 done |
233 echo "</macros>" >> "$autotests" | 239 echo "</macros>" >> "$autotests" |
234 | 240 |
235 echo "Create test data links" | 241 # echo "Create test data links" |
236 link_tmp_files | 242 # Breaks DecoyDatabase |
243 # link_tmp_files | |
237 | 244 |
238 # tests for tools using output_prefix parameters can not be auto generated | 245 # tests for tools using output_prefix parameters can not be auto generated |
239 # hence we output the tests for manual curation in macros_test.xml | 246 # hence we output the tests for manual curation in macros_test.xml |
240 # and remove them from the autotests | 247 # and remove them from the autotests |
241 # -> OpenSwathFileSplitter IDRipper MzMLSplitter | 248 # -> OpenSwathFileSplitter IDRipper MzMLSplitter SeedListGenerator |
249 # TODO reevaluate in >2.8 | |
250 # - https://github.com/OpenMS/OpenMS/pull/5873 | |
251 # - https://github.com/OpenMS/OpenMS/pull/5912 | |
242 # | 252 # |
243 # Furthermore we remove tests for tools without binaries in conda | 253 # Furthermore we remove tests for tools without binaries in conda |
244 # -> MSFragger MaRaClusterAdapter NovorAdapter | 254 # -> MSFragger MaRaClusterAdapter NovorAdapter |
245 # | 255 # |
246 # not able to specify composite test data | 256 # not able to specify composite test data |
247 # -> SpectraSTSearchAdapter | 257 # -> SpectraSTSearchAdapter |
248 if [[ ! -z "$1" ]]; then | 258 if [[ ! -z "$1" ]]; then |
249 echo "" > macros_discarded_auto.xml | 259 echo "" > macros_discarded_auto.xml |
250 for i in OpenSwathFileSplitter IDRipper MzMLSplitter MSFraggerAdapter MaRaClusterAdapter NovorAdapter SpectraSTSearchAdapter | 260 for i in OpenSwathFileSplitter IDRipper MzMLSplitter SeedListGenerator MSFraggerAdapter MaRaClusterAdapter NovorAdapter SpectraSTSearchAdapter |
251 do | 261 do |
252 echo "<xml name=\"manutest_$i\">" >> macros_discarded_auto.xml | 262 echo "<xml name=\"manutest_$i\">" >> macros_discarded_auto.xml |
253 xmlstarlet sel -t -c "/macros/xml[@name='autotest_$i']/test" macros_autotest.xml >> macros_discarded_auto.xml | 263 xmlstarlet sel -t -c "/macros/xml[@name='autotest_$i']/test" macros_autotest.xml >> macros_discarded_auto.xml |
254 echo "</xml>" >> macros_discarded_auto.xml | 264 echo "</xml>" >> macros_discarded_auto.xml |
255 xmlstarlet ed -d "/macros/xml[@name='autotest_$i']/test" macros_autotest.xml > tmp | 265 xmlstarlet ed -d "/macros/xml[@name='autotest_$i']/test" macros_autotest.xml > tmp |
256 mv tmp macros_autotest.xml | 266 mv tmp macros_autotest.xml |
257 done | 267 done |
258 >&2 echo "discarded autogenerated macros for curation in macros_discarded_auto.xml" | 268 >&2 echo "discarded autogenerated macros for curation in macros_discarded_auto.xml" |
259 fi | 269 fi |
260 conda deactivate | 270 conda deactivate |
261 | 271 |
262 ## remove broken symlinks in test-data | 272 ## remove broken symlinks in test-data |
263 find test-data/ -xtype l -delete | 273 find test-data/ -xtype l -delete |
264 | 274 |
265 # if [ ! -z "$created" ]; then | 275 if [ ! -z "$created" ]; then |
266 # echo "Removing temporary directory" | 276 echo "Removing temporary directory" |
267 # rm -rf "$tmp" | 277 rm -rf "$tmp" |
268 # fi | 278 fi |