Mercurial > repos > galaxyp > openms_targetedfileconverter

diff test-data.sh @ 11:9c70a651a53e draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload for repository https://github.com/galaxyproteomics/tools-galaxyp/tree/master/tools/openms commit 3d1e5f37fd16524a415f707772eeb7ead848c5e3
author galaxyp
date Thu, 01 Dec 2022 19:21:24 +0000
parents 6562463bf496
children 43b9ed22e8ce
line wrap: on
line diff
--- a/test-data.sh	Fri Nov 06 20:02:35 2020 +0000
+++ b/test-data.sh	Thu Dec 01 19:21:24 2022 +0000
@@ -1,22 +1,25 @@
 #!/usr/bin/env bash
 
-VERSION=2.6
-FILETYPES="filetypes.txt"
-CONDAPKG="https://anaconda.org/bioconda/openms/2.6.0/download/linux-64/openms-2.6.0-h4afb90d_0.tar.bz2"
+VERSION=2.8
+FILETYPES="aux/filetypes.txt"
+CONDAPKG="https://anaconda.org/bioconda/openms/2.8.0/download/linux-64/openms-2.8.0-h7ca0330_0.tar.bz2"
 
 # import the magic
 . ./generate-foo.sh
 
 # install conda
 if [ -z "$tmp" ]; then
-	tmp=$(mktemp -d)
-	created="yes"
+    tmp=$(mktemp -d)
+    created="yes"
 fi
 
 export OPENMSGIT="$tmp/OpenMS$VERSION.0-git"
 export OPENMSPKG="$tmp/OpenMS$VERSION-pkg/"
-export OPENMSENV="$tmp/OpenMS$VERSION-env"
-export CTDCONVERTER="$tmp/CTDConverter"
+export OPENMSENV="OpenMS$VERSION-env"
+
+if [ -z "$CTDCONVERTER" ]; then
+    export CTDCONVERTER="$tmp/CTDConverter"
+fi
 
 if [[ -z "$1" ]]; then
 	autotests="/dev/null"
@@ -25,11 +28,11 @@
 fi
 
 if type conda > /dev/null; then  
-	true
+    true
 else
-	wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
-	bash Miniconda3-latest-Linux-x86_64.sh -b -p "$tmp/miniconda"
-	source "$tmp/miniconda/bin/activate"
+    wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
+    bash Miniconda3-latest-Linux-x86_64.sh -b -p "$tmp/miniconda"
+    source "$tmp/miniconda/bin/activate"
 fi
 eval "$(conda shell.bash hook)"
 
@@ -42,24 +45,27 @@
 
 echo "Clone OpenMS $VERSION sources"
 if [[ ! -d $OPENMSGIT ]]; then
-	git clone -b release/$VERSION.0 https://github.com/OpenMS/OpenMS.git $OPENMSGIT
-	cd $OPENMSGIT
-	git submodule init
-	git submodule update
-	cd -
+    # TODO >2.8 reenable original release branch .. also in else branch
+    # the plus branch contains commits from https://github.com/OpenMS/OpenMS/pull/5920 and https://github.com/OpenMS/OpenMS/pull/5917
+    # git clone -b release/$VERSION.0 https://github.com/OpenMS/OpenMS.git $OPENMSGIT
+    git clone -b release/$VERSION.0-plus https://github.com/bernt-matthias/OpenMS.git $OPENMSGIT
+    cd $OPENMSGIT
+    git submodule init
+    git submodule update
+    cd -
 else
-	cd $OPENMSGIT
-	git pull origin release/$VERSION.0
-	cd -
+    cd $OPENMSGIT
+    git pull origin release/$VERSION.0-plus
+    cd -
 fi
 
 echo "Create OpenMS $VERSION conda env"
 # TODO currently add lxml (needed by CTDConverter)
 # TODO for some reason a to recent openjdk is used
 if conda env list | grep "$OPENMSENV"; then
-	true
+    true
 else
-	conda create -y --quiet --override-channels --channel iuc --channel conda-forge --channel bioconda --channel defaults -p $OPENMSENV openms=$VERSION openms-thirdparty=$VERSION ctdopts=1.4 lxml
+    conda create -y --quiet --override-channels --channel iuc --channel conda-forge --channel bioconda --channel defaults -n $OPENMSENV openms=$VERSION openms-thirdparty=$VERSION omssa=2.1.9 ctdopts=1.5 lxml
 # chmod -R u-w $OPENMSENV 
 fi
 ###############################################################################
@@ -69,10 +75,10 @@
 echo "Download OpenMS $VERSION package $CONDAPKG"
 
 if [[ ! -d $OPENMSPKG ]]; then
-	mkdir $OPENMSPKG
-	wget -q -P $OPENMSPKG/ "$CONDAPKG"
-	tar -xf $OPENMSPKG/"$(basename $CONDAPKG)" -C $OPENMSPKG/
-	rm $OPENMSPKG/"$(basename $CONDAPKG)"
+    mkdir $OPENMSPKG
+    wget -q -P $OPENMSPKG/ "$CONDAPKG"
+    tar -xf $OPENMSPKG/"$(basename $CONDAPKG)" -C $OPENMSPKG/
+    rm $OPENMSPKG/"$(basename $CONDAPKG)"
 fi
 
 ###############################################################################
@@ -81,40 +87,46 @@
 ###############################################################################
 echo "Clone CTDConverter"
 if [[ ! -d $CTDCONVERTER ]]; then
-	#git clone https://github.com/WorkflowConversion/CTDConverter.git CTDConverter
-	git clone -b topic/cdata https://github.com/bernt-matthias/CTDConverter.git $CTDCONVERTER
+    #git clone https://github.com/WorkflowConversion/CTDConverter.git CTDConverter
+    git clone -b topic/fix-selects2 https://github.com/bernt-matthias/CTDConverter.git $CTDCONVERTER
 else
-	cd $CTDCONVERTER
-	git pull origin topic/cdata
-	cd -
+    cd $CTDCONVERTER
+    git pull origin topic/fix-selects2
+    cd -
 fi
+conda activate $OPENMSENV
+cd $CTDCONVERTER
+python -m pip install . --no-deps
+cd -
+conda deactivate
 
-###############################################################################
-## copy all the test data files to test-data
-## most of it (outputs) will be overwritten later, but its needed for
-## prepare_test_data
-###############################################################################
+
+# ###############################################################################
+# ## copy all the test data files to test-data
+# ## most of it (outputs) will be overwritten later, but its needed for
+# ## prepare_test_data
+# ###############################################################################
 echo "Get test data"
-find test-data -type f,l,d ! -name "*fa"  ! -name "*loc" -delete
+find test-data -type f,l,d ! -name "*fa" ! -name "*loc" ! -name "test-data" -delete
 
 cp $(find $OPENMSGIT/src/tests/topp/ -type f | grep -Ev "third_party_tests.cmake|CMakeLists.txt|check_ini") test-data/
 cp -r $OPENMSGIT/share/OpenMS/MAPPING/ test-data/
 cp -r $OPENMSGIT/share/OpenMS/CHEMISTRY test-data/
 cp -r $OPENMSGIT/share/OpenMS/examples/ test-data/
-if [[ ! -f test-data/MetaboliteSpectralDB.mzML ]]; then 
-	wget -nc https://abibuilder.informatik.uni-tuebingen.de/archive/openms/Tutorials/Data/latest/Example_Data/Metabolomics/databases/MetaboliteSpectralDB.mzML
-	mv MetaboliteSpectralDB.mzML test-data/
+if [ ! -f test-data/MetaboliteSpectralDB.mzML ]; then 
+    wget -nc https://abibuilder.cs.uni-tuebingen.de/archive/openms/Tutorials/Data/latest/Example_Data/Metabolomics/databases/MetaboliteSpectralDB.mzML
+    mv MetaboliteSpectralDB.mzML test-data/
 fi
 ln -fs TOFCalibration_ref_masses test-data/TOFCalibration_ref_masses.txt
 ln -fs TOFCalibration_const test-data/TOFCalibration_const.csv
 
 if [ ! -d test-data/pepnovo_models/ ]; then
-	mkdir -p /tmp/pepnovo
-	wget -nc http://proteomics.ucsd.edu/Software/PepNovo/PepNovo.20120423.zip
-	unzip PepNovo.20120423.zip -d /tmp/pepnovo/
-	mv /tmp/pepnovo/Models test-data/pepnovo_models/
-	rm PepNovo.20120423.zip
-	rm -rf /tmp/pepnovo
+    mkdir -p /tmp/pepnovo
+    wget -nc http://proteomics.ucsd.edu/Software/PepNovo/PepNovo.20120423.zip
+    unzip PepNovo.20120423.zip -d /tmp/pepnovo/
+    mv /tmp/pepnovo/Models test-data/pepnovo_models/
+    rm PepNovo.20120423.zip
+    rm -rf /tmp/pepnovo
 fi
 ###############################################################################
 ## generate ctd files using the binaries in the conda package 
@@ -124,17 +136,13 @@
 rm -rf ctd
 mkdir -p ctd
 
-# TODO because of https://github.com/OpenMS/OpenMS/issues/4641
-# this needs to be done from within test-data
-cd test-data
 for i in $OPENMSPKG/bin/*
 do
-	b=$(basename $i)
-	echo $b
-	$b -write_ctd ../ctd/
-	sed -i -e 's/²/^2/' ../ctd/$b.ctd
+    b=$(basename $i)
+    echo $b
+    $b -write_ctd ctd/
+    sed -i -e 's/²/^2/' ctd/$b.ctd
 done
-cd -
 ###############################################################################
 ## fix ini files: OpenMS test data contains ini files with outdated ini files.
 ## e.g. variables might be in different nodes, outdated variables present, new
@@ -146,17 +154,17 @@
 echo "Update test INI files"
 for ini in test-data/*ini
 do
-	tool=$(cat $ini | grep 'NODE name="' | head -n 1 | sed 's/.*name="\([^"]\+\)".*/\1/')
-	bin=$(which $tool)
-	if [[ -z $bin ]]; then
+    tool=$(cat $ini | grep 'NODE name="' | head -n 1 | sed 's/.*name="\([^"]\+\)".*/\1/')
+    bin=$(which $tool)
+    if [[ -z $bin ]]; then
           >&2 echo "missing binary to convert $ini"
-		  continue
-	fi
-	cp $ini $ini.backup
-	$bin -ini $ini -write_ini $ini > $ini.stdout 2> $ini.stderr
-	if [[ "$?" -ne "0" ]]; then
-		>&2 echo "could not convert $ini"
-	fi
+          continue
+    fi
+    cp $ini $ini.backup
+    $bin -ini $ini -write_ini $ini > $ini.stdout 2> $ini.stderr
+    if [[ "$?" -ne "0" ]]; then
+        >&2 echo "could not convert $ini"
+    fi
 done
 
 ###############################################################################
@@ -185,17 +193,17 @@
 
 prepare_test_data >> prepare_test_data.sh #tmp_test_data.sh
 
-# prepare_test_data > tmp_test_data.sh
-# # remove calls not needed for the tools listed in any .list file
-# echo LIST $LIST
-# if [ ! -z "$LIST" ]; then
-# 	REX=$(echo $LIST | sed 's/ /\n/g' | sed 's@.*/\([^/]\+\).xml$@\1@' | tr '\n' '|' | sed 's/|$//')
-# else
-# 	REX=".*"
-# fi
-# echo REX $REX
-# cat tmp_test_data.sh | egrep "($REX)" >> prepare_test_data.sh
-# rm tmp_test_data.sh
+## prepare_test_data > tmp_test_data.sh
+## # remove calls not needed for the tools listed in any .list file
+## echo LIST $LIST
+## if [ ! -z "$LIST" ]; then
+##     REX=$(echo $LIST | sed 's/ /\n/g' | sed 's@.*/\([^/]\+\).xml$@\1@' | tr '\n' '|' | sed 's/|$//')
+## else
+##     REX=".*"
+## fi
+## echo REX $REX
+## cat tmp_test_data.sh | egrep "($REX)" >> prepare_test_data.sh
+## rm tmp_test_data.sh
 
 echo "Execute test shell script"
 chmod u+x prepare_test_data.sh
@@ -204,14 +212,13 @@
 cd - || exit
 
 
-###############################################################################
-## create/update test data for the manually generated tests
-## - run convert once with the manual tests only and 
-## - update test-data (needs to run 2x)
-###############################################################################
+# ###############################################################################
+# ## create/update test data for the manually generated tests
+# ## - run convert once with the manual tests only and 
+# ## - update test-data (needs to run 2x)
+# ###############################################################################
 echo "Execute test shell script for manually curated tests"
 chmod u+x prepare_test_data_manual.sh
-
 cd ./test-data || exit
 ../prepare_test_data_manual.sh
 cd - || exit
@@ -220,22 +227,28 @@
 ###############################################################################
 ## auto generate tests
 ###############################################################################
+
 echo "Write test macros to $autotests"
 echo "<macros>" > "$autotests"
-for i in $(ls *xml |grep -v macros)
+
+for i in $(ls ctd/*ctd)
 do
-	b=$(basename "$i" .xml)
-	get_tests2 "$b" >> "$autotests"
+    b=$(basename "$i" .ctd)
+    get_tests2 "$b" >> "$autotests"
 done
 echo "</macros>" >> "$autotests"
 
-echo "Create test data links"
-link_tmp_files
+# echo "Create test data links"
+# Breaks DecoyDatabase
+# link_tmp_files
 
 # tests for tools using output_prefix parameters can not be auto generated
 # hence we output the tests for manual curation in macros_test.xml
 # and remove them from the autotests
-# -> OpenSwathFileSplitter IDRipper MzMLSplitter
+# -> OpenSwathFileSplitter IDRipper MzMLSplitter SeedListGenerator
+# TODO reevaluate in >2.8 
+# - https://github.com/OpenMS/OpenMS/pull/5873
+# - https://github.com/OpenMS/OpenMS/pull/5912
 #
 # Furthermore we remove tests for tools without binaries in conda
 # -> MSFragger MaRaClusterAdapter NovorAdapter 
@@ -243,23 +256,23 @@
 # not able to specify composite test data  
 # -> SpectraSTSearchAdapter 
 if [[ ! -z "$1" ]]; then
-	echo "" > macros_discarded_auto.xml
-	for i in OpenSwathFileSplitter IDRipper MzMLSplitter MSFraggerAdapter MaRaClusterAdapter NovorAdapter SpectraSTSearchAdapter
-	do
-		echo "<xml name=\"manutest_$i\">" >>  macros_discarded_auto.xml
-		xmlstarlet sel -t -c "/macros/xml[@name='autotest_$i']/test" macros_autotest.xml >>  macros_discarded_auto.xml
-		echo "</xml>"  >>  macros_discarded_auto.xml
-		xmlstarlet ed -d "/macros/xml[@name='autotest_$i']/test" macros_autotest.xml > tmp
-		mv tmp macros_autotest.xml
-	done
-	>&2 echo "discarded autogenerated macros for curation in macros_discarded_auto.xml"
+    echo "" > macros_discarded_auto.xml
+    for i in OpenSwathFileSplitter IDRipper MzMLSplitter SeedListGenerator MSFraggerAdapter MaRaClusterAdapter NovorAdapter SpectraSTSearchAdapter
+    do
+        echo "<xml name=\"manutest_$i\">" >>  macros_discarded_auto.xml
+        xmlstarlet sel -t -c "/macros/xml[@name='autotest_$i']/test" macros_autotest.xml >>  macros_discarded_auto.xml
+        echo "</xml>"  >>  macros_discarded_auto.xml
+        xmlstarlet ed -d "/macros/xml[@name='autotest_$i']/test" macros_autotest.xml > tmp
+        mv tmp macros_autotest.xml
+    done
+    >&2 echo "discarded autogenerated macros for curation in macros_discarded_auto.xml"
 fi
 conda deactivate
 
 ## remove broken symlinks in test-data
 find test-data/ -xtype l -delete
 
-# if [ ! -z "$created" ]; then
-# 	echo "Removing temporary directory"
-# 	rm -rf "$tmp"
-# fi
+if [ ! -z "$created" ]; then
+    echo "Removing temporary directory"
+    rm -rf "$tmp"
+fi