Mercurial > repos > fubar > tool_factory_2

comparison toolfactory/rgToolFactory2.py @ 103:2ca51677d6d1 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Uploaded
author fubar
date Fri, 27 Nov 2020 22:38:43 +0000
parents c632db66f8c0
children 05cb28bf2bbc
comparison
equal deleted inserted replaced
102:c632db66f8c0 103:2ca51677d6d1
26 # planemo test --engine docker_galaxy --test_data ./test-data/ --docker_extra_volume ./test-data rgToolFactory2.xml 26 # planemo test --engine docker_galaxy --test_data ./test-data/ --docker_extra_volume ./test-data rgToolFactory2.xml
27 27
28 import argparse 28 import argparse
29 import copy 29 import copy
30 import datetime 30 import datetime
31 import grp
31 import json 32 import json
32 import logging 33 import logging
33 import os 34 import os
34 import re 35 import re
35 import shutil 36 import shutil
708 tar.close() 709 tar.close()
709 os.unlink(tfname) 710 os.unlink(tfname)
710 711
711 712
712 713
714
713 def planemo_biodocker_test(self): 715 def planemo_biodocker_test(self):
714 """planemo currently leaks dependencies if used in the same container and gets unhappy after a 716 """planemo currently leaks dependencies if used in the same container and gets unhappy after a
715 first successful run. https://github.com/galaxyproject/planemo/issues/1078#issuecomment-731476930 717 first successful run. https://github.com/galaxyproject/planemo/issues/1078#issuecomment-731476930
716 718
717 Docker biocontainer has planemo with caches filled to save repeated downloads 719 Docker biocontainer has planemo with caches filled to save repeated downloads
718 720
719 721
720 """ 722 """
723 def prun(container,tout,cl,user="biodocker"):
724 rlog = container.exec_run(cl,user=user)
725 slogl = str(rlog).split('\\n')
726 slog = '\n'.join(slogl)
727 tout.write(f"## got rlog {slog} from {cl}\n")
728
729 dgroup = grp.getgrnam('docker')[2]
721 if os.path.exists(self.tlog): 730 if os.path.exists(self.tlog):
722 tout = open(self.tlog, "a") 731 tout = open(self.tlog, "a")
723 else: 732 else:
724 tout = open(self.tlog, "w") 733 tout = open(self.tlog, "w")
725 planemoimage = "quay.io/fubar2/planemo-biocontainer" 734 planemoimage = "quay.io/fubar2/planemo-biocontainer"
726 xreal = "%s.xml" % self.tool_name 735 xreal = "%s.xml" % self.tool_name
727 destdir = "/tmp/tfout"
728 repname = f"{self.tool_name}_planemo_test_report.html" 736 repname = f"{self.tool_name}_planemo_test_report.html"
729 imrep = os.path.join(destdir,repname)
730 ptestrep_path = os.path.join(self.repdir,repname) 737 ptestrep_path = os.path.join(self.repdir,repname)
731 tool_name = self.tool_name 738 tool_name = self.tool_name
732 client = docker.from_env() 739 client = docker.from_env()
733 container = client.containers.run(planemoimage,'sleep 10000m', detach=True) 740 tvol = client.volumes.create()
734 rlog = container.exec_run(f"mkdir -p {destdir}") 741 tvolname = tvol.name
735 slogl = str(rlog).split('\\n') 742 destdir = "/toolfactory/ptest"
736 slog = '\n'.join(slogl) 743 imrep = os.path.join(destdir,repname)
737 tout.write(f"## got rlog {slog} from mkdir {destdir}") 744 container = client.containers.run(planemoimage,'sleep 10000m', detach=True, user="biodocker",
738 ptestpath = os.path.join(destdir,xreal) 745 network="host", volumes={f"{tvolname}": {'bind': '/toolfactory', 'mode': 'rw'}})
739 self.copy_to_container(self.tooloutdir,'/tmp',container) 746 cl = f"groupmod -g {dgroup} docker"
747 prun(container, tout, cl, user="root")
748 cl = f"mkdir -p {destdir}"
749 prun(container, tout, cl, user="root")
750 cl = f"rm -rf {destdir}/*"
751 prun(container, tout, cl, user="root")
752 ptestpath = os.path.join(destdir,'tfout',xreal)
753 self.copy_to_container(self.tooloutdir,destdir,container)
754 cl ='chmod -R a+rwx /toolfactory'
755 prun(container, tout, cl, user="root")
740 rlog = container.exec_run(f"ls -la {destdir}") 756 rlog = container.exec_run(f"ls -la {destdir}")
741 ptestcl = f"planemo test --update_test_data --galaxy_root /home/biodocker/galaxy-central {ptestpath}" 757 ptestcl = f"planemo test --update_test_data --no_cleanup --test_data {destdir}/tfout/test-data --galaxy_root /home/biodocker/galaxy-central {ptestpath}"
742 try: 758 try:
743 rlog = container.exec_run(ptestcl) 759 rlog = container.exec_run(ptestcl)
744 except: 760 except:
745 e = sys.exc_info()[0] 761 e = sys.exc_info()[0]
746 tout.write(f"#### error: {e} from {ptestcl}") 762 tout.write(f"#### error: {e} from {ptestcl}\n")
747 # fails - used to generate test outputs 763 # fails - used to generate test outputs
748 ptestcl = f"planemo test --test_output {imrep} --galaxy_root /home/biodocker/galaxy-central {ptestpath}" 764 cl = f"planemo test --test_output {imrep} --no_cleanup --test_data {destdir}/tfout/test-data --galaxy_root /home/biodocker/galaxy-central {ptestpath}"
749 try: 765 try:
750 rlog = container.exec_run(ptestcl) 766 prun(container,tout,cl)
751 except: 767 except:
752 pass 768 pass
753 slogl = str(rlog).split('\\n')
754 slog = '\n'.join(slogl)
755 tout.write(f"## got rlog {slog} from mkdir {destdir}")
756 testouts = tempfile.mkdtemp(suffix=None, prefix="tftemp",dir=".") 769 testouts = tempfile.mkdtemp(suffix=None, prefix="tftemp",dir=".")
757 self.copy_from_container(destdir,testouts,container) 770 self.copy_from_container(destdir,testouts,container)
758 try: 771 src = os.path.join(testouts,'ptest')
759 shutil.rmtree(os.path.join(testouts,self.tooloutdir,'test-data','test-data')) 772 if os.path.isdir(src):
760 except: 773 shutil.copytree(src, '.', dirs_exist_ok=True)
761 e = sys.exc_info()[0] 774 src = repname
762 tout.write(f"#### error: {e} from {ptestcl}") 775 if os.path.isfile(repname):
763 shutil.copytree(os.path.join(testouts,self.tooloutdir), self.tooloutdir, dirs_exist_ok=True) 776 shutil.copyfile(src,ptestrep_path)
777 else:
778 tout.write(f"No output from run to shutil.copytree in {src}\n")
764 tout.close() 779 tout.close()
765 container.stop() 780 container.stop()
766 container.remove() 781 container.remove()
767 shutil.rmtree(testouts) 782 tvol.remove()
783 #shutil.rmtree(testouts)
768 784
769 def shedLoad(self): 785 def shedLoad(self):
770 """ 786 """
771 {'deleted': False, 787 {'deleted': False,
772 'description': 'Tools for manipulating data', 788 'description': 'Tools for manipulating data',