Mercurial > repos > fubar > tool_factory_2

comparison toolfactory/rgToolFactory2.py @ 101:557d5f06f213 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Uploaded
author fubar
date Thu, 26 Nov 2020 06:17:12 +0000
parents c749364c2283
children c632db66f8c0
comparison
equal deleted inserted replaced
100:c749364c2283 101:557d5f06f213
24 # does the needful. Use GALAXY_TEST_SAVE /foo to save outputs - only the tar.gz - not the rest sadly 24 # does the needful. Use GALAXY_TEST_SAVE /foo to save outputs - only the tar.gz - not the rest sadly
25 # GALAXY_TEST_NO_CLEANUP GALAXY_TEST_TMP_DIR=wherever 25 # GALAXY_TEST_NO_CLEANUP GALAXY_TEST_TMP_DIR=wherever
26 # planemo test --engine docker_galaxy --test_data ./test-data/ --docker_extra_volume ./test-data rgToolFactory2.xml 26 # planemo test --engine docker_galaxy --test_data ./test-data/ --docker_extra_volume ./test-data rgToolFactory2.xml
27 27
28 import argparse 28 import argparse
29 import copy
29 import datetime 30 import datetime
30 import json 31 import json
31 import logging 32 import logging
32 import os 33 import os
33 import re 34 import re
40 41
41 42
42 from bioblend import ConnectionError 43 from bioblend import ConnectionError
43 from bioblend import toolshed 44 from bioblend import toolshed
44 45
45 # import docker 46 import docker
46 47
47 import galaxyxml.tool as gxt 48 import galaxyxml.tool as gxt
48 import galaxyxml.tool.parameters as gxtp 49 import galaxyxml.tool.parameters as gxtp
49 50
50 import lxml 51 import lxml
151 def __init__(self, args=None): 152 def __init__(self, args=None):
152 """ 153 """
153 prepare command line cl for running the tool here 154 prepare command line cl for running the tool here
154 and prepare elements needed for galaxyxml tool generation 155 and prepare elements needed for galaxyxml tool generation
155 """ 156 """
157 self.ourcwd = os.getcwd()
158 self.ourenv = copy.deepcopy(os.environ)
156 self.infiles = [x.split(ourdelim) for x in args.input_files] 159 self.infiles = [x.split(ourdelim) for x in args.input_files]
157 self.outfiles = [x.split(ourdelim) for x in args.output_files] 160 self.outfiles = [x.split(ourdelim) for x in args.output_files]
158 self.addpar = [x.split(ourdelim) for x in args.additional_parameters] 161 self.addpar = [x.split(ourdelim) for x in args.additional_parameters]
159 self.args = args 162 self.args = args
160 self.cleanuppar() 163 self.cleanuppar()
178 "positional", 181 "positional",
179 ], 'args.parampass must be "0","positional" or "argparse"' 182 ], 'args.parampass must be "0","positional" or "argparse"'
180 self.tool_name = re.sub("[^a-zA-Z0-9_]+", "", args.tool_name) 183 self.tool_name = re.sub("[^a-zA-Z0-9_]+", "", args.tool_name)
181 self.tool_id = self.tool_name 184 self.tool_id = self.tool_name
182 self.newtool = gxt.Tool( 185 self.newtool = gxt.Tool(
183 self.args.tool_name, 186 self.tool_name,
184 self.tool_id, 187 self.tool_id,
185 self.args.tool_version, 188 self.args.tool_version,
186 self.args.tool_desc, 189 self.args.tool_desc,
187 FAKEEXE, 190 FAKEEXE,
188 ) 191 )
643 sto = open(self.tlog, "w") 646 sto = open(self.tlog, "w")
644 sto.write( 647 sto.write(
645 "## Executing Toolfactory generated command line = %s\n" % scl 648 "## Executing Toolfactory generated command line = %s\n" % scl
646 ) 649 )
647 sto.flush() 650 sto.flush()
648 subp = subprocess.run(self.cl, shell=False, stdout=sto, stderr=ste) 651 subp = subprocess.run(self.cl, env=self.ourenv, shell=False, stdout=sto, stderr=ste)
649 sto.close() 652 sto.close()
650 ste.close() 653 ste.close()
651 retval = subp.returncode 654 retval = subp.returncode
652 else: # work around special case - stdin and write to stdout 655 else: # work around special case - stdin and write to stdout
653 if len(self.infiles) > 0: 656 if len(self.infiles) > 0:
656 sti = sys.stdin 659 sti = sys.stdin
657 if len(self.outfiles) > 0: 660 if len(self.outfiles) > 0:
658 sto = open(self.outfiles[0][ONAMEPOS], "wb") 661 sto = open(self.outfiles[0][ONAMEPOS], "wb")
659 else: 662 else:
660 sto = sys.stdout 663 sto = sys.stdout
661 subp = subprocess.run(self.cl, shell=False, stdout=sto, stdin=sti) 664 subp = subprocess.run(self.cl, env=self.ourenv, shell=False, stdout=sto, stdin=sti)
662 sto.write("## Executing Toolfactory generated command line = %s\n" % scl) 665 sto.write("## Executing Toolfactory generated command line = %s\n" % scl)
663 retval = subp.returncode 666 retval = subp.returncode
664 sto.close() 667 sto.close()
665 sti.close() 668 sti.close()
666 if os.path.isfile(self.tlog) and os.stat(self.tlog).st_size == 0: 669 if os.path.isfile(self.tlog) and os.stat(self.tlog).st_size == 0:
671 sys.stderr.write(err) 674 sys.stderr.write(err)
672 logging.debug("run done") 675 logging.debug("run done")
673 return retval 676 return retval
674 677
675 678
676 def gal_tool_test(self): 679 def copy_to_container(self, src, dest, container):
677 """ 680 """ Recreate the src directory tree at dest - full path included
678 This handy script writes test outputs even if they don't exist 681 """
679 galaxy-tool-test [-h] [-u GALAXY_URL] [-k KEY] [-a ADMIN_KEY] [--force_path_paste] [-t TOOL_ID] [--tool-version TOOL_VERSION] 682 idir = os.getcwd()
680 [-i TEST_INDEX] [-o OUTPUT] [--append] [-j OUTPUT_JSON] [--verbose] [-c CLIENT_TEST_CONFIG] 683 workdir = os.path.dirname(src)
681 galaxy-tool-test -u http://localhost:8080 -a 3c9afe09f1b7892449d266109639c104 -o /tmp/foo -t hello -j /tmp/foo/hello.json --verbose 684 os.chdir(workdir)
682 handy - just leaves outputs in -o 685 _, tfname = tempfile.mkstemp(suffix=".tar")
686 tar = tarfile.open(tfname, mode='w')
687 srcb = os.path.basename(src)
688 tar.add(srcb)
689 tar.close()
690 data = open(tfname, 'rb').read()
691 container.put_archive(dest, data)
692 os.unlink(tfname)
693 os.chdir(idir)
694
695
696 def copy_from_container(self, src, dest, container):
697 """ recreate the src directory tree at dest using docker sdk
698 """
699 os.makedirs(dest,exist_ok=True)
700 _, tfname = tempfile.mkstemp(suffix=".tar")
701 tf = open(tfname,'wb')
702 bits, stat = container.get_archive(src)
703 for chunk in bits:
704 tf.write(chunk)
705 tf.close()
706 tar = tarfile.open(tfname,'r')
707 tar.extractall(dest)
708 tar.close()
709 os.unlink(tfname)
710
711
712
713 def planemo_biodocker_test(self):
714 """planemo currently leaks dependencies if used in the same container and gets unhappy after a
715 first successful run. https://github.com/galaxyproject/planemo/issues/1078#issuecomment-731476930
716
717 Docker biocontainer has planemo with caches filled to save repeated downloads
718
719
683 """ 720 """
684 if os.path.exists(self.tlog): 721 if os.path.exists(self.tlog):
685 tout = open(self.tlog, "a") 722 tout = open(self.tlog, "a")
686 else: 723 else:
687 tout = open(self.tlog, "w") 724 tout = open(self.tlog, "w")
688 testouts = tempfile.mkdtemp(suffix=None, prefix="tftemp") 725 planemoimage = "quay.io/fubar2/planemo-biocontainer"
726 xreal = "%s.xml" % self.tool_name
727 destdir = "/tmp/tfout"
728 repname = f"{self.tool_name}_planemo_test_report.html"
729 imrep = os.path.join(destdir,repname)
730 ptestrep_path = os.path.join(self.repdir,repname)
731 tool_name = self.tool_name
732 client = docker.from_env()
733 container = client.containers.run(planemoimage,'sleep 10000m', detach=True)
734 rlog = container.exec_run(f"mkdir -p {destdir}")
735 slogl = str(rlog).split('\\n')
736 slog = '\n'.join(slogl)
737 tout.write(f"## got rlog {slog} from mkdir {destdir}")
738 ptestpath = os.path.join(destdir,xreal)
739 self.copy_to_container(self.tooloutdir,'/tmp',container)
740 rlog = container.exec_run(f"ls -la {destdir}")
741 ptestcl = f"planemo test --update_test_data --galaxy_root /home/biodocker/galaxy-central {ptestpath}"
742 try:
743 rlog = container.exec_run(ptestcl)
744 except:
745 e = sys.exc_info()[0]
746 tout.write(f"#### error: {e} from {ptestcl}")
747 # fails - used to generate test outputs
748 ptestcl = f"planemo test --test_output {imrep} --galaxy_root /home/biodocker/galaxy-central {ptestpath}"
749 try:
750 rlog = container.exec_run(ptestcl)
751 except:
752 pass
753 slogl = str(rlog).split('\\n')
754 slog = '\n'.join(slogl)
755 tout.write(f"## got rlog {slog} from mkdir {destdir}")
756 testouts = tempfile.mkdtemp(suffix=None, prefix="tftemp",dir=".")
757 self.copy_from_container(destdir,testouts,container)
758 try:
759 shutil.rmtree(os.path.join(testouts,self.tooloutdir,'test-data','test-data'))
760 except:
761 e = sys.exc_info()[0]
762 tout.write(f"#### error: {e} from {ptestcl}")
763 shutil.copytree(os.path.join(testouts,self.tooloutdir), self.tooloutdir, dirs_exist_ok=True)
764 tout.close()
765 container.stop()
766 container.remove()
767 shutil.rmtree(testouts)
768
769
770 def planemo_biodocker_vol_test(self):
771 """planemo currently leaks dependencies if used in the same container and gets unhappy after a
772 first successful run. https://github.com/galaxyproject/planemo/issues/1078#issuecomment-731476930
773
774 Docker biocontainer has planemo with caches filled to save repeated downloads
775 Cannot get volumes to work right in this version
776
777 """
778 if os.path.exists(self.tlog):
779 tout = open(self.tlog, "a")
780 else:
781 tout = open(self.tlog, "w")
782 planemoimage = "quay.io/fubar2/planemo-biocontainer"
783 xreal = "%s.xml" % self.tool_name
784 repname = f"{self.tool_name}_planemo_test_report.html"
785 ptestrep_path = os.path.join(self.repdir,repname)
786 tool_name = self.tool_name
787 workdir = "export"
788 aworkdir = os.path.abspath(workdir)
789 os.makedirs(workdir, exist_ok=True)
790 os.chmod(workdir,0o777)
791 imworkdir = "/export"
792 # must be mounted as a volume
793 tooldir = os.path.join(workdir,self.tool_name)
794 testdir = os.path.join(tooldir,'test-data')
795 imtooldir = os.path.join(imworkdir,self.tool_name)
796 imtestdir = os.path.join(imtooldir,'test-data')
797 for d in [tooldir,testdir]:
798 if not os.path.exists(d):
799 os.mkdir(d)
800 with os.scandir(self.testdir) as outs:
801 for entry in outs:
802 if not entry.is_file():
803 continue
804 src = os.path.join(self.testdir, entry.name)
805 dest = os.path.join(testdir, entry.name)
806 shutil.copyfile(src, dest)
807 shutil.copyfile(xreal,os.path.join(tooldir,xreal))
808 client = docker.from_env()
809 # mnt = docker.types.Mount(source='workdir', target=imworkdir) # mounts=[mnt],)
810 atestcl = "ls -lt /export"
811 container = client.containers.run(planemoimage,atestcl,
812 volumes={aworkdir:{'bind':'/export','mode':'rw'}}, )
813 tout.write(f"## Ran {atestcl} and got {container}")
814 ptestpath = os.path.join(imtooldir,xreal)
815 ptestcll = f"planemo test --job_output_files {imtooldir} --update_test_data --test_data {imtestdir} --galaxy_root /home/biodocker/galaxy-central {ptestpath}"
816 try:
817 container = client.containers.run(planemoimage,ptestcl,
818 volumes={aworkdir:{'bind':'/export','mode':'rw'}}, )
819 except:
820 pass
821 tout.write(f"## Ran {ptestcl}")
822 with os.scandir(testdir) as outs:
823 for entry in outs:
824 if not entry.is_file():
825 continue
826 src = os.path.join(testdir, entry.name)
827 dest = os.path.join(self.testdir, entry.name)
828 shutil.copyfile(src, dest)
829 imrep_path = os.path.join(imtooldir,repname)
830 ptestcl = f"planemo test --job_output_files {imtooldir} --test_output {imrep_path} --test_data {imtestdir} --galaxy_root /home/biodocker/galaxy-central {ptestpath}"
831 try:
832 container = client.containers.run(planemoimage,ptestcl,
833 volumes={aworkdir:{'bind':'/export','mode':'rw'}}, )
834 except:
835 pass
836 tout.write(f"## Ran {ptestcl}")
837 if os.path.isfile(imrep_path):
838 shutil.copyfile(imrep_path,ptestrep_path)
839 else:
840 tout.write(f"## planemo_biodocker_test - no test report {imrep_path} found")
841 tout.close()
842 #shutil.rmtree(workdir)
843
844
845
846 def gal_tool_test(self):
847 """
848 On path should be a handy script writes test outputs even if they don't exist
849
850 galaxy-tool-test -u http://localhost:8080 -a 3c9afe09f1b7892449d266109639c104 -o /tmp/foo -t hello -j /tmp/foo/hello.json --verbose
851
852 leaves outputs in -o !
853 """
854 gttscript = f"""#!{self.args.galaxy_venv}/bin/python3
855 # -*- coding: utf-8 -*-
856 import re
857 import sys
858 from galaxy.tool_util.verify.script import main
859 if __name__ == '__main__':
860 sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
861 sys.exit(main())
862
863 """
864 galaxy_lib = os.path.join(self.args.galaxy_root,'lib')
865 fakeenv = copy.copy(os.environ)
866 fakeenv ["PATH"] = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
867 fakeenv ["PYTHONPATH"] = f"{galaxy_lib}"
868 if os.path.exists(self.tlog):
869 tout = open(self.tlog, "a")
870 else:
871 tout = open(self.tlog, "w")
872 testouts = tempfile.mkdtemp(suffix=None, prefix="tftemp",dir="/tmp")
873 tout.write(f"#### using {testouts} as tempdir\n")
689 dummy, tfile = tempfile.mkstemp() 874 dummy, tfile = tempfile.mkstemp()
875 gtt = 'galaxy-tool-test'
876 gttf = open(gtt,'w')
877 gttf.write(gttscript)
878 gttf.write('\n')
879 gttf.close()
880 os.chmod(gtt,0o744)
690 cll = [ 881 cll = [
691 os.path.join(self.args.tool_dir,"galaxy-tool-test"), 882 os.path.abspath(gtt),
692 "-u", 883 "-u",
693 self.args.galaxy_url, 884 self.args.galaxy_url,
694 "-k", 885 "-k",
695 self.args.galaxy_api_key, 886 self.args.galaxy_api_key,
696 "-t", 887 "-t",
697 self.args.tool_name, 888 self.tool_name,
698 "-o", 889 "-o",
699 testouts, 890 testouts,
700 ] 891 ]
701 subp = subprocess.run( 892 subp = subprocess.run(
702 cll, shell=False, stderr=dummy, stdout=dummy 893 cll, env=fakeenv, cwd=galaxy_lib, shell=True, stderr=tout, stdout=tout
703 ) 894 )
704 outfiles = [] 895 outfiles = []
705 for p in self.outfiles: 896 for p in self.outfiles:
706 oname = p[ONAMEPOS] 897 oname = p[ONAMEPOS]
707 outfiles.append(oname) 898 outfiles.append(oname)
710 if not entry.is_file(): 901 if not entry.is_file():
711 continue 902 continue
712 dest = os.path.join(self.tooloutdir, entry.name) 903 dest = os.path.join(self.tooloutdir, entry.name)
713 src = os.path.join(testouts, entry.name) 904 src = os.path.join(testouts, entry.name)
714 shutil.copyfile(src, dest) 905 shutil.copyfile(src, dest)
715 dest = os.path.join(self.testdir, entry.name) 906 testdest = os.path.join(self.testdir, entry.name)
716 src = os.path.join(testouts, entry.name) 907 src = os.path.join(testouts, entry.name)
717 shutil.copyfile(src, dest) 908 shutil.copyfile(src, testdest)
718 dest = os.path.join(self.repdir,f"{entry.name}_sample") 909 dest = os.path.join(self.repdir,f"{entry.name}_sample")
719 tout.write(f"## found and moved output {entry.name} to {dest}\n") 910 tout.write(f"## found and moved output {entry.name} to {dest} and {testdest}\n")
720 tout.close() 911 tout.close()
721 shutil.rmtree(testouts) 912 #shutil.rmtree(testouts)
722 return subp.returncode 913 return subp.returncode
723 914
724 def gal_test(self): 915 def gal_test(self):
725 """ 916 """
726 Uses the built in galaxy tool tester run_test.sh 917 Uses the built in galaxy tool tester run_test.sh
727 918
728 export GALAXY_TEST_SAVE="./foo" && export GALAXY_TEST_NO_CLEANUP="1" \ 919 export GALAXY_TEST_SAVE="./foo" && export GALAXY_TEST_NO_CLEANUP="1" \
729 && export GALAXY_TEST_TMP_DIR=./foo && sh run_tests.sh --id rgtf2 --report_file tool_tests_tool_conf.html functional.test_toolbox 920 && export GALAXY_TEST_TMP_DIR=./foo && sh run_tests.sh --id rgtf2 --report_file tool_tests_tool_conf.html functional.test_toolbox
730 921
731 """ 922 """
732 testdir = tempfile.mkdtemp(suffix=None, prefix="tftemp") 923 testdir = tempfile.mkdtemp(suffix=None, prefix="tftemp",dir="/tmp")
733 tool_test_rep = f"{self.tool_name}_galaxy_test_report_html.html" 924 tool_test_rep = f"{self.tool_name}_galaxy_test_report_html.html"
734 if os.path.exists(self.tlog): 925 if os.path.exists(self.tlog):
735 tout = open(self.tlog, "a") 926 tout = open(self.tlog, "a")
736 else: 927 else:
737 tout = open(self.tlog, "w") 928 tout = open(self.tlog, "w")
738 929
739 ourenv = os.environ 930 fakeenv = copy.copy(os.environ)
740 ourenv["GALAXY_TEST_SAVE"] = testdir 931 fakeenv["GALAXY_TEST_SAVE"] = testdir
741 ourenv["GALAXY_TEST_NO_CLEANUP"] = "1" 932 fakeenv["GALAXY_TEST_NO_CLEANUP"] = "1"
742 ourenv["GALAXY_TEST_TMP_DIR"] = testdir 933 fakeenv["GALAXY_TEST_TMP_DIR"] = testdir
934 galaxy_lib = os.path.join(self.args.galaxy_root,'lib')
743 935
744 cll = [ 936 cll = [
745 "sh", f"{self.args.galaxy_root}/run_tests.sh", "--id", self.args.tool_name, 937 "sh", f"{self.args.galaxy_root}/run_tests.sh", "--id", self.tool_name,
746 "--report_file", os.path.join(testdir,tool_test_rep), "functional.test_toolbox", 938 "--report_file", os.path.join(testdir,tool_test_rep), "functional.test_toolbox",
747 ] 939 ]
748 subp = subprocess.run( 940 subp = subprocess.run(
749 cll, env = ourenv, 941 cll, env = fakeenv ,
750 shell=False, cwd=self.args.galaxy_root, stderr=tout, stdout=tout 942 shell=False, cwd=galaxy_lib, stderr=tout, stdout=tout
751 ) 943 )
752 src = os.path.join(testdir, tool_test_rep) 944 src = os.path.join(testdir, tool_test_rep)
753 if os.path.isfile(src): 945 if os.path.isfile(src):
754 dest = os.path.join(self.repdir, tool_test_rep) 946 dest = os.path.join(self.repdir, tool_test_rep)
755 shutil.copyfile(src, dest) 947 shutil.copyfile(src, dest)
782 rnames = [x.get("name", "?") for x in repos] 974 rnames = [x.get("name", "?") for x in repos]
783 rids = [x.get("id", "?") for x in repos] 975 rids = [x.get("id", "?") for x in repos]
784 sto.write(f"############names={rnames} rids={rids}\n") 976 sto.write(f"############names={rnames} rids={rids}\n")
785 sto.write(f"############names={repos}\n") 977 sto.write(f"############names={repos}\n")
786 tfcat = "ToolFactory generated tools" 978 tfcat = "ToolFactory generated tools"
787 if self.args.tool_name not in rnames: 979 if self.tool_name not in rnames:
788 tscat = ts.categories.get_categories() 980 tscat = ts.categories.get_categories()
789 cnames = [x.get("name", "?").strip() for x in tscat] 981 cnames = [x.get("name", "?").strip() for x in tscat]
790 cids = [x.get("id", "?") for x in tscat] 982 cids = [x.get("id", "?") for x in tscat]
791 catID = None 983 catID = None
792 if tfcat.strip() in cnames: 984 if tfcat.strip() in cnames:
802 category_ids=catID, 994 category_ids=catID,
803 ) 995 )
804 tid = res.get("id", None) 996 tid = res.get("id", None)
805 sto.write(f"##########create res={res}\n") 997 sto.write(f"##########create res={res}\n")
806 else: 998 else:
807 i = rnames.index(self.args.tool_name) 999 i = rnames.index(self.tool_name)
808 tid = rids[i] 1000 tid = rids[i]
809 try: 1001 try:
810 res = ts.repositories.update_repository( 1002 res = ts.repositories.update_repository(
811 id=tid, tar_ball_path=self.newtarpath, commit_message=None) 1003 id=tid, tar_ball_path=self.newtarpath, commit_message=None)
812 sto.write(f"#####update res={res}\n") 1004 sto.write(f"#####update res={res}\n")
827 self.args.galaxy_url, 1019 self.args.galaxy_url,
828 "--latest", 1020 "--latest",
829 "-a", 1021 "-a",
830 self.args.galaxy_api_key, 1022 self.args.galaxy_api_key,
831 "--name", 1023 "--name",
832 self.args.tool_name, 1024 self.tool_name,
833 "--owner", 1025 "--owner",
834 "fubar", 1026 "fubar",
835 "--toolshed", 1027 "--toolshed",
836 self.args.toolshed_url, 1028 self.args.toolshed_url,
837 "--section_label", 1029 "--section_label",
838 "ToolFactory", 1030 "ToolFactory",
839 ] 1031 ]
840 tout.write("running\n%s\n" % " ".join(cll)) 1032 tout.write("running\n%s\n" % " ".join(cll))
841 subp = subprocess.run(cll, shell=False, stderr=tout, stdout=tout) 1033 subp = subprocess.run(cll, env=self.ourenv, cwd=self.ourcwd, shell=False, stderr=tout, stdout=tout)
842 tout.write( 1034 tout.write(
843 "installed %s - got retcode %d\n" % (self.args.tool_name, subp.returncode) 1035 "installed %s - got retcode %d\n" % (self.tool_name, subp.returncode)
844 ) 1036 )
845 tout.close() 1037 tout.close()
846 return subp.returncode 1038 return subp.returncode
847 1039
848 def planemo_shedLoad(self): 1040 def planemo_shedLoad(self):
868 ) 1060 )
869 repos = ts.repositories.get_repositories() 1061 repos = ts.repositories.get_repositories()
870 rnames = [x.get("name", "?") for x in repos] 1062 rnames = [x.get("name", "?") for x in repos]
871 rids = [x.get("id", "?") for x in repos] 1063 rids = [x.get("id", "?") for x in repos]
872 #cat = "ToolFactory generated tools" 1064 #cat = "ToolFactory generated tools"
873 if self.args.tool_name not in rnames: 1065 if self.tool_name not in rnames:
874 cll = [ 1066 cll = [
875 "planemo", 1067 "planemo",
876 "shed_create", 1068 "shed_create",
877 "--shed_target", 1069 "--shed_target",
878 "local", 1070 "local",
879 "--owner", 1071 "--owner",
880 "fubar", 1072 "fubar",
881 "--name", 1073 "--name",
882 self.args.tool_name, 1074 self.tool_name,
883 "--shed_key", 1075 "--shed_key",
884 self.args.toolshed_api_key, 1076 self.args.toolshed_api_key,
885 ] 1077 ]
886 try: 1078 try:
887 subp = subprocess.run( 1079 subp = subprocess.run(
888 cll, shell=False, cwd=self.tooloutdir, stdout=tout, stderr=tout 1080 cll, env=self.ourenv, shell=False, cwd=self.tooloutdir, stdout=tout, stderr=tout
889 ) 1081 )
890 except: 1082 except:
891 pass 1083 pass
892 if subp.returncode != 0: 1084 if subp.returncode != 0:
893 tout.write("Repository %s exists\n" % self.args.tool_name) 1085 tout.write("Repository %s exists\n" % self.tool_name)
894 else: 1086 else:
895 tout.write("initiated %s\n" % self.args.tool_name) 1087 tout.write("initiated %s\n" % self.tool_name)
896 cll = [ 1088 cll = [
897 "planemo", 1089 "planemo",
898 "shed_upload", 1090 "shed_upload",
899 "--shed_target", 1091 "--shed_target",
900 "local", 1092 "local",
901 "--owner", 1093 "--owner",
902 "fubar", 1094 "fubar",
903 "--name", 1095 "--name",
904 self.args.tool_name, 1096 self.tool_name,
905 "--shed_key", 1097 "--shed_key",
906 self.args.toolshed_api_key, 1098 self.args.toolshed_api_key,
907 "--tar", 1099 "--tar",
908 self.newtarpath, 1100 self.newtarpath,
909 ] 1101 ]
910 subp = subprocess.run(cll, shell=False, stdout=tout, stderr=tout) 1102 subp = subprocess.run(cll, env=self.ourenv, cwd=self.ourcwd, shell=False, stdout=tout, stderr=tout)
911 tout.write("Ran %s got %d\n" % (" ".join(cll),subp.returncode)) 1103 tout.write("Ran %s got %d\n" % (" ".join(cll),subp.returncode))
912 tout.close() 1104 tout.close()
913 return subp.returncode 1105 return subp.returncode
914 1106
915 def eph_test(self, genoutputs=True): 1107 def eph_test(self, genoutputs=True):
925 "-g", 1117 "-g",
926 self.args.galaxy_url, 1118 self.args.galaxy_url,
927 "-a", 1119 "-a",
928 self.args.galaxy_api_key, 1120 self.args.galaxy_api_key,
929 "--name", 1121 "--name",
930 self.args.tool_name, 1122 self.tool_name,
931 "--owner", 1123 "--owner",
932 "fubar", 1124 "fubar",
933 ] 1125 ]
934 if genoutputs: 1126 if genoutputs:
935 dummy, tfile = tempfile.mkstemp() 1127 dummy, tfile = tempfile.mkstemp()
936 subp = subprocess.run( 1128 subp = subprocess.run(
937 cll, shell=False, stderr=dummy, stdout=dummy 1129 cll, env=self.ourenv, cwd=self.ourcwd, shell=False, stderr=dummy, stdout=dummy
938 ) 1130 )
939 1131
940 with open('tool_test_output.json','rb') as f: 1132 with open('tool_test_output.json','rb') as f:
941 s = json.loads(f.read()) 1133 s = json.loads(f.read())
942 print('read %s' % s) 1134 print('read %s' % s)
962 dest = os.path.join(self.tooloutdir, entry.name) 1154 dest = os.path.join(self.tooloutdir, entry.name)
963 src = os.path.join(ephouts, entry.name) 1155 src = os.path.join(ephouts, entry.name)
964 shutil.copyfile(src, dest) 1156 shutil.copyfile(src, dest)
965 else: 1157 else:
966 subp = subprocess.run( 1158 subp = subprocess.run(
967 cll, shell=False, stderr=tout, stdout=tout) 1159 cll, env=self.ourenv, cwd=self.ourcwd, shell=False, stderr=tout, stdout=tout)
968 tout.write("eph_test Ran %s got %d" % (" ".join(cll), subp.returncode)) 1160 tout.write("eph_test Ran %s got %d" % (" ".join(cll), subp.returncode))
969 tout.close() 1161 tout.close()
970 return subp.returncode 1162 return subp.returncode
971 1163
972 def planemo_test_biocontainer(self, genoutputs=True): 1164 def planemo_test_biocontainer(self, genoutputs=True):
1005 "--update_test_data", 1197 "--update_test_data",
1006 xreal, 1198 xreal,
1007 ] 1199 ]
1008 subp = subprocess.run( 1200 subp = subprocess.run(
1009 cll, 1201 cll,
1202 env=self.ourenv,
1010 shell=False, 1203 shell=False,
1011 cwd=self.tooloutdir, 1204 cwd=self.tooloutdir,
1012 stderr=dummy, 1205 stderr=dummy,
1013 stdout=dummy, 1206 stdout=dummy,
1014 ) 1207 )
1024 "--galaxy_root", 1217 "--galaxy_root",
1025 self.args.galaxy_root, 1218 self.args.galaxy_root,
1026 xreal, 1219 xreal,
1027 ] 1220 ]
1028 subp = subprocess.run( 1221 subp = subprocess.run(
1029 cll, shell=False, cwd=self.tooloutdir, stderr=tout, stdout=tout 1222 cll, env=self.ourenv, shell=False, cwd=self.tooloutdir, stderr=tout, stdout=tout
1030 ) 1223 )
1031 tout.close() 1224 tout.close()
1032 return subp.returncode 1225 return subp.returncode
1033 1226
1034 def planemo_test(self, genoutputs=True): 1227 def planemo_test(self, genoutputs=True):
1066 "--update_test_data", 1259 "--update_test_data",
1067 xreal, 1260 xreal,
1068 ] 1261 ]
1069 subp = subprocess.run( 1262 subp = subprocess.run(
1070 cll, 1263 cll,
1264 env=self.ourenv,
1071 shell=False, 1265 shell=False,
1072 cwd=self.testdir, 1266 cwd=self.testdir,
1073 stderr=dummy, 1267 stderr=dummy,
1074 stdout=dummy, 1268 stdout=dummy,
1075 ) 1269 )
1083 "--galaxy_root", 1277 "--galaxy_root",
1084 self.args.galaxy_root, 1278 self.args.galaxy_root,
1085 xreal, 1279 xreal,
1086 ] 1280 ]
1087 subp = subprocess.run( 1281 subp = subprocess.run(
1088 cll, shell=False, cwd=self.testdir, stderr=tout, stdout=tout 1282 cll, env=self.ourenv, shell=False, cwd=self.testdir, stderr=tout, stdout=tout
1089 ) 1283 )
1090 tout.close() 1284 tout.close()
1091 return subp.returncode 1285 return subp.returncode
1092 1286
1093 1287
1122 dest = os.path.join(self.testdir, "%s_sample" % p[ICLPOS]) 1316 dest = os.path.join(self.testdir, "%s_sample" % p[ICLPOS])
1123 shutil.copyfile(pth, dest) 1317 shutil.copyfile(pth, dest)
1124 1318
1125 def makeToolTar(self): 1319 def makeToolTar(self):
1126 """move outputs into test-data and prepare the tarball""" 1320 """move outputs into test-data and prepare the tarball"""
1127 excludeme = "tool_test_output" 1321 excludeme = "_planemo_test_report.html"
1128 1322
1129 def exclude_function(tarinfo): 1323 def exclude_function(tarinfo):
1130 filename = tarinfo.name 1324 filename = tarinfo.name
1131 return ( 1325 return (
1132 None 1326 None
1133 if filename.startswith(excludeme) 1327 if filename.endswith(excludeme)
1134 else tarinfo 1328 else tarinfo
1135 ) 1329 )
1136 1330
1137 for p in self.outfiles: 1331 for p in self.outfiles:
1138 oname = p[ONAMEPOS] 1332 oname = p[ONAMEPOS]
1169 dest = os.path.join(self.repdir, ofn) 1363 dest = os.path.join(self.repdir, ofn)
1170 src = os.path.join(self.tooloutdir, entry.name) 1364 src = os.path.join(self.tooloutdir, entry.name)
1171 shutil.copyfile(src, dest) 1365 shutil.copyfile(src, dest)
1172 with os.scandir(self.testdir) as outs: 1366 with os.scandir(self.testdir) as outs:
1173 for entry in outs: 1367 for entry in outs:
1174 if not entry.is_file(): 1368 if (not entry.is_file()) or entry.name.endswith('_sample') or entry.name.endswith("_planemo_test_report.html"):
1175 continue 1369 continue
1176 if "." in entry.name: 1370 if "." in entry.name:
1177 nayme, ext = os.path.splitext(entry.name) 1371 nayme, ext = os.path.splitext(entry.name)
1178 else: 1372 else:
1179 ext = ".txt" 1373 ext = ".txt"
1221 "--toolshed_url", default="http://localhost:9009") 1415 "--toolshed_url", default="http://localhost:9009")
1222 # make sure this is identical to tool_sheds_conf.xml localhost != 127.0.0.1 so validation fails 1416 # make sure this is identical to tool_sheds_conf.xml localhost != 127.0.0.1 so validation fails
1223 a("--toolshed_api_key", default="fakekey") 1417 a("--toolshed_api_key", default="fakekey")
1224 a("--galaxy_api_key", default="fakekey") 1418 a("--galaxy_api_key", default="fakekey")
1225 a("--galaxy_root", default="/galaxy-central") 1419 a("--galaxy_root", default="/galaxy-central")
1420 a("--galaxy_venv", default="/galaxy_venv")
1226 args = parser.parse_args() 1421 args = parser.parse_args()
1227 assert not args.bad_user, ( 1422 assert not args.bad_user, (
1228 'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy admin adds %s to "admin_users" in the Galaxy configuration file' 1423 'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy admin adds %s to "admin_users" in the Galaxy configuration file'
1229 % (args.bad_user, args.bad_user) 1424 % (args.bad_user, args.bad_user)
1230 ) 1425 )
1238 args.additional_parameters[i] = args.additional_parameters[i].replace('"', "") 1433 args.additional_parameters[i] = args.additional_parameters[i].replace('"', "")
1239 r = ScriptRunner(args) 1434 r = ScriptRunner(args)
1240 r.writeShedyml() 1435 r.writeShedyml()
1241 r.makeTool() 1436 r.makeTool()
1242 if args.make_Tool == "generate": 1437 if args.make_Tool == "generate":
1243 retcode = r.run() 1438 retcode = r.run() # for testing toolfactory itself
1244 r.moveRunOutputs() 1439 r.moveRunOutputs()
1245 r.makeToolTar() 1440 r.makeToolTar()
1246 else: 1441 else:
1247 r.makeToolTar() 1442 r.planemo_biodocker_test() # test to make outputs and then test
1248 r.planemo_shedLoad()
1249 r.shedLoad()
1250 r.eph_galaxy_load()
1251 retcode = r.gal_tool_test() # writes outputs
1252 r.makeToolTar()
1253 r.planemo_shedLoad()
1254 r.shedLoad()
1255 r.eph_galaxy_load()
1256 retcode = r.gal_test()
1257 r.moveRunOutputs() 1443 r.moveRunOutputs()
1258 r.makeToolTar() 1444 r.makeToolTar()
1259 print(f"second galaxy_test returned {retcode}") 1445 if args.make_Tool == "gentestinstall":
1446 r.shedLoad()
1447 r.eph_galaxy_load()
1260 1448
1261 1449
1262 if __name__ == "__main__": 1450 if __name__ == "__main__":
1263 main() 1451 main()