Mercurial > repos > fubar > tool_factory_2

comparison toolfactory/rgToolFactory2.py @ 98:67628c7dc9f3 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Uploaded
author fubar
date Sun, 22 Nov 2020 06:29:33 +0000
parents 515be7318f2b
children d4d88d393285
comparison
equal deleted inserted replaced
97:515be7318f2b 98:67628c7dc9f3
38 import time 38 import time
39 39
40 40
41 from bioblend import toolshed 41 from bioblend import toolshed
42 42
43 # import docker
44
43 import galaxyxml.tool as gxt 45 import galaxyxml.tool as gxt
44 import galaxyxml.tool.parameters as gxtp 46 import galaxyxml.tool.parameters as gxtp
45 47
46 import lxml 48 import lxml
47 49
181 self.args.tool_version, 183 self.args.tool_version,
182 self.args.tool_desc, 184 self.args.tool_desc,
183 FAKEEXE, 185 FAKEEXE,
184 ) 186 )
185 self.newtarpath = "toolfactory_%s.tgz" % self.tool_name 187 self.newtarpath = "toolfactory_%s.tgz" % self.tool_name
186 self.tooloutdir = "tfout" 188 self.tooloutdir = "./tfout"
187 self.repdir = "TF_run_report_tempdir" 189 self.repdir = "./TF_run_report_tempdir"
188 self.testdir = os.path.join(self.tooloutdir, "test-data") 190 self.testdir = os.path.join(self.tooloutdir, "test-data")
189 if not os.path.exists(self.tooloutdir): 191 if not os.path.exists(self.tooloutdir):
190 os.mkdir(self.tooloutdir) 192 os.mkdir(self.tooloutdir)
191 if not os.path.exists(self.testdir): 193 if not os.path.exists(self.testdir):
192 os.mkdir(self.testdir) # make tests directory 194 os.mkdir(self.testdir) # make tests directory
639 sto = open(self.tlog, "w") 641 sto = open(self.tlog, "w")
640 sto.write( 642 sto.write(
641 "## Executing Toolfactory generated command line = %s\n" % scl 643 "## Executing Toolfactory generated command line = %s\n" % scl
642 ) 644 )
643 sto.flush() 645 sto.flush()
644 p = subprocess.run(self.cl, shell=False, stdout=sto, stderr=ste) 646 subp = subprocess.run(self.cl, shell=False, stdout=sto, stderr=ste)
645 sto.close() 647 sto.close()
646 ste.close() 648 ste.close()
647 retval = p.returncode 649 retval = subp.returncode
648 else: # work around special case - stdin and write to stdout 650 else: # work around special case - stdin and write to stdout
649 if len(self.infiles) > 0: 651 if len(self.infiles) > 0:
650 sti = open(self.infiles[0][IPATHPOS], "rb") 652 sti = open(self.infiles[0][IPATHPOS], "rb")
651 else: 653 else:
652 sti = sys.stdin 654 sti = sys.stdin
653 if len(self.outfiles) > 0: 655 if len(self.outfiles) > 0:
654 sto = open(self.outfiles[0][ONAMEPOS], "wb") 656 sto = open(self.outfiles[0][ONAMEPOS], "wb")
655 else: 657 else:
656 sto = sys.stdout 658 sto = sys.stdout
657 p = subprocess.run(self.cl, shell=False, stdout=sto, stdin=sti) 659 subp = subprocess.run(self.cl, shell=False, stdout=sto, stdin=sti)
658 sto.write("## Executing Toolfactory generated command line = %s\n" % scl) 660 sto.write("## Executing Toolfactory generated command line = %s\n" % scl)
659 retval = p.returncode 661 retval = subp.returncode
660 sto.close() 662 sto.close()
661 sti.close() 663 sti.close()
662 if os.path.isfile(self.tlog) and os.stat(self.tlog).st_size == 0: 664 if os.path.isfile(self.tlog) and os.stat(self.tlog).st_size == 0:
663 os.unlink(self.tlog) 665 os.unlink(self.tlog)
664 if os.path.isfile(self.elog) and os.stat(self.elog).st_size == 0: 666 if os.path.isfile(self.elog) and os.stat(self.elog).st_size == 0:
689 ) 691 )
690 repos = ts.repositories.get_repositories() 692 repos = ts.repositories.get_repositories()
691 rnames = [x.get("name", "?") for x in repos] 693 rnames = [x.get("name", "?") for x in repos]
692 rids = [x.get("id", "?") for x in repos] 694 rids = [x.get("id", "?") for x in repos]
693 sto.write(f"############names={rnames} rids={rids}\n") 695 sto.write(f"############names={rnames} rids={rids}\n")
694 cat = "ToolFactory generated tools" 696 tfcat = "ToolFactory generated tools"
695 if self.args.tool_name not in rnames: 697 if self.args.tool_name not in rnames:
696 tscat = ts.categories.get_categories() 698 tscat = ts.categories.get_categories()
697 cnames = [x.get("name", "?") for x in tscat] 699 cnames = [x.get("name", "?").strip() for x in tscat]
698 cids = [x.get("id", "?") for x in tscat] 700 cids = [x.get("id", "?") for x in tscat]
699 catID = None 701 catID = None
700 if cat in cnames: 702 if tfcat.strip() in cnames:
701 ci = cnames.index(cat) 703 ci = cnames.index(tfcat)
702 catID = cids[ci] 704 catID = cids[ci]
703 res = ts.repositories.create_repository( 705 res = ts.repositories.create_repository(
704 name=self.args.tool_name, 706 name=self.args.tool_name,
705 synopsis="Synopsis:%s" % self.args.tool_desc, 707 synopsis="Synopsis:%s" % self.args.tool_desc,
706 description=self.args.tool_desc, 708 description=self.args.tool_desc,
742 self.args.toolshed_url, 744 self.args.toolshed_url,
743 "--section_label", 745 "--section_label",
744 "ToolFactory", 746 "ToolFactory",
745 ] 747 ]
746 tout.write("running\n%s\n" % " ".join(cll)) 748 tout.write("running\n%s\n" % " ".join(cll))
747 p = subprocess.run(cll, shell=False, stderr=tout, stdout=tout) 749 subp = subprocess.run(cll, shell=False, stderr=tout, stdout=tout)
748 tout.write( 750 tout.write(
749 "installed %s - got retcode %d\n" % (self.args.tool_name, p.returncode) 751 "installed %s - got retcode %d\n" % (self.args.tool_name, subp.returncode)
750 ) 752 )
751 tout.close() 753 tout.close()
752 return p.returncode 754 return subp.returncode
753 755
754 def planemo_shedload(self): 756 def planemo_shedload(self):
755 """ 757 """
756 planemo shed_create --shed_target testtoolshed 758 planemo shed_create --shed_target testtoolshed
757 planemo shed_init --name=<name> 759 planemo shed_init --name=<name>
788 self.args.tool_name, 790 self.args.tool_name,
789 "--shed_key", 791 "--shed_key",
790 self.args.toolshed_api_key, 792 self.args.toolshed_api_key,
791 ] 793 ]
792 try: 794 try:
793 p = subprocess.run( 795 subp = subprocess.run(
794 cll, shell=False, cwd=self.tooloutdir, stdout=tout, stderr=tout 796 cll, shell=False, cwd=self.tooloutdir, stdout=tout, stderr=tout
795 ) 797 )
796 except: 798 except:
797 pass 799 pass
798 if p.returncode != 0: 800 if subp.returncode != 0:
799 tout.write("Repository %s exists\n" % self.args.tool_name) 801 tout.write("Repository %s exists\n" % self.args.tool_name)
800 else: 802 else:
801 tout.write("initiated %s\n" % self.args.tool_name) 803 tout.write("initiated %s\n" % self.args.tool_name)
802 cll = [ 804 cll = [
803 "planemo", 805 "planemo",
811 "--shed_key", 813 "--shed_key",
812 self.args.toolshed_api_key, 814 self.args.toolshed_api_key,
813 "--tar", 815 "--tar",
814 self.newtarpath, 816 self.newtarpath,
815 ] 817 ]
816 p = subprocess.run(cll, shell=False, stdout=tout, stderr=tout) 818 subp = subprocess.run(cll, shell=False, stdout=tout, stderr=tout)
817 tout.write("Ran %s got %d\n" % (" ".join(cll), p.returncode)) 819 tout.write("Ran %s got %d\n" % (" ".join(cll),subp.returncode))
818 tout.close() 820 tout.close()
819 return p.returncode 821 return subp.returncode
820 822
821 def eph_test(self, genoutputs=True): 823 def eph_test(self, genoutputs=True):
822 """problem getting jobid - ephemeris upload is the job before the one we want - but depends on how many inputs 824 """problem getting jobid - ephemeris upload is the job before the one we want - but depends on how many inputs
823 """ 825 """
824 if os.path.exists(self.tlog): 826 if os.path.exists(self.tlog):
837 "--owner", 839 "--owner",
838 "fubar", 840 "fubar",
839 ] 841 ]
840 if genoutputs: 842 if genoutputs:
841 dummy, tfile = tempfile.mkstemp() 843 dummy, tfile = tempfile.mkstemp()
842 p = subprocess.run( 844 subp = subprocess.run(
843 cll, shell=False, stderr=dummy, stdout=dummy 845 cll, shell=False, stderr=dummy, stdout=dummy
844 ) 846 )
845 847
846 with open('tool_test_output.json','rb') as f: 848 with open('tool_test_output.json','rb') as f:
847 s = json.loads(f.read()) 849 s = json.loads(f.read())
867 continue 869 continue
868 dest = os.path.join(self.tooloutdir, entry.name) 870 dest = os.path.join(self.tooloutdir, entry.name)
869 src = os.path.join(ephouts, entry.name) 871 src = os.path.join(ephouts, entry.name)
870 shutil.copyfile(src, dest) 872 shutil.copyfile(src, dest)
871 else: 873 else:
872 p = subprocess.run( 874 subp = subprocess.run(
873 cll, shell=False, stderr=tout, stdout=tout) 875 cll, shell=False, stderr=tout, stdout=tout)
874 tout.write("eph_test Ran %s got %d" % (" ".join(cll), p.returncode)) 876 tout.write("eph_test Ran %s got %d" % (" ".join(cll), subp.returncode))
875 tout.close() 877 tout.close()
876 return p.returncode 878 return subp.returncode
877 879
878 def planemo_test_biocontainer(self, genoutputs=True): 880 def planemo_test_biocontainer(self, genoutputs=True):
879 """planemo is a requirement so is available for testing but testing in a biocontainer 881 """planemo is a requirement so is available for testing but testing in a biocontainer
880 requires some fiddling to use the hacked galaxy-central .venv 882 requires some fiddling to use the hacked galaxy-central .venv
881 883
901 dummy, tfile = tempfile.mkstemp() 903 dummy, tfile = tempfile.mkstemp()
902 cll = [ 904 cll = [
903 ".", os.path.join(self.args.galaxy_root,'.venv','bin','activate'),"&&", 905 ".", os.path.join(self.args.galaxy_root,'.venv','bin','activate'),"&&",
904 "planemo", 906 "planemo",
905 "test", 907 "test",
906 "--test_data", os.path.abspath(self.testdir), 908 "--test_data", self.testdir,
907 "--test_output", os.path.abspath(tool_test_path), 909 "--test_output", tool_test_path,
908 "--skip_venv", 910 "--skip_venv",
909 "--galaxy_root", 911 "--galaxy_root",
910 self.args.galaxy_root, 912 self.args.galaxy_root,
911 "--update_test_data", 913 "--update_test_data",
912 os.path.abspath(xreal), 914 xreal,
913 ] 915 ]
914 p = subprocess.run( 916 subp = subprocess.run(
915 cll, 917 cll,
916 shell=False, 918 shell=False,
917 cwd=self.tooloutdir, 919 cwd=self.tooloutdir,
918 stderr=dummy, 920 stderr=dummy,
919 stdout=dummy, 921 stdout=dummy,
922 else: 924 else:
923 cll = [ 925 cll = [
924 ".", os.path.join(self.args.galaxy_root,'.venv','bin','activate'),"&&", 926 ".", os.path.join(self.args.galaxy_root,'.venv','bin','activate'),"&&",
925 "planemo", 927 "planemo",
926 "test", 928 "test",
927 "--test_data", os.path.abspath(self.testdir), 929 "--test_data", os.path.self.testdir,
928 "--test_output", os.path.abspath(tool_test_path), 930 "--test_output", os.path.tool_test_path,
929 "--skip_venv", 931 "--skip_venv",
930 "--galaxy_root", 932 "--galaxy_root",
931 self.args.galaxy_root, 933 self.args.galaxy_root,
932 os.path.abspath(xreal), 934 xreal,
933 ] 935 ]
934 p = subprocess.run( 936 subp = subprocess.run(
935 cll, shell=False, cwd=self.tooloutdir, stderr=tout, stdout=tout 937 cll, shell=False, cwd=self.tooloutdir, stderr=tout, stdout=tout
936 ) 938 )
937 tout.close() 939 tout.close()
938 return p.returncode 940 return subp.returncode
939 941
940 def planemo_test(self, genoutputs=True): 942 def planemo_test(self, genoutputs=True):
941 """planemo is a requirement so is available for testing but needs a different call if 943 """planemo is a requirement so is available for testing but needs a different call if
942 in the biocontainer - see above 944 in the biocontainer - see above
943 and for generating test outputs if command or test overrides are supplied 945 and for generating test outputs if command or test overrides are supplied
965 if genoutputs: 967 if genoutputs:
966 dummy, tfile = tempfile.mkstemp() 968 dummy, tfile = tempfile.mkstemp()
967 cll = [ 969 cll = [
968 "planemo", 970 "planemo",
969 "test", 971 "test",
970 "--skip_venv",
971 "--galaxy_root", 972 "--galaxy_root",
972 self.args.galaxy_root, 973 self.args.galaxy_root,
973 "--update_test_data", 974 "--update_test_data",
974 os.path.abspath(xreal), 975 xreal,
975 ] 976 ]
976 p = subprocess.run( 977 subp = subprocess.run(
977 cll, 978 cll,
978 shell=False, 979 shell=False,
979 cwd=self.testdir, 980 cwd=self.testdir,
980 stderr=dummy, 981 stderr=dummy,
981 stdout=dummy, 982 stdout=dummy,
983 984
984 else: 985 else:
985 cll = [ 986 cll = [
986 "planemo", 987 "planemo",
987 "test", 988 "test",
988 "--test_data", os.path.abspath(self.testdir), 989 "--test_data", self.testdir,
989 "--test_output", os.path.abspath(tool_test_path), 990 "--test_output",tool_test_path,
990 "--skip_venv",
991 "--galaxy_root", 991 "--galaxy_root",
992 self.args.galaxy_root, 992 self.args.galaxy_root,
993 os.path.abspath(xreal), 993 xreal,
994 ] 994 ]
995 p = subprocess.run( 995 subp = subprocess.run(
996 cll, shell=False, cwd=self.testdir, stderr=tout, stdout=tout 996 cll, shell=False, cwd=self.testdir, stderr=tout, stdout=tout
997 ) 997 )
998 tout.close() 998 tout.close()
999 return p.returncode 999 return subp.returncode
1000 1000
1001 1001
1002 def gal_test(self, genoutputs=True): 1002 def gal_test(self, genoutputs=True):
1003 """ 1003 """
1004 export GALAXY_TEST_SAVE="./foo" && export GALAXY_TEST_NO_CLEANUP="1" \ 1004 export GALAXY_TEST_SAVE="./foo" && export GALAXY_TEST_NO_CLEANUP="1" \
1005 && export GALAXY_TEST_TMP_DIR=./foo && sh run_tests.sh --id rgtf2 --report_file tool_tests_tool_conf.html functional.test_toolbox 1005 && export GALAXY_TEST_TMP_DIR=./foo && sh run_tests.sh --id rgtf2 --report_file tool_tests_tool_conf.html functional.test_toolbox
1006 1006
1007 """ 1007 """
1008 tool_test_rep_path = os.path.join(self.repdir,f"{self.tool_name}_galaxy_test_report.html") 1008 tool_test_rep_path = os.path.join(self.repdir,f"{self.tool_name}_galaxy_test_report_html.html")
1009 if os.path.exists(self.tlog): 1009 if os.path.exists(self.tlog):
1010 tout = open(self.tlog, "a") 1010 tout = open(self.tlog, "a")
1011 else: 1011 else:
1012 tout = open(self.tlog, "w") 1012 tout = open(self.tlog, "w")
1013 testdir = '/tmp/testdir'
1014 ourenv = os.environ
1015 ourenv["GALAXY_TEST_SAVE"] = os.abspath(testdir)
1016 ourenv["GALAXY_TEST_NO_CLEANUP"] = "1"
1017 ourenv["GALAXY_TEST_TMP_DIR"] = os.abspath(testdir)
1013 if genoutputs: 1018 if genoutputs:
1014 dummy, tfile = tempfile.mkstemp() 1019
1020 if not os.path.exists(testdir):
1021 os.mkdir(testdir)
1022 dummy = open('failedtest.log','w')
1015 cll = [ 1023 cll = [
1016 "mkdir -p ./test","&&","export GALAXY_TEST_SAVE='./test'", "&&", "export GALAXY_TEST_NO_CLEANUP='1'", \ 1024 "sh", f"{self.args.galaxy_root}/run_tests.sh", "--id", self.args.tool_name,
1017 "&&", "export GALAXY_TEST_TMP_DIR='./test'", "&&", f"sh {self.args.galaxy_root}/run_tests.sh --id {self.args.tool_id} --report_file {tool_test_rep_path} functional.test_toolbox", 1025 "--report_file", tool_test_rep_path, "functional.test_toolbox",
1018 ] 1026 ]
1019 p = subprocess.run( 1027 subp = subprocess.run(
1020 cll, 1028 cll,
1021 shell=False, 1029 shell=False,
1030 env = ourenv,
1022 cwd=self.testdir, 1031 cwd=self.testdir,
1023 stderr=dummy, 1032 stderr=dummy,
1024 stdout=dummy, 1033 stdout=dummy,
1025 ) 1034 )
1035 dummy.close()
1026 # if all went well, tgz is in ./test and down a nest of tmp directories lurk the output files 1036 # if all went well, tgz is in ./test and down a nest of tmp directories lurk the output files
1027 outfiles = [] 1037 outfiles = []
1028 for p in self.outfiles: 1038 for p in self.outfiles:
1029 oname = p[ONAMEPOS] 1039 oname = p[ONAMEPOS]
1030 outfiles.append(oname) 1040 outfiles.append(oname)
1031 paths = [] 1041 paths = []
1032 for root, dirs, files in os.walk('./test'): 1042 for root, dirs, files in os.walk(testdir):
1033 for f in files: 1043 for f in files:
1044 tout.write(f"{root} {f}\n")
1034 if f in outfiles: 1045 if f in outfiles:
1035 paths.append([root,f]) 1046 paths.append([root,f])
1036 for p in paths: 1047 tout.write(f"## found output {f}")
1037 src = os.path.join(p[0],p[1]) 1048
1038 dest = os.path.join(self.testdir,f"{p[1]}_sample") 1049 if len(paths) != len(self.outfiles):
1050 tout.write(f"## problem - outfiles not all found")
1051 for path in paths:
1052 fname = path[1]
1053 src = os.path.join(path[0],fname)
1054 dest = os.path.join(self.testdir,f"{fname}_sample")
1039 shutil.copyfile(src,dest) 1055 shutil.copyfile(src,dest)
1040
1041 else: 1056 else:
1042 cll = [ 1057 cll = [
1043 "mkdir -p ./test","&&","rm -rf ./test/*","&&","export GALAXY_TEST_SAVE='./test'", "&&", "export GALAXY_TEST_NO_CLEANUP=", \ 1058 "sh", f"{self.args.galaxy_root}/run_tests.sh", "--id", self.args.tool_name,
1044 "&&", "export GALAXY_TEST_TMP_DIR='./test'", "&&", f"sh {self.args.galaxy_root}/run_tests.sh --id {self.args.tool_id} --report_file {tool_test_rep_path} functional.test_toolbox", 1059 "--report_file", tool_test_rep_path, "functional.test_toolbox",
1045 ] 1060 ]
1046 p = subprocess.run( 1061 subp = subprocess.run(
1047 cll, shell=False, cwd=self.testdir, stderr=tout, stdout=tout 1062 cll, env = ourenv,
1063 shell=False, cwd=self.testdir, stderr=tout, stdout=tout
1048 ) 1064 )
1049 tout.close() 1065 tout.close()
1050 return p.returncode 1066 return subp.returncode
1067
1051 1068
1052 1069
1053 def writeShedyml(self): 1070 def writeShedyml(self):
1054 """for planemo""" 1071 """for planemo"""
1055 yuser = self.args.user_email.split("@")[0] 1072 yuser = self.args.user_email.split("@")[0]
1108 ) 1125 )
1109 tf = tarfile.open(self.newtarpath, "w:gz") 1126 tf = tarfile.open(self.newtarpath, "w:gz")
1110 tf.add(name=self.tooloutdir, arcname=self.tool_name, filter=exclude_function) 1127 tf.add(name=self.tooloutdir, arcname=self.tool_name, filter=exclude_function)
1111 tf.close() 1128 tf.close()
1112 shutil.copyfile(self.newtarpath, self.args.new_tool) 1129 shutil.copyfile(self.newtarpath, self.args.new_tool)
1130
1131 def fakeToolTar(self):
1132 """move fake outputs into test-data and prepare the tarball"""
1133 excludeme = "tool_test_output"
1134
1135 def exclude_function(tarinfo):
1136 filename = tarinfo.name
1137 return (
1138 None
1139 if filename.startswith(excludeme)
1140 else tarinfo
1141 )
1142
1143 for p in self.outfiles:
1144 oname = p[ONAMEPOS]
1145 src = os.path.join(self.testdir,oname)
1146 if os.path.isfile(src):
1147 dest = os.path.join(self.testdir, "%s_sample" % oname)
1148 shutil.copyfile(src, dest)
1149 dest = os.path.join(self.repdir, "%s.sample" % (oname))
1150 shutil.copyfile(src, dest)
1151 else:
1152 with open(src,'w') as fayk:
1153 fayk.write('fake!\n')
1154 dest = os.path.join(self.testdir, "%s_sample" % oname)
1155 shutil.copyfile(src, dest)
1156 print(
1157 "### problem - output file %s not found in tooloutdir %s so faked"
1158 % (src, self.tooloutdir)
1159 )
1160 tf = tarfile.open(self.newtarpath, "w:gz")
1161 tf.add(name=self.tooloutdir, arcname=self.tool_name, filter=exclude_function)
1162 tf.close()
1163 shutil.copyfile(self.newtarpath, self.args.new_tool)
1164
1113 1165
1114 def moveRunOutputs(self): 1166 def moveRunOutputs(self):
1115 """need to move planemo or run outputs into toolfactory collection""" 1167 """need to move planemo or run outputs into toolfactory collection"""
1116 with os.scandir(self.tooloutdir) as outs: 1168 with os.scandir(self.tooloutdir) as outs:
1117 for entry in outs: 1169 for entry in outs:
1200 if args.make_Tool == "generate": 1252 if args.make_Tool == "generate":
1201 retcode = r.run() 1253 retcode = r.run()
1202 r.moveRunOutputs() 1254 r.moveRunOutputs()
1203 r.makeToolTar() 1255 r.makeToolTar()
1204 else: 1256 else:
1205 r.makeToolTar() 1257 r.fakeToolTar()
1206 r.shedLoad() 1258 r.shedLoad()
1207 r.eph_galaxy_load() 1259 r.eph_galaxy_load()
1208 retcode = r.gal_test(genoutputs=True) # this fails 1260 retcode = r.gal_test(genoutputs=True) # this fails
1209 r.makeToolTar() 1261 r.makeToolTar()
1210 retcode = r.gal_test(genoutputs=False) 1262 retcode = r.gal_test(genoutputs=False)