Mercurial > repos > fubar > tool_factory_2

comparison toolfactory/rgToolFactory2.py @ 102:c632db66f8c0 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Uploaded
author fubar
date Thu, 26 Nov 2020 06:26:28 +0000
parents 557d5f06f213
children 2ca51677d6d1
comparison
equal deleted inserted replaced
101:557d5f06f213 102:c632db66f8c0
764 tout.close() 764 tout.close()
765 container.stop() 765 container.stop()
766 container.remove() 766 container.remove()
767 shutil.rmtree(testouts) 767 shutil.rmtree(testouts)
768 768
769
770 def planemo_biodocker_vol_test(self):
771 """planemo currently leaks dependencies if used in the same container and gets unhappy after a
772 first successful run. https://github.com/galaxyproject/planemo/issues/1078#issuecomment-731476930
773
774 Docker biocontainer has planemo with caches filled to save repeated downloads
775 Cannot get volumes to work right in this version
776
777 """
778 if os.path.exists(self.tlog):
779 tout = open(self.tlog, "a")
780 else:
781 tout = open(self.tlog, "w")
782 planemoimage = "quay.io/fubar2/planemo-biocontainer"
783 xreal = "%s.xml" % self.tool_name
784 repname = f"{self.tool_name}_planemo_test_report.html"
785 ptestrep_path = os.path.join(self.repdir,repname)
786 tool_name = self.tool_name
787 workdir = "export"
788 aworkdir = os.path.abspath(workdir)
789 os.makedirs(workdir, exist_ok=True)
790 os.chmod(workdir,0o777)
791 imworkdir = "/export"
792 # must be mounted as a volume
793 tooldir = os.path.join(workdir,self.tool_name)
794 testdir = os.path.join(tooldir,'test-data')
795 imtooldir = os.path.join(imworkdir,self.tool_name)
796 imtestdir = os.path.join(imtooldir,'test-data')
797 for d in [tooldir,testdir]:
798 if not os.path.exists(d):
799 os.mkdir(d)
800 with os.scandir(self.testdir) as outs:
801 for entry in outs:
802 if not entry.is_file():
803 continue
804 src = os.path.join(self.testdir, entry.name)
805 dest = os.path.join(testdir, entry.name)
806 shutil.copyfile(src, dest)
807 shutil.copyfile(xreal,os.path.join(tooldir,xreal))
808 client = docker.from_env()
809 # mnt = docker.types.Mount(source='workdir', target=imworkdir) # mounts=[mnt],)
810 atestcl = "ls -lt /export"
811 container = client.containers.run(planemoimage,atestcl,
812 volumes={aworkdir:{'bind':'/export','mode':'rw'}}, )
813 tout.write(f"## Ran {atestcl} and got {container}")
814 ptestpath = os.path.join(imtooldir,xreal)
815 ptestcll = f"planemo test --job_output_files {imtooldir} --update_test_data --test_data {imtestdir} --galaxy_root /home/biodocker/galaxy-central {ptestpath}"
816 try:
817 container = client.containers.run(planemoimage,ptestcl,
818 volumes={aworkdir:{'bind':'/export','mode':'rw'}}, )
819 except:
820 pass
821 tout.write(f"## Ran {ptestcl}")
822 with os.scandir(testdir) as outs:
823 for entry in outs:
824 if not entry.is_file():
825 continue
826 src = os.path.join(testdir, entry.name)
827 dest = os.path.join(self.testdir, entry.name)
828 shutil.copyfile(src, dest)
829 imrep_path = os.path.join(imtooldir,repname)
830 ptestcl = f"planemo test --job_output_files {imtooldir} --test_output {imrep_path} --test_data {imtestdir} --galaxy_root /home/biodocker/galaxy-central {ptestpath}"
831 try:
832 container = client.containers.run(planemoimage,ptestcl,
833 volumes={aworkdir:{'bind':'/export','mode':'rw'}}, )
834 except:
835 pass
836 tout.write(f"## Ran {ptestcl}")
837 if os.path.isfile(imrep_path):
838 shutil.copyfile(imrep_path,ptestrep_path)
839 else:
840 tout.write(f"## planemo_biodocker_test - no test report {imrep_path} found")
841 tout.close()
842 #shutil.rmtree(workdir)
843
844
845
846 def gal_tool_test(self):
847 """
848 On path should be a handy script writes test outputs even if they don't exist
849
850 galaxy-tool-test -u http://localhost:8080 -a 3c9afe09f1b7892449d266109639c104 -o /tmp/foo -t hello -j /tmp/foo/hello.json --verbose
851
852 leaves outputs in -o !
853 """
854 gttscript = f"""#!{self.args.galaxy_venv}/bin/python3
855 # -*- coding: utf-8 -*-
856 import re
857 import sys
858 from galaxy.tool_util.verify.script import main
859 if __name__ == '__main__':
860 sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
861 sys.exit(main())
862
863 """
864 galaxy_lib = os.path.join(self.args.galaxy_root,'lib')
865 fakeenv = copy.copy(os.environ)
866 fakeenv ["PATH"] = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
867 fakeenv ["PYTHONPATH"] = f"{galaxy_lib}"
868 if os.path.exists(self.tlog):
869 tout = open(self.tlog, "a")
870 else:
871 tout = open(self.tlog, "w")
872 testouts = tempfile.mkdtemp(suffix=None, prefix="tftemp",dir="/tmp")
873 tout.write(f"#### using {testouts} as tempdir\n")
874 dummy, tfile = tempfile.mkstemp()
875 gtt = 'galaxy-tool-test'
876 gttf = open(gtt,'w')
877 gttf.write(gttscript)
878 gttf.write('\n')
879 gttf.close()
880 os.chmod(gtt,0o744)
881 cll = [
882 os.path.abspath(gtt),
883 "-u",
884 self.args.galaxy_url,
885 "-k",
886 self.args.galaxy_api_key,
887 "-t",
888 self.tool_name,
889 "-o",
890 testouts,
891 ]
892 subp = subprocess.run(
893 cll, env=fakeenv, cwd=galaxy_lib, shell=True, stderr=tout, stdout=tout
894 )
895 outfiles = []
896 for p in self.outfiles:
897 oname = p[ONAMEPOS]
898 outfiles.append(oname)
899 with os.scandir(testouts) as outs:
900 for entry in outs:
901 if not entry.is_file():
902 continue
903 dest = os.path.join(self.tooloutdir, entry.name)
904 src = os.path.join(testouts, entry.name)
905 shutil.copyfile(src, dest)
906 testdest = os.path.join(self.testdir, entry.name)
907 src = os.path.join(testouts, entry.name)
908 shutil.copyfile(src, testdest)
909 dest = os.path.join(self.repdir,f"{entry.name}_sample")
910 tout.write(f"## found and moved output {entry.name} to {dest} and {testdest}\n")
911 tout.close()
912 #shutil.rmtree(testouts)
913 return subp.returncode
914
915 def gal_test(self):
916 """
917 Uses the built in galaxy tool tester run_test.sh
918
919 export GALAXY_TEST_SAVE="./foo" && export GALAXY_TEST_NO_CLEANUP="1" \
920 && export GALAXY_TEST_TMP_DIR=./foo && sh run_tests.sh --id rgtf2 --report_file tool_tests_tool_conf.html functional.test_toolbox
921
922 """
923 testdir = tempfile.mkdtemp(suffix=None, prefix="tftemp",dir="/tmp")
924 tool_test_rep = f"{self.tool_name}_galaxy_test_report_html.html"
925 if os.path.exists(self.tlog):
926 tout = open(self.tlog, "a")
927 else:
928 tout = open(self.tlog, "w")
929
930 fakeenv = copy.copy(os.environ)
931 fakeenv["GALAXY_TEST_SAVE"] = testdir
932 fakeenv["GALAXY_TEST_NO_CLEANUP"] = "1"
933 fakeenv["GALAXY_TEST_TMP_DIR"] = testdir
934 galaxy_lib = os.path.join(self.args.galaxy_root,'lib')
935
936 cll = [
937 "sh", f"{self.args.galaxy_root}/run_tests.sh", "--id", self.tool_name,
938 "--report_file", os.path.join(testdir,tool_test_rep), "functional.test_toolbox",
939 ]
940 subp = subprocess.run(
941 cll, env = fakeenv ,
942 shell=False, cwd=galaxy_lib, stderr=tout, stdout=tout
943 )
944 src = os.path.join(testdir, tool_test_rep)
945 if os.path.isfile(src):
946 dest = os.path.join(self.repdir, tool_test_rep)
947 shutil.copyfile(src, dest)
948 else:
949 tout.write(f"### {src} not found\n")
950 tout.close()
951 return subp.returncode
952
953
954 def shedLoad(self): 769 def shedLoad(self):
955 """ 770 """
956 {'deleted': False, 771 {'deleted': False,
957 'description': 'Tools for manipulating data', 772 'description': 'Tools for manipulating data',
958 'id': '175812cd7caaf439', 773 'id': '175812cd7caaf439',
1218 self.args.galaxy_root, 1033 self.args.galaxy_root,
1219 xreal, 1034 xreal,
1220 ] 1035 ]
1221 subp = subprocess.run( 1036 subp = subprocess.run(
1222 cll, env=self.ourenv, shell=False, cwd=self.tooloutdir, stderr=tout, stdout=tout 1037 cll, env=self.ourenv, shell=False, cwd=self.tooloutdir, stderr=tout, stdout=tout
1223 )
1224 tout.close()
1225 return subp.returncode
1226
1227 def planemo_test(self, genoutputs=True):
1228 """planemo is a requirement so is available for testing but needs a different call if
1229 in the biocontainer - see above
1230 and for generating test outputs if command or test overrides are supplied
1231 test outputs are sent to repdir for display
1232 planemo test --engine docker_galaxy --galaxy_root /galaxy-central pyrevpos/pyrevpos.xml
1233
1234 Planemo runs:
1235 python ./scripts/functional_tests.py -v --with-nosehtml --html-report-file
1236 /export/galaxy-central/database/job_working_directory/000/17/working/TF_run_report_tempdir/tacrev_planemo_test_report.html
1237 --with-xunit --xunit-file /tmp/tmpt90p7f9h/xunit.xml --with-structureddata
1238 --structured-data-file
1239 /export/galaxy-central/database/job_working_directory/000/17/working/tfout/tool_test_output.json functional.test_toolbox
1240
1241
1242 for the planemo-biocontainer,
1243 planemo test --conda_dependency_resolution --skip_venv --galaxy_root /galthrow/ rgToolFactory2.xml
1244
1245 """
1246 xreal = "%s.xml" % self.tool_name
1247 tool_test_path = os.path.join(self.repdir,f"{self.tool_name}_planemo_test_report.html")
1248 if os.path.exists(self.tlog):
1249 tout = open(self.tlog, "a")
1250 else:
1251 tout = open(self.tlog, "w")
1252 if genoutputs:
1253 dummy, tfile = tempfile.mkstemp()
1254 cll = [
1255 "planemo",
1256 "test",
1257 "--galaxy_root",
1258 self.args.galaxy_root,
1259 "--update_test_data",
1260 xreal,
1261 ]
1262 subp = subprocess.run(
1263 cll,
1264 env=self.ourenv,
1265 shell=False,
1266 cwd=self.testdir,
1267 stderr=dummy,
1268 stdout=dummy,
1269 )
1270
1271 else:
1272 cll = [
1273 "planemo",
1274 "test",
1275 "--test_data", self.testdir,
1276 "--test_output",tool_test_path,
1277 "--galaxy_root",
1278 self.args.galaxy_root,
1279 xreal,
1280 ]
1281 subp = subprocess.run(
1282 cll, env=self.ourenv, shell=False, cwd=self.testdir, stderr=tout, stdout=tout
1283 ) 1038 )
1284 tout.close() 1039 tout.close()
1285 return subp.returncode 1040 return subp.returncode
1286 1041
1287 1042