# HG changeset patch # User fubar # Date 1606371432 0 # Node ID 557d5f06f213df8e0d0a2d309b96281c69b40acb # Parent c749364c2283efc9d533f10cc5da88ce372bba2a Uploaded diff -r c749364c2283 -r 557d5f06f213 toolfactory/rgToolFactory2.py --- a/toolfactory/rgToolFactory2.py Mon Nov 23 03:12:37 2020 +0000 +++ b/toolfactory/rgToolFactory2.py Thu Nov 26 06:17:12 2020 +0000 @@ -26,6 +26,7 @@ # planemo test --engine docker_galaxy --test_data ./test-data/ --docker_extra_volume ./test-data rgToolFactory2.xml import argparse +import copy import datetime import json import logging @@ -42,7 +43,7 @@ from bioblend import ConnectionError from bioblend import toolshed -# import docker +import docker import galaxyxml.tool as gxt import galaxyxml.tool.parameters as gxtp @@ -153,6 +154,8 @@ prepare command line cl for running the tool here and prepare elements needed for galaxyxml tool generation """ + self.ourcwd = os.getcwd() + self.ourenv = copy.deepcopy(os.environ) self.infiles = [x.split(ourdelim) for x in args.input_files] self.outfiles = [x.split(ourdelim) for x in args.output_files] self.addpar = [x.split(ourdelim) for x in args.additional_parameters] @@ -180,7 +183,7 @@ self.tool_name = re.sub("[^a-zA-Z0-9_]+", "", args.tool_name) self.tool_id = self.tool_name self.newtool = gxt.Tool( - self.args.tool_name, + self.tool_name, self.tool_id, self.args.tool_version, self.args.tool_desc, @@ -645,7 +648,7 @@ "## Executing Toolfactory generated command line = %s\n" % scl ) sto.flush() - subp = subprocess.run(self.cl, shell=False, stdout=sto, stderr=ste) + subp = subprocess.run(self.cl, env=self.ourenv, shell=False, stdout=sto, stderr=ste) sto.close() ste.close() retval = subp.returncode @@ -658,7 +661,7 @@ sto = open(self.outfiles[0][ONAMEPOS], "wb") else: sto = sys.stdout - subp = subprocess.run(self.cl, shell=False, stdout=sto, stdin=sti) + subp = subprocess.run(self.cl, env=self.ourenv, shell=False, stdout=sto, stdin=sti) sto.write("## Executing Toolfactory generated command line = %s\n" % scl) retval = subp.returncode sto.close() @@ -673,33 +676,221 @@ return retval - def gal_tool_test(self): + def copy_to_container(self, src, dest, container): + """ Recreate the src directory tree at dest - full path included + """ + idir = os.getcwd() + workdir = os.path.dirname(src) + os.chdir(workdir) + _, tfname = tempfile.mkstemp(suffix=".tar") + tar = tarfile.open(tfname, mode='w') + srcb = os.path.basename(src) + tar.add(srcb) + tar.close() + data = open(tfname, 'rb').read() + container.put_archive(dest, data) + os.unlink(tfname) + os.chdir(idir) + + + def copy_from_container(self, src, dest, container): + """ recreate the src directory tree at dest using docker sdk + """ + os.makedirs(dest,exist_ok=True) + _, tfname = tempfile.mkstemp(suffix=".tar") + tf = open(tfname,'wb') + bits, stat = container.get_archive(src) + for chunk in bits: + tf.write(chunk) + tf.close() + tar = tarfile.open(tfname,'r') + tar.extractall(dest) + tar.close() + os.unlink(tfname) + + + + def planemo_biodocker_test(self): + """planemo currently leaks dependencies if used in the same container and gets unhappy after a + first successful run. https://github.com/galaxyproject/planemo/issues/1078#issuecomment-731476930 + + Docker biocontainer has planemo with caches filled to save repeated downloads + + """ - This handy script writes test outputs even if they don't exist - galaxy-tool-test [-h] [-u GALAXY_URL] [-k KEY] [-a ADMIN_KEY] [--force_path_paste] [-t TOOL_ID] [--tool-version TOOL_VERSION] - [-i TEST_INDEX] [-o OUTPUT] [--append] [-j OUTPUT_JSON] [--verbose] [-c CLIENT_TEST_CONFIG] - galaxy-tool-test -u http://localhost:8080 -a 3c9afe09f1b7892449d266109639c104 -o /tmp/foo -t hello -j /tmp/foo/hello.json --verbose - handy - just leaves outputs in -o + if os.path.exists(self.tlog): + tout = open(self.tlog, "a") + else: + tout = open(self.tlog, "w") + planemoimage = "quay.io/fubar2/planemo-biocontainer" + xreal = "%s.xml" % self.tool_name + destdir = "/tmp/tfout" + repname = f"{self.tool_name}_planemo_test_report.html" + imrep = os.path.join(destdir,repname) + ptestrep_path = os.path.join(self.repdir,repname) + tool_name = self.tool_name + client = docker.from_env() + container = client.containers.run(planemoimage,'sleep 10000m', detach=True) + rlog = container.exec_run(f"mkdir -p {destdir}") + slogl = str(rlog).split('\\n') + slog = '\n'.join(slogl) + tout.write(f"## got rlog {slog} from mkdir {destdir}") + ptestpath = os.path.join(destdir,xreal) + self.copy_to_container(self.tooloutdir,'/tmp',container) + rlog = container.exec_run(f"ls -la {destdir}") + ptestcl = f"planemo test --update_test_data --galaxy_root /home/biodocker/galaxy-central {ptestpath}" + try: + rlog = container.exec_run(ptestcl) + except: + e = sys.exc_info()[0] + tout.write(f"#### error: {e} from {ptestcl}") + # fails - used to generate test outputs + ptestcl = f"planemo test --test_output {imrep} --galaxy_root /home/biodocker/galaxy-central {ptestpath}" + try: + rlog = container.exec_run(ptestcl) + except: + pass + slogl = str(rlog).split('\\n') + slog = '\n'.join(slogl) + tout.write(f"## got rlog {slog} from mkdir {destdir}") + testouts = tempfile.mkdtemp(suffix=None, prefix="tftemp",dir=".") + self.copy_from_container(destdir,testouts,container) + try: + shutil.rmtree(os.path.join(testouts,self.tooloutdir,'test-data','test-data')) + except: + e = sys.exc_info()[0] + tout.write(f"#### error: {e} from {ptestcl}") + shutil.copytree(os.path.join(testouts,self.tooloutdir), self.tooloutdir, dirs_exist_ok=True) + tout.close() + container.stop() + container.remove() + shutil.rmtree(testouts) + + + def planemo_biodocker_vol_test(self): + """planemo currently leaks dependencies if used in the same container and gets unhappy after a + first successful run. https://github.com/galaxyproject/planemo/issues/1078#issuecomment-731476930 + + Docker biocontainer has planemo with caches filled to save repeated downloads + Cannot get volumes to work right in this version + """ if os.path.exists(self.tlog): tout = open(self.tlog, "a") else: tout = open(self.tlog, "w") - testouts = tempfile.mkdtemp(suffix=None, prefix="tftemp") + planemoimage = "quay.io/fubar2/planemo-biocontainer" + xreal = "%s.xml" % self.tool_name + repname = f"{self.tool_name}_planemo_test_report.html" + ptestrep_path = os.path.join(self.repdir,repname) + tool_name = self.tool_name + workdir = "export" + aworkdir = os.path.abspath(workdir) + os.makedirs(workdir, exist_ok=True) + os.chmod(workdir,0o777) + imworkdir = "/export" + # must be mounted as a volume + tooldir = os.path.join(workdir,self.tool_name) + testdir = os.path.join(tooldir,'test-data') + imtooldir = os.path.join(imworkdir,self.tool_name) + imtestdir = os.path.join(imtooldir,'test-data') + for d in [tooldir,testdir]: + if not os.path.exists(d): + os.mkdir(d) + with os.scandir(self.testdir) as outs: + for entry in outs: + if not entry.is_file(): + continue + src = os.path.join(self.testdir, entry.name) + dest = os.path.join(testdir, entry.name) + shutil.copyfile(src, dest) + shutil.copyfile(xreal,os.path.join(tooldir,xreal)) + client = docker.from_env() + # mnt = docker.types.Mount(source='workdir', target=imworkdir) # mounts=[mnt],) + atestcl = "ls -lt /export" + container = client.containers.run(planemoimage,atestcl, + volumes={aworkdir:{'bind':'/export','mode':'rw'}}, ) + tout.write(f"## Ran {atestcl} and got {container}") + ptestpath = os.path.join(imtooldir,xreal) + ptestcll = f"planemo test --job_output_files {imtooldir} --update_test_data --test_data {imtestdir} --galaxy_root /home/biodocker/galaxy-central {ptestpath}" + try: + container = client.containers.run(planemoimage,ptestcl, + volumes={aworkdir:{'bind':'/export','mode':'rw'}}, ) + except: + pass + tout.write(f"## Ran {ptestcl}") + with os.scandir(testdir) as outs: + for entry in outs: + if not entry.is_file(): + continue + src = os.path.join(testdir, entry.name) + dest = os.path.join(self.testdir, entry.name) + shutil.copyfile(src, dest) + imrep_path = os.path.join(imtooldir,repname) + ptestcl = f"planemo test --job_output_files {imtooldir} --test_output {imrep_path} --test_data {imtestdir} --galaxy_root /home/biodocker/galaxy-central {ptestpath}" + try: + container = client.containers.run(planemoimage,ptestcl, + volumes={aworkdir:{'bind':'/export','mode':'rw'}}, ) + except: + pass + tout.write(f"## Ran {ptestcl}") + if os.path.isfile(imrep_path): + shutil.copyfile(imrep_path,ptestrep_path) + else: + tout.write(f"## planemo_biodocker_test - no test report {imrep_path} found") + tout.close() + #shutil.rmtree(workdir) + + + + def gal_tool_test(self): + """ + On path should be a handy script writes test outputs even if they don't exist + + galaxy-tool-test -u http://localhost:8080 -a 3c9afe09f1b7892449d266109639c104 -o /tmp/foo -t hello -j /tmp/foo/hello.json --verbose + + leaves outputs in -o ! + """ + gttscript = f"""#!{self.args.galaxy_venv}/bin/python3 +# -*- coding: utf-8 -*- +import re +import sys +from galaxy.tool_util.verify.script import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) + + """ + galaxy_lib = os.path.join(self.args.galaxy_root,'lib') + fakeenv = copy.copy(os.environ) + fakeenv ["PATH"] = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + fakeenv ["PYTHONPATH"] = f"{galaxy_lib}" + if os.path.exists(self.tlog): + tout = open(self.tlog, "a") + else: + tout = open(self.tlog, "w") + testouts = tempfile.mkdtemp(suffix=None, prefix="tftemp",dir="/tmp") + tout.write(f"#### using {testouts} as tempdir\n") dummy, tfile = tempfile.mkstemp() + gtt = 'galaxy-tool-test' + gttf = open(gtt,'w') + gttf.write(gttscript) + gttf.write('\n') + gttf.close() + os.chmod(gtt,0o744) cll = [ - os.path.join(self.args.tool_dir,"galaxy-tool-test"), + os.path.abspath(gtt), "-u", self.args.galaxy_url, "-k", self.args.galaxy_api_key, "-t", - self.args.tool_name, + self.tool_name, "-o", testouts, ] subp = subprocess.run( - cll, shell=False, stderr=dummy, stdout=dummy + cll, env=fakeenv, cwd=galaxy_lib, shell=True, stderr=tout, stdout=tout ) outfiles = [] for p in self.outfiles: @@ -712,13 +903,13 @@ dest = os.path.join(self.tooloutdir, entry.name) src = os.path.join(testouts, entry.name) shutil.copyfile(src, dest) - dest = os.path.join(self.testdir, entry.name) + testdest = os.path.join(self.testdir, entry.name) src = os.path.join(testouts, entry.name) - shutil.copyfile(src, dest) + shutil.copyfile(src, testdest) dest = os.path.join(self.repdir,f"{entry.name}_sample") - tout.write(f"## found and moved output {entry.name} to {dest}\n") + tout.write(f"## found and moved output {entry.name} to {dest} and {testdest}\n") tout.close() - shutil.rmtree(testouts) + #shutil.rmtree(testouts) return subp.returncode def gal_test(self): @@ -729,25 +920,26 @@ && export GALAXY_TEST_TMP_DIR=./foo && sh run_tests.sh --id rgtf2 --report_file tool_tests_tool_conf.html functional.test_toolbox """ - testdir = tempfile.mkdtemp(suffix=None, prefix="tftemp") + testdir = tempfile.mkdtemp(suffix=None, prefix="tftemp",dir="/tmp") tool_test_rep = f"{self.tool_name}_galaxy_test_report_html.html" if os.path.exists(self.tlog): tout = open(self.tlog, "a") else: tout = open(self.tlog, "w") - ourenv = os.environ - ourenv["GALAXY_TEST_SAVE"] = testdir - ourenv["GALAXY_TEST_NO_CLEANUP"] = "1" - ourenv["GALAXY_TEST_TMP_DIR"] = testdir + fakeenv = copy.copy(os.environ) + fakeenv["GALAXY_TEST_SAVE"] = testdir + fakeenv["GALAXY_TEST_NO_CLEANUP"] = "1" + fakeenv["GALAXY_TEST_TMP_DIR"] = testdir + galaxy_lib = os.path.join(self.args.galaxy_root,'lib') cll = [ - "sh", f"{self.args.galaxy_root}/run_tests.sh", "--id", self.args.tool_name, + "sh", f"{self.args.galaxy_root}/run_tests.sh", "--id", self.tool_name, "--report_file", os.path.join(testdir,tool_test_rep), "functional.test_toolbox", ] subp = subprocess.run( - cll, env = ourenv, - shell=False, cwd=self.args.galaxy_root, stderr=tout, stdout=tout + cll, env = fakeenv , + shell=False, cwd=galaxy_lib, stderr=tout, stdout=tout ) src = os.path.join(testdir, tool_test_rep) if os.path.isfile(src): @@ -784,7 +976,7 @@ sto.write(f"############names={rnames} rids={rids}\n") sto.write(f"############names={repos}\n") tfcat = "ToolFactory generated tools" - if self.args.tool_name not in rnames: + if self.tool_name not in rnames: tscat = ts.categories.get_categories() cnames = [x.get("name", "?").strip() for x in tscat] cids = [x.get("id", "?") for x in tscat] @@ -804,7 +996,7 @@ tid = res.get("id", None) sto.write(f"##########create res={res}\n") else: - i = rnames.index(self.args.tool_name) + i = rnames.index(self.tool_name) tid = rids[i] try: res = ts.repositories.update_repository( @@ -829,7 +1021,7 @@ "-a", self.args.galaxy_api_key, "--name", - self.args.tool_name, + self.tool_name, "--owner", "fubar", "--toolshed", @@ -838,9 +1030,9 @@ "ToolFactory", ] tout.write("running\n%s\n" % " ".join(cll)) - subp = subprocess.run(cll, shell=False, stderr=tout, stdout=tout) + subp = subprocess.run(cll, env=self.ourenv, cwd=self.ourcwd, shell=False, stderr=tout, stdout=tout) tout.write( - "installed %s - got retcode %d\n" % (self.args.tool_name, subp.returncode) + "installed %s - got retcode %d\n" % (self.tool_name, subp.returncode) ) tout.close() return subp.returncode @@ -870,7 +1062,7 @@ rnames = [x.get("name", "?") for x in repos] rids = [x.get("id", "?") for x in repos] #cat = "ToolFactory generated tools" - if self.args.tool_name not in rnames: + if self.tool_name not in rnames: cll = [ "planemo", "shed_create", @@ -879,20 +1071,20 @@ "--owner", "fubar", "--name", - self.args.tool_name, + self.tool_name, "--shed_key", self.args.toolshed_api_key, ] try: subp = subprocess.run( - cll, shell=False, cwd=self.tooloutdir, stdout=tout, stderr=tout + cll, env=self.ourenv, shell=False, cwd=self.tooloutdir, stdout=tout, stderr=tout ) except: pass if subp.returncode != 0: - tout.write("Repository %s exists\n" % self.args.tool_name) + tout.write("Repository %s exists\n" % self.tool_name) else: - tout.write("initiated %s\n" % self.args.tool_name) + tout.write("initiated %s\n" % self.tool_name) cll = [ "planemo", "shed_upload", @@ -901,13 +1093,13 @@ "--owner", "fubar", "--name", - self.args.tool_name, + self.tool_name, "--shed_key", self.args.toolshed_api_key, "--tar", self.newtarpath, ] - subp = subprocess.run(cll, shell=False, stdout=tout, stderr=tout) + subp = subprocess.run(cll, env=self.ourenv, cwd=self.ourcwd, shell=False, stdout=tout, stderr=tout) tout.write("Ran %s got %d\n" % (" ".join(cll),subp.returncode)) tout.close() return subp.returncode @@ -927,14 +1119,14 @@ "-a", self.args.galaxy_api_key, "--name", - self.args.tool_name, + self.tool_name, "--owner", "fubar", ] if genoutputs: dummy, tfile = tempfile.mkstemp() subp = subprocess.run( - cll, shell=False, stderr=dummy, stdout=dummy + cll, env=self.ourenv, cwd=self.ourcwd, shell=False, stderr=dummy, stdout=dummy ) with open('tool_test_output.json','rb') as f: @@ -964,7 +1156,7 @@ shutil.copyfile(src, dest) else: subp = subprocess.run( - cll, shell=False, stderr=tout, stdout=tout) + cll, env=self.ourenv, cwd=self.ourcwd, shell=False, stderr=tout, stdout=tout) tout.write("eph_test Ran %s got %d" % (" ".join(cll), subp.returncode)) tout.close() return subp.returncode @@ -1007,6 +1199,7 @@ ] subp = subprocess.run( cll, + env=self.ourenv, shell=False, cwd=self.tooloutdir, stderr=dummy, @@ -1026,7 +1219,7 @@ xreal, ] subp = subprocess.run( - cll, shell=False, cwd=self.tooloutdir, stderr=tout, stdout=tout + cll, env=self.ourenv, shell=False, cwd=self.tooloutdir, stderr=tout, stdout=tout ) tout.close() return subp.returncode @@ -1068,6 +1261,7 @@ ] subp = subprocess.run( cll, + env=self.ourenv, shell=False, cwd=self.testdir, stderr=dummy, @@ -1085,7 +1279,7 @@ xreal, ] subp = subprocess.run( - cll, shell=False, cwd=self.testdir, stderr=tout, stdout=tout + cll, env=self.ourenv, shell=False, cwd=self.testdir, stderr=tout, stdout=tout ) tout.close() return subp.returncode @@ -1124,13 +1318,13 @@ def makeToolTar(self): """move outputs into test-data and prepare the tarball""" - excludeme = "tool_test_output" + excludeme = "_planemo_test_report.html" def exclude_function(tarinfo): filename = tarinfo.name return ( None - if filename.startswith(excludeme) + if filename.endswith(excludeme) else tarinfo ) @@ -1171,7 +1365,7 @@ shutil.copyfile(src, dest) with os.scandir(self.testdir) as outs: for entry in outs: - if not entry.is_file(): + if (not entry.is_file()) or entry.name.endswith('_sample') or entry.name.endswith("_planemo_test_report.html"): continue if "." in entry.name: nayme, ext = os.path.splitext(entry.name) @@ -1223,6 +1417,7 @@ a("--toolshed_api_key", default="fakekey") a("--galaxy_api_key", default="fakekey") a("--galaxy_root", default="/galaxy-central") + a("--galaxy_venv", default="/galaxy_venv") args = parser.parse_args() assert not args.bad_user, ( 'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy admin adds %s to "admin_users" in the Galaxy configuration file' @@ -1240,23 +1435,16 @@ r.writeShedyml() r.makeTool() if args.make_Tool == "generate": - retcode = r.run() + retcode = r.run() # for testing toolfactory itself r.moveRunOutputs() r.makeToolTar() else: - r.makeToolTar() - r.planemo_shedLoad() - r.shedLoad() - r.eph_galaxy_load() - retcode = r.gal_tool_test() # writes outputs - r.makeToolTar() - r.planemo_shedLoad() - r.shedLoad() - r.eph_galaxy_load() - retcode = r.gal_test() + r.planemo_biodocker_test() # test to make outputs and then test r.moveRunOutputs() r.makeToolTar() - print(f"second galaxy_test returned {retcode}") + if args.make_Tool == "gentestinstall": + r.shedLoad() + r.eph_galaxy_load() if __name__ == "__main__": diff -r c749364c2283 -r 557d5f06f213 toolfactory/rgToolFactory2.xml --- a/toolfactory/rgToolFactory2.xml Mon Nov 23 03:12:37 2020 +0000 +++ b/toolfactory/rgToolFactory2.xml Thu Nov 26 06:17:12 2020 +0000 @@ -135,7 +135,8 @@ galaxyxml bioblend ephemeris - planemo + docker-py + quay.io/fubar2/planemo-biocontainer:latest