# HG changeset patch
# User fubar
# Date 1619748632 0
# Node ID efefe43f23c8f3c861822ae277b917a128f571db
# Parent  e2c8c2fa192d5ab5c5c54a1e941a991cb22e7389
Uploaded

diff -r e2c8c2fa192d -r efefe43f23c8 toolfactory/ToolFactory.py
--- a/toolfactory/ToolFactory.py	Tue Apr 27 23:33:49 2021 +0000
+++ b/toolfactory/ToolFactory.py	Fri Apr 30 02:10:32 2021 +0000
@@ -28,6 +28,7 @@
 import tarfile
 import tempfile
 import time
+import urllib
 
 from bioblend import ConnectionError
 from bioblend import galaxy
@@ -70,13 +71,14 @@
             citation_tuples.append(("bibtex", citation[len("bibtex") :].strip()))
     return citation_tuples
 
-class ToolConfUpdater():
+
+class Tool_Conf_Updater():
     # update config/tool_conf.xml with a new tool unpacked in /tools
     # requires highly insecure docker settings - like write to tool_conf.xml and to tools !
     # if in a container possibly not so courageous.
     # Fine on your own laptop but security red flag for most production instances
 
-  def __init__(self, args, tool_conf_path, new_tool_archive_path, new_tool_name, tool_dir):
+    def __init__(self, args, tool_conf_path, new_tool_archive_path, new_tool_name, tool_dir):
         self.args = args
         self.tool_conf_path = os.path.join(args.galaxy_root,tool_conf_path)
         self.tool_dir = os.path.join(args.galaxy_root, tool_dir)
@@ -138,7 +140,7 @@
         if False and self.args.packages and self.args.packages > '':
             self.install_deps()
 
-class ScriptRunner:
+class Tool_Factory:
     """Wrapper for an arbitrary script
     uses galaxyxml
 
@@ -194,9 +196,7 @@
             )
         self.args = args
         self.cleanuppar()
-        self.lastclredirect = None
         self.lastxclredirect = None
-        self.cl = []
         self.xmlcl = []
         self.is_positional = self.args.parampass == "positional"
         if self.args.sysexe:
@@ -209,7 +209,6 @@
                 self.executeme = [self.args.packages.split(",")[0].split(":")[0].strip(), ]
             else:
                 self.executeme = None
-        aCL = self.cl.append
         aXCL = self.xmlcl.append
         assert args.parampass in [
             "0",
@@ -252,13 +251,10 @@
             self.test_override = None
         if self.args.script_path:
             for ex in self.executeme:
-                aCL(ex)
                 aXCL(ex)
-            aCL(self.sfile)
             aXCL("$runme")
         else:
             for ex in self.executeme:
-                aCL(ex)
                 aXCL(ex)
 
         if self.args.parampass == "0":
@@ -273,35 +269,23 @@
 
     def clsimple(self):
         """no parameters or repeats - uses < and > for i/o"""
-        aCL = self.cl.append
         aXCL = self.xmlcl.append
         if len(self.infiles) > 0:
-            aCL("<")
-            aCL(self.infiles[0]["infilename"])
             aXCL("<")
             aXCL("$%s" % self.infiles[0]["infilename"])
         if len(self.outfiles) > 0:
-            aCL(">")
-            aCL(self.outfiles[0]["name"])
             aXCL(">")
             aXCL("$%s" % self.outfiles[0]["name"])
         if self.args.cl_user_suffix:  # DIY CL end
             clp = shlex.split(self.args.cl_user_suffix)
             for c in clp:
-                aCL(c)
                 aXCL(c)
 
     def prepargp(self):
-        clsuffix = []
         xclsuffix = []
         for i, p in enumerate(self.infiles):
             nam = p["infilename"]
             if p["origCL"].strip().upper() == "STDIN":
-                appendme = [
-                    nam,
-                    nam,
-                    "< %s" % nam,
-                ]
                 xappendme = [
                     nam,
                     nam,
@@ -312,16 +296,12 @@
                 over = ""
                 if rep:
                     over = f'#for $rep in $R_{nam}:\n--{nam} "$rep.{nam}"\n#end for'
-                appendme = [p["CL"], p["CL"], ""]
                 xappendme = [p["CL"], "$%s" % p["CL"], over]
-            clsuffix.append(appendme)
             xclsuffix.append(xappendme)
         for i, p in enumerate(self.outfiles):
             if p["origCL"].strip().upper() == "STDOUT":
-                self.lastclredirect = [">", p["name"]]
                 self.lastxclredirect = [">", "$%s" % p["name"]]
             else:
-                clsuffix.append([p["name"], p["name"], ""])
                 xclsuffix.append([p["name"], "$%s" % p["name"], ""])
         for p in self.addpar:
             nam = p["name"]
@@ -330,40 +310,27 @@
                 over = f'#for $rep in $R_{nam}:\n--{nam} "$rep.{nam}"\n#end for'
             else:
                 over = p["override"]
-            clsuffix.append([p["CL"], nam, over])
             xclsuffix.append([p["CL"], '"$%s"' % nam, over])
         for p in self.selpar:
-            clsuffix.append([p["CL"], p["name"], p["override"]])
             xclsuffix.append([p["CL"], '"$%s"' % p["name"], p["override"]])
         self.xclsuffix = xclsuffix
-        self.clsuffix = clsuffix
 
     def prepclpos(self):
-        clsuffix = []
         xclsuffix = []
         for i, p in enumerate(self.infiles):
             if p["origCL"].strip().upper() == "STDIN":
-                appendme = [
-                    "999",
-                    p["infilename"],
-                    "< $%s" % p["infilename"],
-                ]
                 xappendme = [
                     "999",
                     p["infilename"],
                     "< $%s" % p["infilename"],
                 ]
             else:
-                appendme = [p["CL"], p["infilename"], ""]
                 xappendme = [p["CL"], "$%s" % p["infilename"], ""]
-            clsuffix.append(appendme)
             xclsuffix.append(xappendme)
         for i, p in enumerate(self.outfiles):
             if p["origCL"].strip().upper() == "STDOUT":
-                self.lastclredirect = [">", p["name"]]
                 self.lastxclredirect = [">", "$%s" % p["name"]]
             else:
-                clsuffix.append([p["CL"], p["name"], ""])
                 xclsuffix.append([p["CL"], "$%s" % p["name"], ""])
         for p in self.addpar:
             nam = p["name"]
@@ -371,15 +338,11 @@
             if rep:
                 print(f'### warning. Repeats for {nam} ignored - not permitted in positional parameter command lines!')
             over = p["override"]
-            clsuffix.append([p["CL"], nam, over])
             xclsuffix.append([p["CL"], '"$%s"' % nam, over])
         for p in self.selpar:
-            clsuffix.append([p["CL"], p["name"], p["override"]])
             xclsuffix.append([p["CL"], '"$%s"' % p["name"], p["override"]])
-        clsuffix.sort()
         xclsuffix.sort()
         self.xclsuffix = xclsuffix
-        self.clsuffix = clsuffix
 
     def prepScript(self):
         rx = open(self.args.script_path, "r").readlines()
@@ -443,12 +406,6 @@
 
     def clpositional(self):
         # inputs in order then params
-        aCL = self.cl.append
-        for (k, v, koverride) in self.clsuffix:
-            if " " in v:
-                aCL("%s" % v)
-            else:
-                aCL(v)
         aXCL = self.xmlcl.append
         for (k, v, koverride) in self.xclsuffix:
             aXCL(v)
@@ -458,13 +415,11 @@
         if self.args.cl_user_suffix:  # DIY CL end
             clp = shlex.split(self.args.cl_user_suffix)
             for c in clp:
-                aCL(c)
                 aXCL(c)
 
 
     def clargparse(self):
         """argparse style"""
-        aCL = self.cl.append
         aXCL = self.xmlcl.append
         # inputs then params in argparse named form
 
@@ -479,22 +434,12 @@
                     k = "--%s" % k
                 aXCL(k)
                 aXCL(v)
-        for (k, v, koverride) in self.clsuffix:
-            if koverride > "":
-                k = koverride
-            elif len(k.strip()) == 1:
-                k = "-%s" % k
-            else:
-                k = "--%s" % k
-            aCL(k)
-            aCL(v)
         if self.lastxclredirect:
             aXCL(self.lastxclredirect[0])
             aXCL(self.lastxclredirect[1])
         if self.args.cl_user_suffix:  # DIY CL end
             clp = shlex.split(self.args.cl_user_suffix)
             for c in clp:
-                aCL(c)
                 aXCL(c)
 
     def getNdash(self, newname):
@@ -857,80 +802,6 @@
         xf.close()
         # ready for the tarball
 
-    def run(self):  #noqa
-        """
-        generate test outputs by running a command line
-        won't work if command or test override in play - planemo is the
-        easiest way to generate test outputs for that case so is
-        automagically selected
-        """
-        scl = " ".join(self.cl)
-        err = None
-        logname = f"{self.tool_name}_runner_log"
-        if self.args.parampass != "0":
-            if self.lastclredirect:
-                logf = open(self.lastclredirect[1], "wb")  # is name of an output file
-            else:
-                logf = open(logname,'w')
-                logf.write("No dependencies so sending CL = '%s' to the fast direct runner instead of planemo to generate tests" % scl)
-            subp = subprocess.run(
-                self.cl, shell=False, stdout=logf, stderr=logf
-            )
-            logf.close()
-            retval = subp.returncode
-        else:  # work around special case - stdin and write to stdout
-            if len(self.infiles) > 0:
-                sti = open(self.infiles[0]["name"], "rb")
-            else:
-                sti = sys.stdin
-            if len(self.outfiles) > 0:
-                sto = open(self.outfiles[0]["name"], "wb")
-            else:
-                sto = sys.stdout
-            subp = subprocess.run(
-                self.cl, shell=False, stdout=sto, stdin=sti
-            )
-            retval = subp.returncode
-            sto.close()
-            sti.close()
-        if retval != 0 and err:  # problem
-            sys.stderr.write(err)
-        for p in self.outfiles:
-            oname = p["name"]
-            tdest = os.path.join(self.testdir, "%s_sample" % oname)
-            if not os.path.isfile(tdest):
-                if os.path.isfile(oname):
-                    shutil.copyfile(oname, tdest)
-                    dest = os.path.join(self.repdir, "%s.sample.%s" % (oname,p['format']))
-                    shutil.copyfile(oname, dest)
-                else:
-                    if report_fail:
-                        tout.write(
-                            "###Tool may have failed - output file %s not found in testdir after planemo run %s."
-                            % (oname, self.testdir)
-                        )
-        for p in self.infiles:
-            pth = p["name"]
-            dest = os.path.join(self.testdir, "%s_sample" % p["infilename"])
-            shutil.copyfile(pth, dest)
-            dest = os.path.join(self.repdir, "%s_sample.%s" % (p["infilename"],p["format"]))
-            shutil.copyfile(pth, dest)
-        with os.scandir('.') as outs:
-            for entry in outs:
-                newname = entry.name
-                if not entry.is_file() or entry.name.endswith('_sample'):
-                    continue
-                if not (entry.name.endswith('.html') or entry.name.endswith('.gz') or entry.name.endswith(".tgz")):
-                    fname, ext = os.path.splitext(entry.name)
-                    if len(ext) > 1:
-                        newname = f"{fname}_{ext[1:]}.txt"
-                    else:
-                        newname = f"{fname}.txt"
-                dest = os.path.join(self.repdir, newname)
-                src = entry.name
-                shutil.copyfile(src, dest)
-        return retval
-
     def writeShedyml(self):
         """for planemo"""
         yuser = self.args.user_email.split("@")[0]
@@ -1068,6 +939,7 @@
     a("--galaxy_venv", default="/galaxy_venv")
     a("--collection", action="append", default=[])
     a("--include_tests", default=False, action="store_true")
+    a("--admin_only", default=False, action="store_true")
     a("--install", default=False, action="store_true")
     a("--run_test", default=False, action="store_true")
     a("--local_tools", default='tools') # relative to $__root_dir__
@@ -1079,30 +951,20 @@
     a("--toolshed_api_key", default="fakekey")
     a("--galaxy_api_key", default="8993d65865e6d6d1773c2c34a1cc207d")
     args = parser.parse_args()
-    assert not args.bad_user, (
-        'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy \
+    if args.admin_only:
+        assert not args.bad_user, (
+          'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy \
 admin adds %s to "admin_users" in the galaxy.yml Galaxy configuration file'
-        % (args.bad_user, args.bad_user)
+           % (args.bad_user, args.bad_user)
     )
     assert args.tool_name, "## Tool Factory expects a tool name - eg --tool_name=DESeq"
-    assert (
-        args.sysexe or args.packages
-    ), "## Tool Factory wrapper expects an interpreter \
-or an executable package in --sysexe or --packages"
-    print('Hello from',os.getcwd())
-    r = ScriptRunner(args)
+    r = Tool_Factory(args)
     r.writeShedyml()
     r.makeTool()
     r.makeToolTar()
-    if args.run_test:
-        if not args.packages or args.packages.strip() == "bash":
-            r.run()
-            r.makeToolTar()
-        else:
-            tt = ToolTester(report_dir=r.repdir, in_tool_archive=r.newtarpath, new_tool_archive=r.args.new_tool, galaxy_root=args.galaxy_root, include_tests=False)
     if args.install:
         #try:
-        tcu = ToolConfUpdater(args=args, tool_dir=args.local_tools,
+        tcu = Tool_Conf_Updater(args=args, tool_dir=args.local_tools,
         new_tool_archive_path=r.newtarpath, tool_conf_path=args.tool_conf_path,
         new_tool_name=r.tool_name)
         #except Exception:
diff -r e2c8c2fa192d -r efefe43f23c8 toolfactory/ToolFactory.xml
--- a/toolfactory/ToolFactory.xml	Tue Apr 27 23:33:49 2021 +0000
+++ b/toolfactory/ToolFactory.xml	Fri Apr 30 02:10:32 2021 +0000
@@ -28,24 +28,16 @@
           </repeat>
       </xml>
      <xml name="tool_metadata">
-         <param name="tool_version" label="Tool Version - bump this to warn users trying to redo old analyses" type="text" value="0.01"
+        <param name="tool_version" label="Tool Version - bump this to warn users trying to redo old analyses" type="text" value="0.01"
             help="If you change your script and regenerate the 'same' tool, you should inform Galaxy (and users) by changing (bumping is traditional) this number"/>
-            <param name="tool_desc" label="Tool Synopsis" type="text" value=""
-             help="Supply a brief tool description for the Galaxy tool menu entry" />
-             <param name="install" label="Attempt to install in the host Galaxy"
-                help="This will fail unless running in a very unusual configuration such as a specialised Docker container"
-                type="boolean" checked="True" truevalue="1"  falsevalue="0" />
-         <conditional name="do_test">
-            <param name="run_test" label="Run planemo test on the new archive to add test outputs making a proper tool - takes time"
-                help="Archives must be run and updated using planemo to populate all the test outputs. Will be very quick if no dependencies (eg a bash script). May take a very long time depending on dependencies"
-                type="boolean" checked="False" truevalue="1"  falsevalue="0" />
-                <when value="0">
-                    <param name="tail" type="hidden" value="not_tested"/>
-                </when>
-                <when value="1">
-                    <param name="tail" type="hidden" value="tested"/>
-                </when>
-            </conditional>
+        <param name="tool_desc" label="Tool Synopsis" type="text" value=""
+            help="Supply a brief tool description for the Galaxy tool menu entry" />
+        <param name="install" label="Attempt to install in the host Galaxy"
+            help="This will fail unless running in a very unusual configuration such as a specialised Docker container"
+            type="boolean" checked="False" truevalue="1"  falsevalue="0" />
+         <param name="run_test" label="Finalise new archive with test outputs. Runs externally. Outputs will appear in history when ready"
+                    help="Archives must be run and updated using planemo to populate all the test outputs. May take a long time depending on dependencies"
+                    type="boolean" checked="True" truevalue="1"  falsevalue="0" />
             <param name="help_text" label="Tool form documentation and help text for users" type="text" area="true"
             value="**What it Does**"
              help="Supply user documentation to appear on the new tool form as reStructured text - http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html" >
@@ -232,7 +224,7 @@
     #if $install:
        --install
     #end if
-    #if $do_test.run_test:
+    #if $run_test:
        --run_test
     #end if
    #if $io_param.ppass.parampass != '0':
@@ -405,11 +397,7 @@
     <expand macro="tool_metadata" />
   </inputs>
   <outputs>
-    <data format="toolshed.gz" name="new_tool" label="${tool_name}_${do_test.tail}_toolshed.gz" />
-   <collection name="TF_run_report" type="list" label="${tool_name} test run outputs">
-      <filter>do_test['run_test']</filter>
-      <discover_datasets pattern="__name_and_ext__" directory="TF_run_report" />
-  </collection>
+    <data format="toolshed.gz" name="new_tool" label="${tool_name}_not_tested.toolshed.gz" />
   </outputs>
 <tests>
 <test>
@@ -432,7 +420,6 @@
     <param name="choosescript" value="yes" />
     <param name="script_path" value="$runme"/>
     <param name="install" value="0"/>
-    <param name="run_test" value="1"/>
     <output name="new_tool" file="toolfactory_pyrevpos_tgz_sample" compare="sim_size" delta="6000" />
 </test>
 </tests>
diff -r e2c8c2fa192d -r efefe43f23c8 toolfactory/ToolFactory_tester.xml
--- a/toolfactory/ToolFactory_tester.xml	Tue Apr 27 23:33:49 2021 +0000
+++ b/toolfactory/ToolFactory_tester.xml	Fri Apr 30 02:10:32 2021 +0000
@@ -10,6 +10,7 @@
   </stdio>
   <version_command><![CDATA[echo "1"]]></version_command>
   <command><![CDATA[
+mkdir 'TF_run_report';
 python
 $runme
 --in_tool_archive
@@ -17,9 +18,9 @@
 --new_tested_tool_archive
 $new_tested_tool_archive
 --galaxy_root
-"$galaxyroot"
+"$__root_dir__"
 >
-$tf_archive_tester_log;
+"TF_run_report/${in_tool_archive.name}_test_log.txt"
 ]]></command>
   <configfiles>
     <configfile name="runme"><![CDATA[#raw
@@ -61,9 +62,6 @@
     return time.strftime("%d/%m/%Y %H:%M:%S", time.localtime(time.time()))
 
 class ToolTester():
-    # requires highly insecure docker settings - like write to tool_conf.xml and to tools !
-    # if in a container possibly not so courageous.
-    # Fine on your own laptop but security red flag for most production instances
     # uncompress passed tar, run planemo and rebuild a new tarball with tests
 
     def __init__(self, args=None, in_tool_archive='/galaxy-central/tools/newtool/newtool_toolshed.gz', new_tool_archive=None):
@@ -97,8 +95,10 @@
 
     def call_planemo(self,xmlpath,ourdir):
         penv = os.environ
-        #penv['HOME'] = os.path.join(self.args.galaxy_root,'planemo')
-        #penv["GALAXY_VIRTUAL_ENV"] = os.path.join(penv['HOME'],'.planemo','gx_venv_3.9')
+        penv['HOME'] = os.path.join(self.args.galaxy_root,'planemo')
+        newpath = f"{penv['HOME']}:{penv['PATH']}"
+        penv['PATH'] = newpath
+        penv["GALAXY_VIRTUAL_ENV"] = os.path.join(self.args.galaxy_root,'.venv')
         penv["PIP_CACHE_DIR"] = os.path.join(self.args.galaxy_root,'pipcache')
         toolfile = os.path.split(xmlpath)[1]
         tool_name = self.tool_name
@@ -106,9 +106,11 @@
         cll = [
             "planemo",
             "test",
-            "--biocontainers",
+            "--no_cleanup",
+            "--test_data",
+            os.path.abspath(ourdir),
             "--test_output",
-            os.path.abspath(tool_test_output),
+            os.path.abspath(self.tool_test_output),
             "--galaxy_root",
             self.args.galaxy_root,
             "--update_test_data",
@@ -119,6 +121,7 @@
             cll,
             #capture_output=True,
             encoding='utf8',
+            cwd = os.path.abspath(self.tool_name),
             env = penv,
             shell=False,
         )
@@ -130,12 +133,6 @@
             x = os.path.split(xreal)[1]
             xout = os.path.join(self.tooloutdir,x)
             shutil.copyfile(xreal, xout)
-        # for p in self.infiles:
-            # pth = p["name"]
-            # dest = os.path.join(self.testdir, "%s_sample" % p["infilename"])
-            # shutil.copyfile(pth, dest)
-            # dest = os.path.join(self.repdir, "%s_sample" % p["infilename"])
-            # shutil.copyfile(pth, dest)
 
     def makeToolTar(self):
         """move outputs into test-data and prepare the tarball"""
@@ -174,7 +171,7 @@
                 shutil.copyfile(src, dest)
         with os.scandir('.') as outs:
             for entry in outs:
-                if not entry.is_file():
+                if not entry.is_file() or entry.name == "conda_activate.log":
                     continue
                 if "." in entry.name:
                     _, ext = os.path.splitext(entry.name)
@@ -212,8 +209,6 @@
     def update_tests(self,ourdir):
         for xmlf in self.ourxmls:
             capture = self.call_planemo(xmlf,ourdir)
-            #sys.stderr.write('%s, stdout=%s, stderr=%s' % (xmlf, capture.stdout, capture.stdout))
-            #print('%s, stdout=%s, stderr=%s' % (capture.stdout, capture.stdout,xmlf))
 
 def main():
     """
@@ -239,11 +234,9 @@
   <inputs>
     <param name="new_tool_name" value="" type="hidden"/>
     <param name="in_tool_archive" type="data" optional="false" label="Select a no_test tarfile to test and update for a toolshed" help="" format="toolshed.gz" multiple="false"/>
-    <param name="galaxyroot" type="text" value="/home/ross/gal21" label="Galaxy root for planemo to use - MUST be made available in the Galaxy job runner configuration" help=""/>
   </inputs>
   <outputs>
     <data name="new_tested_tool_archive" format="toolshed.gz" label="${in_tool_archive.name.split('_')[0]}_tested_toolshed.gz"  hidden="false"/>
-    <data name="tf_archive_tester_log" format="txt" label="${in_tool_archive.name}_test_log" hidden="false"/>
     <collection name="TF_run_report" type="list" label="${in_tool_archive.name} test Run reports">
       <discover_datasets pattern="__name_and_ext__" directory="TF_run_report" visible="false"/>
     </collection>
@@ -251,9 +244,7 @@
   <tests>
     <test>
       <output name="new_tested_tool_archive" value="new_tested_tool_archive_sample" compare="sim_size" delta_frac="0.5"/>
-      <output name="tf_archive_tester_log" value="tf_archive_tester_log_sample" compare="sim_size" delta_frac="0.1"/>
       <param name="in_tool_archive" value="in_tool_archive_sample"/>
-      <param name="galaxyroot" value="/home/ross/gal21"/>
       <output_collection name="TF_run_report"/>
     </test>
   </tests>
@@ -265,6 +256,15 @@
 
 Script::
 
+    # see https://github.com/fubar2/toolfactory
+    #
+    # copyright ross lazarus (ross stop lazarus at gmail stop com) May 2012
+    #
+    # all rights reserved
+    # Licensed under the LGPL
+    # suggestions for improvement and bug fixes welcome at
+    # https://github.com/fubar2/toolfactory
+
     import argparse
     import copy
     import os
@@ -275,19 +275,20 @@
     import tempfile
     import time
     import xml.etree.ElementTree as ET
+
+
     myversion = "V2.2 April 2021"
     verbose = True
     debug = True
     toolFactoryURL = "https://github.com/fubar2/toolfactory"
+
     def timenow():
         """return current time as a string"""
         return time.strftime("%d/%m/%Y %H:%M:%S", time.localtime(time.time()))
 
     class ToolTester():
-        # requires highly insecure docker settings - like write to tool_conf.xml and to tools !
-        # if in a container possibly not so courageous.
-        # Fine on your own laptop but security red flag for most production instances
         # uncompress passed tar, run planemo and rebuild a new tarball with tests
+
         def __init__(self, args=None, in_tool_archive='/galaxy-central/tools/newtool/newtool_toolshed.gz', new_tool_archive=None):
             self.args = args
             self.new_tool_archive = new_tool_archive
@@ -298,7 +299,6 @@
             ourdir = os.path.commonpath(flist) # eg pyrevpos
             self.tool_name = ourdir
             ourxmls = [x for x in flist if x.lower().endswith('.xml') and os.path.split(x)[0] == ourdir]
-            # planemo_test/planemo_test.xml
             assert len(ourxmls) > 0
             self.ourxmls = ourxmls # [os.path.join(tool_path,x) for x in ourxmls]
             res = tff.extractall()
@@ -319,13 +319,14 @@
 
         def call_planemo(self,xmlpath,ourdir):
             penv = os.environ
-            penv['HOME'] = '/home/ross/galaxy-release_21.01'
+            penv["PIP_CACHE_DIR"] = os.path.join(self.args.galaxy_root,'pipcache')
             toolfile = os.path.split(xmlpath)[1]
             tool_name = self.tool_name
             tool_test_output = f"{tool_name}_planemo_test_report.html"
             cll = [
                 "planemo",
                 "test",
+                "--biocontainers",
                 "--test_output",
                 os.path.abspath(tool_test_output),
                 "--galaxy_root",
@@ -336,7 +337,7 @@
             print(cll)
             p = subprocess.run(
                 cll,
-                capture_output=True,
+                #capture_output=True,
                 encoding='utf8',
                 env = penv,
                 shell=False,
@@ -349,22 +350,18 @@
                 x = os.path.split(xreal)[1]
                 xout = os.path.join(self.tooloutdir,x)
                 shutil.copyfile(xreal, xout)
-            # for p in self.infiles:
-                # pth = p["name"]
-                # dest = os.path.join(self.testdir, "%s_sample" % p["infilename"])
-                # shutil.copyfile(pth, dest)
-                # dest = os.path.join(self.repdir, "%s_sample" % p["infilename"])
-                # shutil.copyfile(pth, dest)
 
         def makeToolTar(self):
             """move outputs into test-data and prepare the tarball"""
             excludeme = "_planemo_test_report.html"
+
             def exclude_function(tarinfo):
                 filename = tarinfo.name
                 return None if filename.endswith(excludeme) else tarinfo
+
             newtar = 'new_%s_toolshed.gz' % self.tool_name
             ttf = tarfile.open(newtar, "w:gz")
-            ttf.add(name=self.tooloutdir,
+            ttf.add(name=self.tool_name,
                 arcname=self.tool_name,
                 filter=exclude_function)
             ttf.close()
@@ -425,16 +422,16 @@
                         src = os.path.join(self.testdir, entry.name)
                         shutil.copyfile(src, dest)
 
+
         def update_tests(self,ourdir):
             for xmlf in self.ourxmls:
                 capture = self.call_planemo(xmlf,ourdir)
-                #sys.stderr.write('%s, stdout=%s, stderr=%s' % (xmlf, capture.stdout, capture.stdout))
-                print('%s, stdout=%s, stderr=%s' % (capture.stdout, capture.stdout,xmlf))
 
     def main():
         """
         This is a Galaxy wrapper.
         It expects to be called by a special purpose tool.xml
+
         """
         parser = argparse.ArgumentParser()
         a = parser.add_argument
@@ -444,9 +441,11 @@
         args = parser.parse_args()
         print('Hello from',os.getcwd())
         tt = ToolTester(args=args, in_tool_archive=args.in_tool_archive, new_tool_archive=args.new_tested_tool_archive)
+
     if __name__ == "__main__":
         main()
 
+
 ]]></help>
   <citations>
     <citation type="doi">10.1093/bioinformatics/bts573</citation>
diff -r e2c8c2fa192d -r efefe43f23c8 toolfactory/install-history.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/toolfactory/install-history.py	Fri Apr 30 02:10:32 2021 +0000
@@ -0,0 +1,41 @@
+import argparse
+import os
+
+
+from bioblend import galaxy
+
+
+def _parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-g", "--galaxy", help='URL of target galaxy')
+    parser.add_argument("-p", "--password", help='Galaxy admin password')
+    parser.add_argument("-e", "--email", help='Galaxy admin email')
+    parser.add_argument("-a", "--key", help='Galaxy admin key', default=None)
+    parser.add_argument("-i", "--history_path", help='Path to history gz files to be loaded')
+    return parser
+
+def main():
+    """
+    load a folder of histories or a single gz
+    """
+    args = _parser().parse_args()
+    if args.key:
+        gi = galaxy.GalaxyInstance(url=args.galaxy, key=args.key)
+    else:
+        gi = galaxy.GalaxyInstance(url=args.galaxy, email=args.email, password=args.password)
+    hdir = args.history_path
+    # h = gi.histories.get_most_recently_used_history()
+    if os.path.isdir(hdir):
+        for fp in os.listdir(hdir):
+            hp = os.path.join(hdir,fp)
+            if os.path.isfile(hp):
+                x = gi.histories.import_history(file_path=hp, url=None)
+                print('installed ',hp,'res=',x)
+    else:
+        x = gi.histories.import_history(file_path=hdir, url=None)
+        print('installed',hdir,'res=',x)
+
+
+if __name__ == "__main__":
+    main()
+
diff -r e2c8c2fa192d -r efefe43f23c8 toolfactory/install_tf_demos.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/toolfactory/install_tf_demos.py	Fri Apr 30 02:10:32 2021 +0000
@@ -0,0 +1,42 @@
+import argparse
+import urllib.request
+
+from bioblend import galaxy
+
+WF = "https://drive.google.com/uc?export=download&id=13xE8o7tucHGNA0qYkEP98FfUGl2wdOU5"
+HIST = (
+    "https://zenodo.org/record/4686436/files/TFdemo_wf_april13_planemo.ga?download=1"
+)
+WF_FILE = "tf_workflow.ga"
+HIST_FILE = "tf_history.tgz"
+
+
+def _parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-g", "--galaxy", help="URL of target galaxy", default="http://localhost:8080"
+    )
+    parser.add_argument("-a", "--key", help="Galaxy admin key", default="8993d65865e6d6d1773c2c34a1cc207d")
+    return parser
+
+
+def main():
+    """
+    load the planemo tool_factory demonstration history and tool generating workflow
+    fails in planemo served galaxies because there seems to be no user in trans?
+    """
+    args = _parser().parse_args()
+    urllib.request.urlretrieve(WF, WF_FILE)
+    urllib.request.urlretrieve(HIST, HIST_FILE)
+    assert args.key, "Need an administrative key for the target Galaxy supplied please"
+    gi = galaxy.GalaxyInstance(
+        url=args.galaxy, key=args.key, email="planemo@galaxyproject.org"
+    )
+    x = gi.workflows.import_workflow_from_local_path(WF_FILE, publish=True)
+    print(f"installed {WF_FILE} Returned = {x}\n")
+    x = gi.histories.import_history(file_path=HIST_FILE)
+    print(f"installed {HIST_FILE} Returned = {x}\n")
+
+
+if __name__ == "__main__":
+    main()
diff -r e2c8c2fa192d -r efefe43f23c8 toolfactory/testclass.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/toolfactory/testclass.py	Fri Apr 30 02:10:32 2021 +0000
@@ -0,0 +1,108 @@
+import argparse
+import json
+import os
+import requests
+
+
+from bioblend import galaxy
+
+class ToolTester():
+    # test a newly installed tool using bioblend
+    """
+
+ https://github.com/nsoranzo/bioblend-tutorial/blob/master/historical_exercises/api-scripts.exercises/run_tool.py
+import sys
+import json
+import requests
+import output
+
+BASE_URL = 'http://localhost:8080'
+
+# -----------------------------------------------------------------------------
+def run_tool( tool_id, history_id, **kwargs ):
+    full_url = BASE_URL + '/api/tools'
+
+    #EXERCISE: POST...
+
+# -----------------------------------------------------------------------------
+if __name__ == '__main__':
+    # e.g. ./run_tool.py Filter1 ebfb8f50c6abde6d '{ "input" : { "src": "hda", "id": "77f74776fd03cbc5" }, "cond" : "c6>=100.0" }'
+    # e.g. ./run_tool.py sort1 f597429621d6eb2b '{ "input": { "src": "hda", "id": "b472e2eb553fa0d1" }, "column": "c6", "style": "alpha", "column_set_0|other_column" : "c2", "column_set_0|other_style": "num" }'
+    tool_id, history_id = sys.argv[1:3]
+    params = json.loads( sys.argv[3] ) if len( sys.argv ) >= 4 else {}
+    response = run_tool( tool_id, history_id, **params )
+    output.output_response( response )
+
+
+    def get_testdata(self,urlin,fout):
+       '''
+        grab a test file
+        GET /api/tools/{tool_id}/test_data_download?tool_version={tool_version}&filename={filename}
+        http://localhost:8080/api/tools/tacrev/test_data_download?tool_version=2.00&filename=in
+        '''
+     """
+    def __init__(self, args):
+        self.galaxy = args.galaxy
+        self.key = args.key
+        self.tool_id = args.tool_id
+
+    def run_test(self):
+        """
+    GET /api/tools/{tool_id}/test_data_download?tool_version={tool_version}&filename={filename}
+    http://localhost:8080/api/tools/tacrev/test_data_download?tool_version=2.00&filename=input1
+        """
+        inputs = {}
+        gi = galaxy.GalaxyInstance(url=self.galaxy, key=self.key, verify=False)
+        chistory = gi.histories.get_most_recently_used_history()
+        chistory_id = chistory['id']
+        #contents = gi.histories.show_history(chistory_id, contents=True)
+        #print('####chistory',chistory,'\n#### contents=',contents)
+        #history = gi.histories.create_history(name=f"{self.tool_id}_test_history")
+        #new_hist_id = history['id']
+        fapi = ''.join([self.galaxy, '/api/tools/', self.tool_id, '/build'])
+        build = gi.make_get_request(url=fapi,params={"history_id":chistory_id}).json()
+        fapi = ''.join([self.galaxy, '/api/tools/', self.tool_id, '/test_data'])
+        test_data = requests.get(fapi, params={'key':self.key, 'history_id':chistory_id})# gi.make_get_request(url=fapi,params={"history_id":chistory_id,'key':self.key}).json()
+        print(test_data)
+        testinputs = test_data.json()[0].get('inputs',None)
+        print('testinputs',testinputs)
+        stateinputs = build.get('state_inputs',None) # 'input1': {'values': [{'id': '7b326180327c3fcc', 'src': 'hda'}]}}
+        if testinputs:
+            for k in testinputs.keys():
+                v = testinputs[k]
+                if '|' in k:
+                    nk = k.split('|')[-1]
+                    inputs[nk] = v
+                else:
+                    inputs[k] = v
+        if stateinputs:
+            print('stateinputs',stateinputs)
+            for k in stateinputs.keys():
+                inp = stateinputs[k]
+                if isinstance(inp,dict):
+                    if inp.get('values',None):
+                         for anin in inp['values']:
+                            if anin.get('id', None) and anin.get('src', None):
+                                gi.histories.copy_dataset(chistory_id, anin['id'], source=anin['src'])
+                                print('******copied id', anin['id'])
+                                up = {k:anin}
+                                print(up)
+                                inputs.update(up) # replace the input def
+        print('after state inputs', inputs)
+        fapi = ''.join([self.galaxy, '/api/tools'])
+        r = gi.tools.run_tool(chistory_id, self.tool_id, inputs, input_format='legacy')
+        print(f"Called test on {self.tool_id} - got {r}")
+
+def _parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-g", "--galaxy", help='URL of target galaxy',default="http://localhost:8080")
+    parser.add_argument("-a", "--key", help='Galaxy admin key', default="13073fde17d06591ce36e596e3c29904")
+    parser.add_argument("-t", "--tool_id", help='Tool id to test', default="plotter")
+    return parser
+
+
+if __name__ == "__main__":
+    args = _parser().parse_args()
+    tt = ToolTester(args)
+    tt.run_test()
+
diff -r e2c8c2fa192d -r efefe43f23c8 toolfactory/toolwatcher.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/toolfactory/toolwatcher.py	Fri Apr 30 02:10:32 2021 +0000
@@ -0,0 +1,123 @@
+#!/usr/bin/python
+from datetime import datetime, timedelta
+from io import BytesIO as BIO
+import logging
+import os
+import subprocess
+import tarfile
+import time
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler
+from watchdog.events import PatternMatchingEventHandler
+
+class ToolHandler(PatternMatchingEventHandler):
+
+    def __init__(self, watchme):
+        PatternMatchingEventHandler.__init__(self, patterns=['*.xml'],
+                ignore_directories=False, case_sensitive=False)
+        self.last_modified = datetime.now()
+        self.tool_dir = watchme
+        self.work_dir = os.getcwd()
+        self.galaxy_root = os.path.split(watchme)[0]
+        logging.info(self.galaxy_root)
+        self.tar_dir = os.path.join(self.galaxy_root, 'tooltardir')
+        if not os.path.exists(self.tar_dir):
+                os.mkdir(self.tar_dir)
+
+    def on_created(self, event):
+        self.on_modified(event)
+
+    def on_modified(self, event):
+        if datetime.now() - self.last_modified < timedelta(seconds=1):
+            return
+        else:
+            if os.path.exists(event.src_path):
+                self.last_modified = datetime.now()
+                logging.info(f"{event.src_path} was {event.event_type}")
+                p = self.planemo_test(event.src_path)
+                if p:
+                    if p.returncode == 0:
+                        newtarpath = self.makeToolTar(event.src_path)
+                        logging.info('### Tested toolshed tarball %s written' % newtarpath)
+                    else:
+                        logging.debug('### planemo stdout:')
+                        logging.debug(p.stdout)
+                        logging.debug('### planemo stderr:')
+                        logging.debug(p.stderr)
+                        logging.info('### Planemo call return code =' % p.returncode)
+            else:
+                logging.info('Directory %s deleted' % event.src_path)
+
+    def planemo_test(self, xml_path):
+        toolpath, toolfile = os.path.split(xml_path)
+        dirlist = os.listdir(toolpath)
+        toolname = os.path.basename(toolpath)
+        logging.info('### test dirlist %s, path %s toolname %s' % (dirlist, xml_path, toolname))
+        xmls = [x for x in dirlist if os.path.splitext(x)[1] == '.xml']
+        if not len(xmls) > 0:
+            logging.warning('Found no xml files after change to %s' % xml_path)
+            return None
+        tool_test_output = os.path.join(toolpath, f"{toolname}_planemo_test_report.html")
+        cll = [
+            "planemo",
+            "test",
+            "--test_output",
+            tool_test_output,
+            "--galaxy_root",
+            self.galaxy_root,
+            "--update_test_data",
+            xml_path,
+        ]
+        logging.info('### calling %s' % ' '.join(cll))
+        p = subprocess.run(
+            cll,
+            cwd = toolpath,
+            shell=False,
+            capture_output=True,
+            encoding='utf8',
+        )
+        return p
+
+    def makeToolTar(self, xml_path):
+        """move outputs into test-data and prepare the tarball"""
+        excludeme = "_planemo_test_report.html"
+
+        def exclude_function(tarinfo):
+            filename = tarinfo.name
+            return None if filename.endswith(excludeme) else tarinfo
+
+        tooldir, xml_file = os.path.split(xml_path)
+        os.chdir(self.tool_dir)
+        toolname = os.path.splitext(xml_file)[0]
+        newtarpath = os.path.join(self.tar_dir, '%s_toolshed.gz' % toolname)
+        tf = tarfile.open(newtarpath, "w:gz")
+        tf.add(
+            name=toolname,
+            arcname=toolname,
+            filter=exclude_function,
+        )
+        tf.close()
+        os.chdir(self.work_dir)
+        return newtarpath
+
+
+if __name__ == "__main__":
+    watchme = '/home/ross/gal21/tools'
+    logging.basicConfig(level=logging.INFO,
+                    #filename = os.path.join(watchme,"toolwatcher.log")
+                    #filemode = "w",
+                    format='%(asctime)s - %(message)s',
+                    datefmt='%Y-%m-%d %H:%M:%S')
+    event_handler = ToolHandler(watchme=watchme)
+    observer = Observer()
+    observer.schedule(event_handler, path=watchme, recursive=True)
+    observer.start()
+    try:
+        while True:
+            time.sleep(1)
+    except KeyboardInterrupt:
+        observer.stop()
+    observer.join()
+
+
+