Mercurial > repos > fubar > toolfactory_gtn

comparison toolfactory/ToolFactory.py @ 5:e2c8c2fa192d draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Uploaded
author fubar
date Tue, 27 Apr 2021 23:33:49 +0000
parents 2a46da701dde
children efefe43f23c8
comparison
equal deleted inserted replaced
4:2a46da701dde 5:e2c8c2fa192d
68 citation_tuples.append(("doi", citation[len("doi") :].strip())) 68 citation_tuples.append(("doi", citation[len("doi") :].strip()))
69 else: 69 else:
70 citation_tuples.append(("bibtex", citation[len("bibtex") :].strip())) 70 citation_tuples.append(("bibtex", citation[len("bibtex") :].strip()))
71 return citation_tuples 71 return citation_tuples
72 72
73 class ToolTester():
74 # requires highly insecure docker settings - like write to tool_conf.xml and to tools !
75 # if in a container possibly not so courageous.
76 # Fine on your own laptop but security red flag for most production instances
77 # uncompress passed tar, run planemo and rebuild a new tarball with tests
78
79 def __init__(self, report_dir, in_tool_archive, new_tool_archive, include_tests, galaxy_root):
80 self.new_tool_archive = new_tool_archive
81 self.include_tests = include_tests
82 self.galaxy_root = galaxy_root
83 self.repdir = report_dir
84 assert in_tool_archive and tarfile.is_tarfile(in_tool_archive)
85 # this is not going to go well with arbitrary names. TODO introspect tool xml!
86 tff = tarfile.open(in_tool_archive, "r:*")
87 flist = tff.getnames()
88 ourdir = os.path.commonpath(flist) # eg pyrevpos
89 self.tool_name = ourdir
90 ourxmls = [x for x in flist if x.lower().endswith('.xml') and os.path.split(x)[0] == ourdir]
91 # planemo_test/planemo_test.xml
92 assert len(ourxmls) > 0
93 self.ourxmls = ourxmls # [os.path.join(tool_path,x) for x in ourxmls]
94 res = tff.extractall()
95 tff.close()
96 self.update_tests(ourdir)
97 self.tooloutdir = ourdir
98 self.testdir = os.path.join(self.tooloutdir, "test-data")
99 if not os.path.exists(self.tooloutdir):
100 os.mkdir(self.tooloutdir)
101 if not os.path.exists(self.testdir):
102 os.mkdir(self.testdir)
103 if not os.path.exists(self.repdir):
104 os.mkdir(self.repdir)
105 if not os.path.exists(self.tooloutdir):
106 os.mkdir(self.tooloutdir)
107 if not os.path.exists(self.testdir):
108 os.mkdir(self.testdir)
109 if not os.path.exists(self.repdir):
110 os.mkdir(self.repdir)
111 self.moveRunOutputs()
112 self.makeToolTar()
113
114 def call_planemo(self,xmlpath,ourdir):
115 penv = os.environ
116 penv['HOME'] = os.path.join(self.galaxy_root,'planemo')
117 #penv["GALAXY_VIRTUAL_ENV"] = os.path.join(penv['HOME'],'.planemo','gx_venv_3.9')
118 penv["PIP_CACHE_DIR"] = os.path.join(self.galaxy_root,'pipcache')
119 toolfile = os.path.split(xmlpath)[1]
120 tool_name = self.tool_name
121 tool_test_output = os.path.join(self.repdir, f"{tool_name}_planemo_test_report.html")
122 cll = ["planemo",
123 "test",
124 #"--job_config_file",
125 # os.path.join(self.galaxy_root,"config","job_conf.xml"),
126 #"--galaxy_python_version",
127 #"3.9",
128 "--test_output",
129 os.path.abspath(tool_test_output),
130 "--galaxy_root",
131 self.galaxy_root,
132 "--update_test_data",
133 os.path.abspath(xmlpath),
134 ]
135 print("Call planemo cl =", cll)
136 p = subprocess.run(
137 cll,
138 capture_output=True,
139 encoding='utf8',
140 env = penv,
141 shell=False,
142 )
143 return p
144
145 def makeToolTar(self):
146 """move outputs into test-data and prepare the tarball"""
147 excludeme = "_planemo_test_report.html"
148
149 def exclude_function(tarinfo):
150 filename = tarinfo.name
151 return None if filename.endswith(excludeme) else tarinfo
152
153 newtar = 'new_%s_toolshed.gz' % self.tool_name
154 ttf = tarfile.open(newtar, "w:gz")
155 ttf.add(name=self.tooloutdir,
156 arcname=self.tool_name,
157 filter=exclude_function)
158 ttf.close()
159 shutil.copyfile(newtar, self.new_tool_archive)
160
161 def move_One(self,scandir):
162 with os.scandir('.') as outs:
163 for entry in outs:
164 newname = entry.name
165 if not entry.is_file() or entry.name.endswith('_sample'):
166 continue
167 if not (entry.name.endswith('.html') or entry.name.endswith('.gz') or entry.name.endswith(".tgz")):
168 fname, ext = os.path.splitext(entry.name)
169 if len(ext) > 1:
170 newname = f"{fname}_{ext[1:]}.txt"
171 else:
172 newname = f"{fname}.txt"
173 dest = os.path.join(self.repdir, newname)
174 src = entry.name
175 shutil.copyfile(src, dest)
176
177 def moveRunOutputs(self):
178 """need to move planemo or run outputs into toolfactory collection"""
179 self.move_One(self.tooloutdir)
180 self.move_One('.')
181 if self.include_tests:
182 self.move_One(self.testdir)
183
184 def update_tests(self,ourdir):
185 for xmlf in self.ourxmls:
186 capture = self.call_planemo(xmlf,ourdir)
187 logf = open(f"%s_run_report" % (self.tool_name),'w')
188 logf.write("stdout:")
189 logf.write(capture.stdout)
190 logf.write("stderr:")
191 logf.write(capture.stderr)
192
193
194 class ToolConfUpdater(): 73 class ToolConfUpdater():
195 # update config/tool_conf.xml with a new tool unpacked in /tools 74 # update config/tool_conf.xml with a new tool unpacked in /tools
196 # requires highly insecure docker settings - like write to tool_conf.xml and to tools ! 75 # requires highly insecure docker settings - like write to tool_conf.xml and to tools !
197 # if in a container possibly not so courageous. 76 # if in a container possibly not so courageous.
198 # Fine on your own laptop but security red flag for most production instances 77 # Fine on your own laptop but security red flag for most production instances
199 78
200 def __init__(self, args, tool_conf_path, new_tool_archive_path, new_tool_name, tool_dir): 79 def __init__(self, args, tool_conf_path, new_tool_archive_path, new_tool_name, tool_dir):
201 self.args = args 80 self.args = args
202 self.tool_conf_path = tool_conf_path 81 self.tool_conf_path = os.path.join(args.galaxy_root,tool_conf_path)
82 self.tool_dir = os.path.join(args.galaxy_root, tool_dir)
203 self.our_name = 'ToolFactory' 83 self.our_name = 'ToolFactory'
204 tff = tarfile.open(new_tool_archive_path, "r:*") 84 tff = tarfile.open(new_tool_archive_path, "r:*")
205 flist = tff.getnames() 85 flist = tff.getnames()
206 ourdir = os.path.commonpath(flist) # eg pyrevpos 86 ourdir = os.path.commonpath(flist) # eg pyrevpos
207 self.tool_id = ourdir # they are the same for TF tools 87 self.tool_id = ourdir # they are the same for TF tools
208 ourxml = [x for x in flist if x.lower().endswith('.xml')] 88 ourxml = [x for x in flist if x.lower().endswith('.xml')]
209 res = tff.extractall(tool_dir) 89 res = tff.extractall()
210 tff.close() 90 tff.close()
91 self.run_rsync(ourdir, self.tool_dir)
211 self.update_toolconf(ourdir,ourxml) 92 self.update_toolconf(ourdir,ourxml)
93
94 def run_rsync(self, srcf, dstf):
95 src = os.path.abspath(srcf)
96 dst = os.path.abspath(dstf)
97 if os.path.isdir(src):
98 cll = ['rsync', '-vr', src, dst]
99 else:
100 cll = ['rsync', '-v', src, dst]
101 p = subprocess.run(
102 cll,
103 capture_output=False,
104 encoding='utf8',
105 shell=False,
106 )
212 107
213 def install_deps(self): 108 def install_deps(self):
214 gi = galaxy.GalaxyInstance(url=self.args.galaxy_url, key=self.args.galaxy_api_key) 109 gi = galaxy.GalaxyInstance(url=self.args.galaxy_url, key=self.args.galaxy_api_key)
215 x = gi.tools.install_dependencies(self.tool_id) 110 x = gi.tools.install_dependencies(self.tool_id)
216 print(f"Called install_dependencies on {self.tool_id} - got {x}") 111 print(f"Called install_dependencies on {self.tool_id} - got {x}")
217 112
218 def update_toolconf(self,ourdir,ourxml): # path is relative to tools 113 def update_toolconf(self,ourdir,ourxml): # path is relative to tools
219 updated = False 114 updated = False
220 tree = ET.parse(self.tool_conf_path) 115 localconf = './local_tool_conf.xml'
116 self.run_rsync(self.tool_conf_path,localconf)
117 tree = ET.parse(localconf)
221 root = tree.getroot() 118 root = tree.getroot()
222 hasTF = False 119 hasTF = False
223 TFsection = None 120 TFsection = None
224 for e in root.findall('section'): 121 for e in root.findall('section'):
225 if e.attrib['name'] == self.our_name: 122 if e.attrib['name'] == self.our_name:
233 for xml in ourxml: # may be > 1 130 for xml in ourxml: # may be > 1
234 if not xml in conf_tools: # new 131 if not xml in conf_tools: # new
235 updated = True 132 updated = True
236 ET.SubElement(TFsection, 'tool', {'file':xml}) 133 ET.SubElement(TFsection, 'tool', {'file':xml})
237 ET.indent(tree) 134 ET.indent(tree)
238 tree.write(self.tool_conf_path, pretty_print=True) 135 newconf = f"{self.tool_id}_conf"
136 tree.write(newconf, pretty_print=True)
137 self.run_rsync(newconf,self.tool_conf_path)
239 if False and self.args.packages and self.args.packages > '': 138 if False and self.args.packages and self.args.packages > '':
240 self.install_deps() 139 self.install_deps()
241 140
242 class ScriptRunner: 141 class ScriptRunner:
243 """Wrapper for an arbitrary script 142 """Wrapper for an arbitrary script
1169 a("--galaxy_venv", default="/galaxy_venv") 1068 a("--galaxy_venv", default="/galaxy_venv")
1170 a("--collection", action="append", default=[]) 1069 a("--collection", action="append", default=[])
1171 a("--include_tests", default=False, action="store_true") 1070 a("--include_tests", default=False, action="store_true")
1172 a("--install", default=False, action="store_true") 1071 a("--install", default=False, action="store_true")
1173 a("--run_test", default=False, action="store_true") 1072 a("--run_test", default=False, action="store_true")
1174 a("--local_tools", default='tools') # relative to galaxy_root 1073 a("--local_tools", default='tools') # relative to $__root_dir__
1175 a("--tool_conf_path", default='/galaxy_root/config/tool_conf.xml') 1074 a("--tool_conf_path", default='config/tool_conf.xml') # relative to $__root_dir__
1176 a("--galaxy_url", default="http://localhost:8080") 1075 a("--galaxy_url", default="http://localhost:8080")
1177 a("--toolshed_url", default="http://localhost:9009") 1076 a("--toolshed_url", default="http://localhost:9009")
1178 # make sure this is identical to tool_sheds_conf.xml 1077 # make sure this is identical to tool_sheds_conf.xml
1179 # localhost != 127.0.0.1 so validation fails 1078 # localhost != 127.0.0.1 so validation fails
1180 a("--toolshed_api_key", default="fakekey") 1079 a("--toolshed_api_key", default="fakekey")
1201 r.makeToolTar() 1100 r.makeToolTar()
1202 else: 1101 else:
1203 tt = ToolTester(report_dir=r.repdir, in_tool_archive=r.newtarpath, new_tool_archive=r.args.new_tool, galaxy_root=args.galaxy_root, include_tests=False) 1102 tt = ToolTester(report_dir=r.repdir, in_tool_archive=r.newtarpath, new_tool_archive=r.args.new_tool, galaxy_root=args.galaxy_root, include_tests=False)
1204 if args.install: 1103 if args.install:
1205 #try: 1104 #try:
1206 tcu = ToolConfUpdater(args=args, tool_dir=os.path.join(args.galaxy_root,args.local_tools), 1105 tcu = ToolConfUpdater(args=args, tool_dir=args.local_tools,
1207 new_tool_archive_path=r.newtarpath, tool_conf_path=os.path.join(args.galaxy_root,'config','tool_conf.xml'), 1106 new_tool_archive_path=r.newtarpath, tool_conf_path=args.tool_conf_path,
1208 new_tool_name=r.tool_name) 1107 new_tool_name=r.tool_name)
1209 #except Exception: 1108 #except Exception:
1210 # print("### Unable to install the new tool. Are you sure you have all the required special settings?") 1109 # print("### Unable to install the new tool. Are you sure you have all the required special settings?")
1211 1110
1212 if __name__ == "__main__": 1111 if __name__ == "__main__":