Mercurial > repos > guerler > springsuite

comparison planemo/lib/python3.7/site-packages/galaxy/tool_util/deps/views.py @ 1:56ad4e20f292 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1"
author guerler
date Fri, 31 Jul 2020 00:32:28 -0400
parents
children
comparison
equal deleted inserted replaced
0:d30785e31577 1:56ad4e20f292
1 from galaxy import exceptions
2 from galaxy.util import asbool, listify
3 from .dependencies import ToolInfo
4 from .resolvers import (
5 ContainerDependency,
6 NullDependency,
7 )
8
9
10 class DependencyResolversView(object):
11 """ Provide a RESTfulish/JSONy interface to a galaxy.tool_util.deps.DependencyResolver
12 object. This can be adapted by the Galaxy web framework or other web apps.
13 """
14
15 def __init__(self, app):
16 self._app = app
17
18 def index(self):
19 return [r.to_dict() for r in self._dependency_resolvers]
20
21 def show(self, index):
22 return self._dependency_resolver(index).to_dict()
23
24 def reload(self):
25 self.toolbox.reload_dependency_manager()
26
27 def manager_requirements(self):
28 requirements = []
29 for index, resolver in enumerate(self._dependency_resolvers):
30 if not hasattr(resolver, "list_dependencies"):
31 continue
32 for requirement in resolver.list_dependencies():
33 requirements.append({"index": index, "requirement": requirement.to_dict()})
34 return requirements
35
36 def resolver_requirements(self, index):
37 requirements = []
38 resolver = self._dependency_resolver(index)
39 if not hasattr(resolver, "list_dependencies"):
40 raise exceptions.NotImplemented()
41 for requirement in resolver.list_dependencies():
42 requirements.append(requirement.to_dict())
43 return requirements
44
45 def manager_dependency(self, **kwds):
46 return self._dependency(**kwds)
47
48 def resolver_dependency(self, index, **kwds):
49 return self._dependency(**kwds)
50
51 def show_dependencies(self, tool_requirements_d, installed_tool_dependencies=None, **kwd):
52 """
53 Resolves dependencies to build a requirements status in the admin panel/API
54 """
55 to_deps_kwds = {
56 'install': False,
57 'return_null': True,
58 'installed_tool_dependencies': installed_tool_dependencies
59 }
60 to_deps_kwds.update(kwd)
61 dependencies_per_tool = {tool: self._dependency_manager.requirements_to_dependencies(requirements,
62 **to_deps_kwds)
63 for tool, requirements in tool_requirements_d.items()}
64 return dependencies_per_tool
65
66 def uninstall_dependencies(self, index=None, resolver_type=None, container_type=None, **payload):
67 """Attempt to uninstall requirements. Returns 0 if successfull, else None."""
68 requirements = payload.get('requirements')
69 if not requirements:
70 return None
71 if index:
72 resolver = self._dependency_resolvers[index]
73 if resolver.can_uninstall_dependencies:
74 return resolver.uninstall(requirements)
75 elif resolver_type:
76 for resolver in self._dependency_resolvers:
77 if resolver.resolver_type == resolver_type and resolver.can_uninstall_dependencies:
78 return resolver.uninstall(requirements)
79 elif container_type:
80 for resolver in self._dependency_resolvers:
81 if getattr(resolver, "container_type", None) == container_type and resolver.can_uninstall_dependencies:
82 return resolver.uninstall(requirements)
83 else:
84 for index in self.uninstallable_resolvers:
85 return_code = self._dependency_resolvers[index].uninstall(requirements)
86 if return_code == 0:
87 return return_code
88 return None
89
90 @property
91 def unused_dependency_paths(self):
92 """List dependencies that are not currently installed."""
93 unused_dependencies = []
94 toolbox_requirements_status = self.toolbox_requirements_status
95 for resolver in self._dependency_resolvers:
96 if hasattr(resolver, 'unused_dependency_paths'):
97 unused_dependencies.extend(resolver.unused_dependency_paths(toolbox_requirements_status))
98 return set(unused_dependencies)
99
100 def remove_unused_dependency_paths(self, envs):
101 """
102 Remove dependencies that are not currently used.
103
104 Returns a list of all environments that have been successfully removed.
105 """
106 envs_to_remove = set(envs)
107 toolbox_requirements_status = self.toolbox_requirements_status
108 removed_environments = set()
109 for resolver in self._dependency_resolvers:
110 if hasattr(resolver, 'unused_dependency_paths') and hasattr(resolver, 'uninstall_environments'):
111 unused_dependencies = resolver.unused_dependency_paths(toolbox_requirements_status)
112 can_remove = envs_to_remove & set(unused_dependencies)
113 exit_code = resolver.uninstall_environments(can_remove)
114 if exit_code == 0:
115 removed_environments = removed_environments.union(can_remove)
116 envs_to_remove = envs_to_remove.difference(can_remove)
117 return list(removed_environments)
118
119 def install_dependencies(self, requirements, **kwds):
120 kwds['install'] = True
121 return self._dependency_manager._requirements_to_dependencies_dict(requirements, **kwds)
122
123 def install_dependency(self, index=None, **payload):
124 """
125 Installs dependency using highest priority resolver that supports dependency installation
126 (Currently only the conda resolver supports this). If index is given, attempt
127 installation directly using the corresponding resolver.
128 Returns True on success, False on failure.
129 payload is dictionary that must container name, version and type,
130 e.g. {'name': 'numpy', version='1.9.1', type='package'}
131 """
132 if index:
133 return self._install_dependency(index, **payload)
134 else:
135 for index in self.installable_resolvers:
136 success = self._install_dependency(index, **payload)
137 if success:
138 return success
139 return False
140
141 def _install_dependency(self, index, **payload):
142 """
143 Resolver install dependency should return True when installation succeeds,
144 False if not successful
145 """
146 resolver = self._dependency_resolver(index)
147 if not hasattr(resolver, "install_dependency"):
148 raise exceptions.NotImplemented()
149
150 name, version, type, extra_kwds = self._parse_dependency_info(payload)
151 return resolver.install_dependency(
152 name=name,
153 version=version,
154 type=type,
155 **extra_kwds
156 )
157
158 def _dependency(self, index=None, **kwds):
159 if index is not None:
160 index = int(index)
161
162 name, version, type, extra_kwds = self._parse_dependency_info(kwds)
163 resolve_kwds = dict(
164 job_directory=None,
165 index=index,
166 **extra_kwds
167 )
168 dependency = self._dependency_manager.find_dep(
169 name, version=version, type=type, **resolve_kwds
170 )
171 return dependency.to_dict()
172
173 def _parse_dependency_info(self, kwds):
174 extra_kwds = kwds.copy()
175 name = extra_kwds.pop("name", None)
176 if name is None:
177 raise exceptions.RequestParameterMissingException("Missing 'name' parameter required for resolution.")
178 version = extra_kwds.pop("version", None)
179 type = extra_kwds.pop("type", "package")
180 return name, version, type, extra_kwds
181
182 def _dependency_resolver(self, index):
183 index = int(index)
184 return self._dependency_resolvers[index]
185
186 @property
187 def _dependency_manager(self):
188 return self._app.toolbox.dependency_manager
189
190 @property
191 def _dependency_resolvers(self):
192 dependency_manager = self._dependency_manager
193 dependency_resolvers = dependency_manager.dependency_resolvers
194 return dependency_resolvers
195
196 @property
197 def installable_resolvers(self):
198 """
199 List index for all active resolvers that have the 'install_dependency' attribute.
200 """
201 return [index for index, resolver in enumerate(self._dependency_resolvers) if hasattr(resolver, "install_dependency") and not resolver.disabled]
202
203 @property
204 def uninstallable_resolvers(self):
205 """
206 List index for all active resolvers that can uninstall dependencies that have been installed through this resolver.
207 """
208 return [index for index, resolver in enumerate(self._dependency_resolvers) if resolver.can_uninstall_dependencies and not resolver.disabled]
209
210 @property
211 def tool_ids_by_requirements(self):
212 """Dictionary with requirements as keys, and tool_ids as values."""
213 tool_ids_by_requirements = {}
214 if not self._app.toolbox.tools_by_id:
215 return {}
216 for tid, tool in self._app.toolbox.tools_by_id.items():
217 if tool.tool_requirements not in tool_ids_by_requirements:
218 tool_ids_by_requirements[tool.tool_requirements] = [tid]
219 else:
220 tool_ids_by_requirements[tool.tool_requirements].append(tid)
221 return tool_ids_by_requirements
222
223 @property
224 def toolbox_requirements_status(self):
225 return self.summarize_requirements()
226
227 def summarize_requirements(self, **kwds):
228 summary_kwds = {}
229 if 'index' in kwds:
230 summary_kwds['index'] = int(kwds['index'])
231 if 'include_containers' in kwds:
232 summary_kwds['include_containers'] = asbool(kwds['include_containers'])
233 if 'container_type' in kwds:
234 summary_kwds['container_type'] = kwds['container_type']
235 if 'resolver_type' in kwds:
236 summary_kwds['resolver_type'] = kwds['resolver_type']
237 if 'search' in kwds:
238 summary_kwds['search'] = asbool(kwds['search'])
239 if 'install' in kwds:
240 summary_kwds['install'] = asbool(kwds['install'])
241
242 tool_ids_by_requirements = self.tool_ids_by_requirements
243 statuses = {r: self.get_requirements_status(tool_requirements_d={tids[0]: r},
244 installed_tool_dependencies=self._app.toolbox.tools_by_id[tids[0]].installed_tool_dependencies, **summary_kwds)
245 for r, tids in tool_ids_by_requirements.items()}
246 if kwds.get("for_json", False):
247 # All public attributes of this class should be returning JSON - this is meant to mimic a restful API.
248 rval = []
249 for requirements, status in statuses.items():
250 item = {}
251 item["requirements"] = requirements.to_dict()
252 item["status"] = status
253 item["tool_ids"] = tool_ids_by_requirements[requirements]
254 rval.append(item)
255 statuses = rval
256 return statuses
257
258 def summarize_tools(self, **kwds):
259 summary_kwds = {}
260 if 'index' in kwds:
261 summary_kwds['index'] = int(kwds['index'])
262 if 'include_containers' in kwds:
263 summary_kwds['include_containers'] = asbool(kwds['include_containers'])
264 if 'container_type' in kwds:
265 summary_kwds['container_type'] = kwds['container_type']
266 if 'resolver_type' in kwds:
267 summary_kwds['resolver_type'] = kwds['resolver_type']
268 if 'search' in kwds:
269 summary_kwds['search'] = asbool(kwds['search'])
270 if 'install' in kwds:
271 summary_kwds['install'] = asbool(kwds['install'])
272
273 tool_ids = pop_tool_ids(kwds)
274
275 rval = []
276 for tid, tool in self._app.toolbox.tools_by_id.items():
277 if tool_ids and tid not in tool_ids:
278 continue
279
280 requirements = tool.tool_requirements
281 status = self.get_requirements_status(tool_requirements_d={tid: requirements},
282 installed_tool_dependencies=tool.installed_tool_dependencies,
283 tool_instance=tool, **summary_kwds)
284 item = {}
285 item["requirements"] = requirements.to_dict()
286 item["status"] = status
287 item["tool_ids"] = [tid]
288 rval.append(item)
289 return rval
290
291 def get_requirements_status(self, tool_requirements_d, installed_tool_dependencies=None, **kwd):
292 dependencies = self.show_dependencies(tool_requirements_d, installed_tool_dependencies, **kwd)
293 # dependencies is a dict keyed on tool_ids, value is a ToolRequirements object for that tool.
294 # We use the union of resolvable ToolRequirements to get resolved dependencies without duplicates.
295 requirements = [r.resolvable for r in tool_requirements_d.values()]
296 flat_tool_requirements = set().union(*requirements)
297 flat_dependencies = []
298 for requirements_odict in dependencies.values():
299 for requirement in requirements_odict:
300 if requirement in flat_tool_requirements:
301 flat_dependencies.append(requirements_odict[requirement])
302 flat_tool_requirements.remove(requirement)
303 return [d.to_dict() for d in flat_dependencies]
304
305 def clean(self, index=None, **kwds):
306 if index:
307 resolver = self._dependency_resolver(index)
308 if not hasattr(resolver, "clean"):
309 raise exceptions.NotImplemented()
310 else:
311 resolver.clean()
312 return "OK"
313 else:
314 [resolver.clean(**kwds) for resolver in self._dependency_resolvers if hasattr(resolver, 'clean')]
315 return "OK"
316
317
318 class ContainerResolutionView(object):
319 """
320 """
321
322 def __init__(self, app):
323 self._app = app
324
325 def index(self):
326 return [r.to_dict() for r in self._container_resolvers]
327
328 def show(self, index):
329 return self._container_resolver(index).to_dict()
330
331 def resolve(self, **kwds):
332 find_best_kwds = {
333 'install': False,
334 'enabled_container_types': ['docker', 'singularity'],
335 'resolution_cache': kwds.get("resolution_cache"),
336 }
337
338 if 'index' in kwds:
339 find_best_kwds['index'] = int(kwds['index'])
340 if 'container_type' in kwds:
341 find_best_kwds['enabled_container_types'] = [kwds['container_type']]
342 if 'resolver_type' in kwds:
343 find_best_kwds['resolver_type'] = kwds['resolver_type']
344 if 'install' in kwds:
345 find_best_kwds['install'] = asbool(kwds['install'])
346
347 tool_info_kwds = {}
348 tool_id = kwds["tool_id"]
349 tool = self._app.toolbox.tools_by_id[tool_id]
350
351 requirements = tool.tool_requirements
352 tool_info_kwds = dict(requirements=requirements)
353 requirements_only = asbool(kwds.get("requirements_only", False))
354 # If requirements_only, simply use the tool to load a requirement set from,
355 # mimics the default behavior of searching for containers through the dependency resolution
356 # component. Not useful for tool execution but perhaps when summarizing mulled containers for requirements.
357 if not requirements_only:
358 tool_info_kwds['container_descriptions'] = tool.containers
359 tool_info_kwds['requires_galaxy_python_environment'] = tool.requires_galaxy_python_environment
360 tool_info_kwds['tool_id'] = tool.id
361 tool_info_kwds['tool_version'] = tool.version
362
363 find_best_kwds["tool_info"] = ToolInfo(**tool_info_kwds)
364
365 # Consider implementing 'search' to match dependency resolution API.
366 resolved_container_description = self._app.container_finder.resolve(
367 **find_best_kwds
368 )
369 if resolved_container_description:
370 status = ContainerDependency(resolved_container_description.container_description, container_resolver=resolved_container_description.container_resolver).to_dict()
371 else:
372 status = NullDependency().to_dict()
373 return {
374 "tool_id": kwds["tool_id"],
375 "status": status,
376 "requirements": requirements.to_dict()
377 }
378
379 def resolve_toolbox(self, **kwds):
380 rval = []
381 resolve_kwds = kwds.copy()
382 tool_ids = pop_tool_ids(resolve_kwds)
383 resolve_kwds["resolution_cache"] = self._app.container_finder.resolution_cache()
384 if tool_ids is not None:
385 tool_ids = listify(tool_ids)
386 for tool_id, tool in self._app.toolbox.tools_by_id.items():
387 if tool_ids is not None and tool_id not in tool_ids:
388 continue
389
390 if tool.tool_action.produces_real_jobs:
391 rval.append(self.resolve(tool_id=tool_id, **resolve_kwds))
392 return rval
393
394 @property
395 def _container_resolvers(self):
396 return self._app.container_finder.container_resolvers
397
398 def _container_resolver(self, index):
399 index = int(index)
400 return self._container_resolvers[index]
401
402
403 def pop_tool_ids(kwds):
404 tool_ids = None
405 if "tool_ids" in kwds:
406 tool_ids = listify(kwds.pop("tool_ids"))
407 if "tool_ids[]" in kwds:
408 tool_ids = listify(kwds.pop("tool_ids[]"))
409 if "tool_id" in kwds:
410 tool_ids = [kwds.pop("tool_id")]
411 return tool_ids