Mercurial > repos > guerler > springsuite

comparison planemo/lib/python3.7/site-packages/galaxy/tool_util/deps/containers.py @ 0:d30785e31577 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1"
author guerler
date Fri, 31 Jul 2020 00:18:57 -0400
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:d30785e31577
1 import collections
2 import logging
3 import os
4
5 from galaxy.util import (
6 asbool,
7 plugin_config
8 )
9 from .container_classes import (
10 CONTAINER_CLASSES,
11 DOCKER_CONTAINER_TYPE,
12 NULL_CONTAINER,
13 SINGULARITY_CONTAINER_TYPE,
14 )
15 from .container_resolvers import ResolutionCache
16 from .container_resolvers.explicit import (
17 ExplicitContainerResolver,
18 ExplicitSingularityContainerResolver,
19 )
20 from .container_resolvers.mulled import (
21 BuildMulledDockerContainerResolver,
22 BuildMulledSingularityContainerResolver,
23 CachedMulledDockerContainerResolver,
24 CachedMulledSingularityContainerResolver,
25 MulledDockerContainerResolver,
26 MulledSingularityContainerResolver,
27 )
28 from .requirements import (
29 ContainerDescription,
30 )
31
32 log = logging.getLogger(__name__)
33
34
35 DEFAULT_CONTAINER_TYPE = DOCKER_CONTAINER_TYPE
36 ALL_CONTAINER_TYPES = [DOCKER_CONTAINER_TYPE, SINGULARITY_CONTAINER_TYPE]
37
38 ResolvedContainerDescription = collections.namedtuple('ResolvedContainerDescription', ['container_resolver', 'container_description'])
39
40
41 class ContainerFinder(object):
42
43 def __init__(self, app_info, mulled_resolution_cache=None):
44 self.app_info = app_info
45 self.container_registry = ContainerRegistry(app_info, mulled_resolution_cache=mulled_resolution_cache)
46
47 def _enabled_container_types(self, destination_info):
48 return [t for t in ALL_CONTAINER_TYPES if self.__container_type_enabled(t, destination_info)]
49
50 def find_best_container_description(self, enabled_container_types, tool_info, **kwds):
51 """Regardless of destination properties - find best container for tool.
52
53 Given container types and container.ToolInfo description of the tool."""
54 return self.container_registry.find_best_container_description(enabled_container_types, tool_info, **kwds)
55
56 def resolve(self, enabled_container_types, tool_info, **kwds):
57 """Regardless of destination properties - find ResolvedContainerDescription for tool."""
58 return self.container_registry.resolve(enabled_container_types, tool_info, **kwds)
59
60 def find_container(self, tool_info, destination_info, job_info):
61 enabled_container_types = self._enabled_container_types(destination_info)
62
63 # Short-cut everything else and just skip checks if no container type is enabled.
64 if not enabled_container_types:
65 return NULL_CONTAINER
66
67 def __destination_container(container_description=None, container_id=None, container_type=None):
68 if container_description:
69 container_id = container_description.identifier
70 container_type = container_description.type
71 container = self.__destination_container(
72 container_id,
73 container_type,
74 tool_info,
75 destination_info,
76 job_info,
77 container_description,
78 )
79 return container
80
81 def container_from_description_from_dicts(destination_container_dicts):
82 for destination_container_dict in destination_container_dicts:
83 container_description = ContainerDescription.from_dict(destination_container_dict)
84 if container_description:
85 container = __destination_container(container_description)
86 if container:
87 return container
88
89 if "container_override" in destination_info:
90 container = container_from_description_from_dicts(destination_info["container_override"])
91 if container:
92 return container
93
94 # If destination forcing Galaxy to use a particular container do it,
95 # this is likely kind of a corner case. For instance if deployers
96 # do not trust the containers annotated in tools.
97 for container_type in CONTAINER_CLASSES.keys():
98 container_id = self.__overridden_container_id(container_type, destination_info)
99 if container_id:
100 container = __destination_container(container_type=container_type, container_id=container_id)
101 if container:
102 return container
103
104 # Otherwise lets see if we can find container for the tool.
105 container_description = self.find_best_container_description(enabled_container_types, tool_info)
106 container = __destination_container(container_description)
107 if container:
108 return container
109
110 # If we still don't have a container, check to see if any container
111 # types define a default container id and use that.
112 if "container" in destination_info:
113 container = container_from_description_from_dicts(destination_info["container"])
114 if container:
115 return container
116
117 for container_type in CONTAINER_CLASSES.keys():
118 container_id = self.__default_container_id(container_type, destination_info)
119 if container_id:
120 container = __destination_container(container_type=container_type, container_id=container_id)
121 if container:
122 return container
123
124 return NULL_CONTAINER
125
126 def resolution_cache(self):
127 cache = ResolutionCache()
128 if self.container_registry.mulled_resolution_cache is not None:
129 cache.mulled_resolution_cache = self.container_registry.mulled_resolution_cache
130 return cache
131
132 def __overridden_container_id(self, container_type, destination_info):
133 if not self.__container_type_enabled(container_type, destination_info):
134 return None
135 if "%s_container_id_override" % container_type in destination_info:
136 return destination_info.get("%s_container_id_override" % container_type)
137 if "%s_image_override" % container_type in destination_info:
138 return self.__build_container_id_from_parts(container_type, destination_info, mode="override")
139
140 def __build_container_id_from_parts(self, container_type, destination_info, mode):
141 repo = ""
142 owner = ""
143 repo_key = "%s_repo_%s" % (container_type, mode)
144 owner_key = "%s_owner_%s" % (container_type, mode)
145 if repo_key in destination_info:
146 repo = destination_info[repo_key] + "/"
147 if owner_key in destination_info:
148 owner = destination_info[owner_key] + "/"
149 cont_id = repo + owner + destination_info["%s_image_%s" % (container_type, mode)]
150 tag_key = "%s_tag_%s" % (container_type, mode)
151 if tag_key in destination_info:
152 cont_id += ":" + destination_info[tag_key]
153 return cont_id
154
155 def __default_container_id(self, container_type, destination_info):
156 if not self.__container_type_enabled(container_type, destination_info):
157 return None
158 key = "%s_default_container_id" % container_type
159 # Also allow docker_image...
160 if key not in destination_info:
161 key = "%s_image" % container_type
162 if key in destination_info:
163 return destination_info.get(key)
164 elif "%s_image_default" % container_type in destination_info:
165 return self.__build_container_id_from_parts(container_type, destination_info, mode="default")
166 return None
167
168 def __destination_container(self, container_id, container_type, tool_info, destination_info, job_info, container_description=None):
169 # TODO: ensure destination_info is dict-like
170 if not self.__container_type_enabled(container_type, destination_info):
171 return NULL_CONTAINER
172
173 # TODO: Right now this assumes all containers available when a
174 # container type is - there should be more thought put into this.
175 # Checking which are available - settings policies for what can be
176 # auto-fetched, etc....
177 return CONTAINER_CLASSES[container_type](container_id, self.app_info, tool_info, destination_info, job_info, container_description)
178
179 def __container_type_enabled(self, container_type, destination_info):
180 return asbool(destination_info.get("%s_enabled" % container_type, False))
181
182
183 class NullContainerFinder(object):
184
185 def find_container(self, tool_info, destination_info, job_info):
186 return []
187
188
189 class ContainerRegistry(object):
190 """Loop through enabled ContainerResolver plugins and find first match."""
191
192 def __init__(self, app_info, mulled_resolution_cache=None):
193 self.resolver_classes = self.__resolvers_dict()
194 self.enable_mulled_containers = app_info.enable_mulled_containers
195 self.app_info = app_info
196 self.container_resolvers = self.__build_container_resolvers(app_info)
197 self.mulled_resolution_cache = mulled_resolution_cache
198
199 def __build_container_resolvers(self, app_info):
200 conf_file = getattr(app_info, 'containers_resolvers_config_file', None)
201 if not conf_file:
202 return self.__default_containers_resolvers()
203 if not os.path.exists(conf_file):
204 log.debug("Unable to find config file '%s'", conf_file)
205 return self.__default_containers_resolvers()
206 plugin_source = plugin_config.plugin_source_from_path(conf_file)
207 return self._parse_resolver_conf(plugin_source)
208
209 def _parse_resolver_conf(self, plugin_source):
210 extra_kwds = {
211 'app_info': self.app_info
212 }
213 return plugin_config.load_plugins(self.resolver_classes, plugin_source, extra_kwds)
214
215 def __default_containers_resolvers(self):
216 default_resolvers = [
217 ExplicitContainerResolver(self.app_info),
218 ExplicitSingularityContainerResolver(self.app_info),
219 ]
220 if self.enable_mulled_containers:
221 default_resolvers.extend([
222 CachedMulledDockerContainerResolver(self.app_info, namespace="biocontainers"),
223 CachedMulledDockerContainerResolver(self.app_info, namespace="local"),
224 CachedMulledSingularityContainerResolver(self.app_info, namespace="biocontainers"),
225 CachedMulledSingularityContainerResolver(self.app_info, namespace="local"),
226 MulledDockerContainerResolver(self.app_info, namespace="biocontainers"),
227 MulledSingularityContainerResolver(self.app_info, namespace="biocontainers"),
228 BuildMulledDockerContainerResolver(self.app_info),
229 BuildMulledSingularityContainerResolver(self.app_info),
230 ])
231 return default_resolvers
232
233 def __resolvers_dict(self):
234 import galaxy.tool_util.deps.container_resolvers
235 return plugin_config.plugins_dict(galaxy.tool_util.deps.container_resolvers, 'resolver_type')
236
237 def find_best_container_description(self, enabled_container_types, tool_info, **kwds):
238 """Yield best container description of supplied types matching tool info."""
239 try:
240 resolved_container_description = self.resolve(enabled_container_types, tool_info, **kwds)
241 except Exception:
242 log.exception("Could not get container description for tool '%s'", tool_info.tool_id)
243 return None
244 return None if resolved_container_description is None else resolved_container_description.container_description
245
246 def resolve(self, enabled_container_types, tool_info, index=None, resolver_type=None, install=True, resolution_cache=None):
247 for i, container_resolver in enumerate(self.container_resolvers):
248 if index is not None and i != index:
249 continue
250
251 if resolver_type is not None and resolver_type != container_resolver.resolver_type:
252 continue
253
254 if hasattr(container_resolver, "container_type"):
255 if container_resolver.container_type not in enabled_container_types:
256 continue
257
258 if not install and container_resolver.builds_on_resolution:
259 continue
260
261 container_description = container_resolver.resolve(enabled_container_types, tool_info, resolution_cache=resolution_cache)
262 log.info("Checking with container resolver [%s] found description [%s]" % (container_resolver, container_description))
263 if container_description:
264 assert container_description.type in enabled_container_types
265 return ResolvedContainerDescription(container_resolver, container_description)
266
267 return None