Mercurial > repos > guerler > springsuite
diff planemo/lib/python3.7/site-packages/galaxy/tool_util/deps/containers.py @ 0:d30785e31577 draft
"planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1"
author | guerler |
---|---|
date | Fri, 31 Jul 2020 00:18:57 -0400 |
parents | |
children |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/planemo/lib/python3.7/site-packages/galaxy/tool_util/deps/containers.py Fri Jul 31 00:18:57 2020 -0400 @@ -0,0 +1,267 @@ +import collections +import logging +import os + +from galaxy.util import ( + asbool, + plugin_config +) +from .container_classes import ( + CONTAINER_CLASSES, + DOCKER_CONTAINER_TYPE, + NULL_CONTAINER, + SINGULARITY_CONTAINER_TYPE, +) +from .container_resolvers import ResolutionCache +from .container_resolvers.explicit import ( + ExplicitContainerResolver, + ExplicitSingularityContainerResolver, +) +from .container_resolvers.mulled import ( + BuildMulledDockerContainerResolver, + BuildMulledSingularityContainerResolver, + CachedMulledDockerContainerResolver, + CachedMulledSingularityContainerResolver, + MulledDockerContainerResolver, + MulledSingularityContainerResolver, +) +from .requirements import ( + ContainerDescription, +) + +log = logging.getLogger(__name__) + + +DEFAULT_CONTAINER_TYPE = DOCKER_CONTAINER_TYPE +ALL_CONTAINER_TYPES = [DOCKER_CONTAINER_TYPE, SINGULARITY_CONTAINER_TYPE] + +ResolvedContainerDescription = collections.namedtuple('ResolvedContainerDescription', ['container_resolver', 'container_description']) + + +class ContainerFinder(object): + + def __init__(self, app_info, mulled_resolution_cache=None): + self.app_info = app_info + self.container_registry = ContainerRegistry(app_info, mulled_resolution_cache=mulled_resolution_cache) + + def _enabled_container_types(self, destination_info): + return [t for t in ALL_CONTAINER_TYPES if self.__container_type_enabled(t, destination_info)] + + def find_best_container_description(self, enabled_container_types, tool_info, **kwds): + """Regardless of destination properties - find best container for tool. + + Given container types and container.ToolInfo description of the tool.""" + return self.container_registry.find_best_container_description(enabled_container_types, tool_info, **kwds) + + def resolve(self, enabled_container_types, tool_info, **kwds): + """Regardless of destination properties - find ResolvedContainerDescription for tool.""" + return self.container_registry.resolve(enabled_container_types, tool_info, **kwds) + + def find_container(self, tool_info, destination_info, job_info): + enabled_container_types = self._enabled_container_types(destination_info) + + # Short-cut everything else and just skip checks if no container type is enabled. + if not enabled_container_types: + return NULL_CONTAINER + + def __destination_container(container_description=None, container_id=None, container_type=None): + if container_description: + container_id = container_description.identifier + container_type = container_description.type + container = self.__destination_container( + container_id, + container_type, + tool_info, + destination_info, + job_info, + container_description, + ) + return container + + def container_from_description_from_dicts(destination_container_dicts): + for destination_container_dict in destination_container_dicts: + container_description = ContainerDescription.from_dict(destination_container_dict) + if container_description: + container = __destination_container(container_description) + if container: + return container + + if "container_override" in destination_info: + container = container_from_description_from_dicts(destination_info["container_override"]) + if container: + return container + + # If destination forcing Galaxy to use a particular container do it, + # this is likely kind of a corner case. For instance if deployers + # do not trust the containers annotated in tools. + for container_type in CONTAINER_CLASSES.keys(): + container_id = self.__overridden_container_id(container_type, destination_info) + if container_id: + container = __destination_container(container_type=container_type, container_id=container_id) + if container: + return container + + # Otherwise lets see if we can find container for the tool. + container_description = self.find_best_container_description(enabled_container_types, tool_info) + container = __destination_container(container_description) + if container: + return container + + # If we still don't have a container, check to see if any container + # types define a default container id and use that. + if "container" in destination_info: + container = container_from_description_from_dicts(destination_info["container"]) + if container: + return container + + for container_type in CONTAINER_CLASSES.keys(): + container_id = self.__default_container_id(container_type, destination_info) + if container_id: + container = __destination_container(container_type=container_type, container_id=container_id) + if container: + return container + + return NULL_CONTAINER + + def resolution_cache(self): + cache = ResolutionCache() + if self.container_registry.mulled_resolution_cache is not None: + cache.mulled_resolution_cache = self.container_registry.mulled_resolution_cache + return cache + + def __overridden_container_id(self, container_type, destination_info): + if not self.__container_type_enabled(container_type, destination_info): + return None + if "%s_container_id_override" % container_type in destination_info: + return destination_info.get("%s_container_id_override" % container_type) + if "%s_image_override" % container_type in destination_info: + return self.__build_container_id_from_parts(container_type, destination_info, mode="override") + + def __build_container_id_from_parts(self, container_type, destination_info, mode): + repo = "" + owner = "" + repo_key = "%s_repo_%s" % (container_type, mode) + owner_key = "%s_owner_%s" % (container_type, mode) + if repo_key in destination_info: + repo = destination_info[repo_key] + "/" + if owner_key in destination_info: + owner = destination_info[owner_key] + "/" + cont_id = repo + owner + destination_info["%s_image_%s" % (container_type, mode)] + tag_key = "%s_tag_%s" % (container_type, mode) + if tag_key in destination_info: + cont_id += ":" + destination_info[tag_key] + return cont_id + + def __default_container_id(self, container_type, destination_info): + if not self.__container_type_enabled(container_type, destination_info): + return None + key = "%s_default_container_id" % container_type + # Also allow docker_image... + if key not in destination_info: + key = "%s_image" % container_type + if key in destination_info: + return destination_info.get(key) + elif "%s_image_default" % container_type in destination_info: + return self.__build_container_id_from_parts(container_type, destination_info, mode="default") + return None + + def __destination_container(self, container_id, container_type, tool_info, destination_info, job_info, container_description=None): + # TODO: ensure destination_info is dict-like + if not self.__container_type_enabled(container_type, destination_info): + return NULL_CONTAINER + + # TODO: Right now this assumes all containers available when a + # container type is - there should be more thought put into this. + # Checking which are available - settings policies for what can be + # auto-fetched, etc.... + return CONTAINER_CLASSES[container_type](container_id, self.app_info, tool_info, destination_info, job_info, container_description) + + def __container_type_enabled(self, container_type, destination_info): + return asbool(destination_info.get("%s_enabled" % container_type, False)) + + +class NullContainerFinder(object): + + def find_container(self, tool_info, destination_info, job_info): + return [] + + +class ContainerRegistry(object): + """Loop through enabled ContainerResolver plugins and find first match.""" + + def __init__(self, app_info, mulled_resolution_cache=None): + self.resolver_classes = self.__resolvers_dict() + self.enable_mulled_containers = app_info.enable_mulled_containers + self.app_info = app_info + self.container_resolvers = self.__build_container_resolvers(app_info) + self.mulled_resolution_cache = mulled_resolution_cache + + def __build_container_resolvers(self, app_info): + conf_file = getattr(app_info, 'containers_resolvers_config_file', None) + if not conf_file: + return self.__default_containers_resolvers() + if not os.path.exists(conf_file): + log.debug("Unable to find config file '%s'", conf_file) + return self.__default_containers_resolvers() + plugin_source = plugin_config.plugin_source_from_path(conf_file) + return self._parse_resolver_conf(plugin_source) + + def _parse_resolver_conf(self, plugin_source): + extra_kwds = { + 'app_info': self.app_info + } + return plugin_config.load_plugins(self.resolver_classes, plugin_source, extra_kwds) + + def __default_containers_resolvers(self): + default_resolvers = [ + ExplicitContainerResolver(self.app_info), + ExplicitSingularityContainerResolver(self.app_info), + ] + if self.enable_mulled_containers: + default_resolvers.extend([ + CachedMulledDockerContainerResolver(self.app_info, namespace="biocontainers"), + CachedMulledDockerContainerResolver(self.app_info, namespace="local"), + CachedMulledSingularityContainerResolver(self.app_info, namespace="biocontainers"), + CachedMulledSingularityContainerResolver(self.app_info, namespace="local"), + MulledDockerContainerResolver(self.app_info, namespace="biocontainers"), + MulledSingularityContainerResolver(self.app_info, namespace="biocontainers"), + BuildMulledDockerContainerResolver(self.app_info), + BuildMulledSingularityContainerResolver(self.app_info), + ]) + return default_resolvers + + def __resolvers_dict(self): + import galaxy.tool_util.deps.container_resolvers + return plugin_config.plugins_dict(galaxy.tool_util.deps.container_resolvers, 'resolver_type') + + def find_best_container_description(self, enabled_container_types, tool_info, **kwds): + """Yield best container description of supplied types matching tool info.""" + try: + resolved_container_description = self.resolve(enabled_container_types, tool_info, **kwds) + except Exception: + log.exception("Could not get container description for tool '%s'", tool_info.tool_id) + return None + return None if resolved_container_description is None else resolved_container_description.container_description + + def resolve(self, enabled_container_types, tool_info, index=None, resolver_type=None, install=True, resolution_cache=None): + for i, container_resolver in enumerate(self.container_resolvers): + if index is not None and i != index: + continue + + if resolver_type is not None and resolver_type != container_resolver.resolver_type: + continue + + if hasattr(container_resolver, "container_type"): + if container_resolver.container_type not in enabled_container_types: + continue + + if not install and container_resolver.builds_on_resolution: + continue + + container_description = container_resolver.resolve(enabled_container_types, tool_info, resolution_cache=resolution_cache) + log.info("Checking with container resolver [%s] found description [%s]" % (container_resolver, container_description)) + if container_description: + assert container_description.type in enabled_container_types + return ResolvedContainerDescription(container_resolver, container_description) + + return None