Mercurial > repos > shellac > guppy_basecaller

comparison env/lib/python3.7/site-packages/cwltool/singularity.py @ 0:26e78fe6e8c4 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
author shellac
date Sat, 02 May 2020 07:14:21 -0400
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:26e78fe6e8c4
1 """Support for executing Docker containers using the Singularity 2.x engine."""
2 from __future__ import absolute_import
3
4 import os
5 import os.path
6 import re
7 import shutil
8 import tempfile
9 import sys
10 from distutils import spawn
11 from io import open # pylint: disable=redefined-builtin
12 from typing import Dict, List, MutableMapping, Optional, Tuple
13
14 from schema_salad.sourceline import SourceLine
15 from typing_extensions import Text # pylint: disable=unused-import
16 # move to a regular typing import when Python 3.3-3.6 is no longer supported
17
18 from .context import RuntimeContext # pylint: disable=unused-import
19 from .errors import WorkflowException
20 from .job import ContainerCommandLineJob
21 from .loghandler import _logger
22 from .pathmapper import PathMapper, MapperEnt # pylint: disable=unused-import
23 from .pathmapper import ensure_writable, ensure_non_writable
24 from .process import UnsupportedRequirement
25 from .utils import docker_windows_path_adjust
26
27 if os.name == 'posix':
28 if sys.version_info < (3, 5):
29 from subprocess32 import ( # nosec # pylint: disable=import-error,no-name-in-module
30 check_call, check_output, CalledProcessError, DEVNULL, PIPE, Popen,
31 TimeoutExpired)
32 else:
33 from subprocess import ( # nosec # pylint: disable=import-error,no-name-in-module
34 check_call, check_output, CalledProcessError, DEVNULL, PIPE, Popen,
35 TimeoutExpired)
36
37 else: # we're not on Unix, so none of this matters
38 pass
39
40 _USERNS = None
41 _SINGULARITY_VERSION = ""
42
43 def _singularity_supports_userns(): # type: ()->bool
44 global _USERNS # pylint: disable=global-statement
45 if _USERNS is None:
46 try:
47 hello_image = os.path.join(os.path.dirname(__file__), 'hello.simg')
48 result = Popen( # nosec
49 [u"singularity", u"exec", u"--userns", hello_image, u"true"],
50 stderr=PIPE, stdout=DEVNULL,
51 universal_newlines=True).communicate(timeout=60)[1]
52 _USERNS = "No valid /bin/sh" in result
53 except TimeoutExpired:
54 _USERNS = False
55 return _USERNS
56
57
58 def get_version(): # type: ()->Text
59 global _SINGULARITY_VERSION # pylint: disable=global-statement
60 if not _SINGULARITY_VERSION:
61 _SINGULARITY_VERSION = check_output(["singularity", "--version"], universal_newlines=True)
62 if _SINGULARITY_VERSION.startswith("singularity version "):
63 _SINGULARITY_VERSION = _SINGULARITY_VERSION[20:]
64 return _SINGULARITY_VERSION
65
66 def is_version_2_6(): # type: ()->bool
67 return get_version().startswith("2.6")
68
69 def is_version_3_or_newer(): # type: ()->bool
70 return int(get_version()[0]) >= 3
71
72 def is_version_3_1_or_newer(): # type: ()->bool
73 version = get_version().split('.')
74 return int(version[0]) >= 4 or (int(version[0]) == 3 and int(version[1]) >= 1)
75
76 def _normalize_image_id(string): # type: (Text)->Text
77 return string.replace('/', '_') + '.img'
78
79 def _normalize_sif_id(string): # type: (Text)->Text
80 return string.replace('/', '_') + '.sif'
81
82 class SingularityCommandLineJob(ContainerCommandLineJob):
83
84 @staticmethod
85 def get_image(dockerRequirement, # type: Dict[Text, Text]
86 pull_image, # type: bool
87 force_pull=False # type: bool
88 ):
89 # type: (...) -> bool
90 """
91 Acquire the software container image in the specified dockerRequirement.
92
93 Uses Singularity and returns the success as a bool. Updates the
94 provided dockerRequirement with the specific dockerImageId to the full
95 path of the local image, if found. Likewise the
96 dockerRequirement['dockerPull'] is updated to a docker:// URI if needed.
97 """
98 found = False
99
100 candidates = []
101
102 cache_folder = None
103 if "CWL_SINGULARITY_CACHE" in os.environ:
104 cache_folder = os.environ["CWL_SINGULARITY_CACHE"]
105 elif is_version_2_6() and "SINGULARITY_PULLFOLDER" in os.environ:
106 cache_folder = os.environ["SINGULARITY_PULLFOLDER"]
107
108 if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
109 match = re.search(pattern=r'([a-z]*://)', string=dockerRequirement["dockerPull"])
110 img_name = _normalize_image_id(dockerRequirement['dockerPull'])
111 candidates.append(img_name)
112 if is_version_3_or_newer():
113 sif_name = _normalize_sif_id(dockerRequirement['dockerPull'])
114 candidates.append(sif_name)
115 dockerRequirement["dockerImageId"] = sif_name
116 else:
117 dockerRequirement["dockerImageId"] = img_name
118 if not match:
119 dockerRequirement["dockerPull"] = "docker://" + dockerRequirement["dockerPull"]
120 elif "dockerImageId" in dockerRequirement:
121 if os.path.isfile(dockerRequirement['dockerImageId']):
122 found = True
123 candidates.append(dockerRequirement['dockerImageId'])
124 candidates.append(_normalize_image_id(dockerRequirement['dockerImageId']))
125 if is_version_3_or_newer():
126 candidates.append(_normalize_sif_id(dockerRequirement['dockerPull']))
127
128 targets = [os.getcwd()]
129 if "CWL_SINGULARITY_CACHE" in os.environ:
130 targets.append(os.environ["CWL_SINGULARITY_CACHE"])
131 if is_version_2_6() and "SINGULARITY_PULLFOLDER" in os.environ:
132 targets.append(os.environ["SINGULARITY_PULLFOLDER"])
133 for target in targets:
134 for dirpath, subdirs, files in os.walk(target):
135 for entry in files:
136 if entry in candidates:
137 path = os.path.join(dirpath, entry)
138 if os.path.isfile(path):
139 _logger.info(
140 "Using local copy of Singularity image found in %s",
141 dirpath)
142 dockerRequirement["dockerImageId"] = path
143 found = True
144 if (force_pull or not found) and pull_image:
145 cmd = [] # type: List[Text]
146 if "dockerPull" in dockerRequirement:
147 if cache_folder:
148 env = os.environ.copy()
149 if is_version_2_6():
150 env['SINGULARITY_PULLFOLDER'] = cache_folder
151 cmd = ["singularity", "pull", "--force", "--name",
152 dockerRequirement["dockerImageId"],
153 str(dockerRequirement["dockerPull"])]
154 else:
155 cmd = ["singularity", "pull", "--force", "--name",
156 "{}/{}".format(
157 cache_folder,
158 dockerRequirement["dockerImageId"]),
159 str(dockerRequirement["dockerPull"])]
160
161 _logger.info(Text(cmd))
162 check_call(cmd, env=env, stdout=sys.stderr) # nosec
163 dockerRequirement["dockerImageId"] = '{}/{}'.format(
164 cache_folder, dockerRequirement["dockerImageId"])
165 found = True
166 else:
167 cmd = ["singularity", "pull", "--force", "--name",
168 str(dockerRequirement["dockerImageId"]),
169 str(dockerRequirement["dockerPull"])]
170 _logger.info(Text(cmd))
171 check_call(cmd, stdout=sys.stderr) # nosec
172 found = True
173
174 elif "dockerFile" in dockerRequirement:
175 raise WorkflowException(SourceLine(
176 dockerRequirement, 'dockerFile').makeError(
177 "dockerFile is not currently supported when using the "
178 "Singularity runtime for Docker containers."))
179 elif "dockerLoad" in dockerRequirement:
180 if is_version_3_1_or_newer():
181 if 'dockerImageId' in dockerRequirement:
182 name = "{}.sif".format(dockerRequirement["dockerImageId"])
183 else:
184 name = "{}.sif".format(dockerRequirement["dockerLoad"])
185 cmd = ["singularity", "build", name,
186 "docker-archive://{}".format(dockerRequirement["dockerLoad"])]
187 _logger.info(Text(cmd))
188 check_call(cmd, stdout=sys.stderr) # nosec
189 found = True
190 dockerRequirement['dockerImageId'] = name
191 raise WorkflowException(SourceLine(
192 dockerRequirement, 'dockerLoad').makeError(
193 "dockerLoad is not currently supported when using the "
194 "Singularity runtime (version less than 3.1) for Docker containers."))
195 elif "dockerImport" in dockerRequirement:
196 raise WorkflowException(SourceLine(
197 dockerRequirement, 'dockerImport').makeError(
198 "dockerImport is not currently supported when using the "
199 "Singularity runtime for Docker containers."))
200
201 return found
202
203 def get_from_requirements(self,
204 r, # type: Dict[Text, Text]
205 pull_image, # type: bool
206 force_pull=False, # type: bool
207 tmp_outdir_prefix=None # type: Optional[Text]
208 ):
209 # type: (...) -> Optional[Text]
210 """
211 Return the filename of the Singularity image.
212
213 (e.g. hello-world-latest.{img,sif}).
214 """
215 if not bool(spawn.find_executable('singularity')):
216 raise WorkflowException('singularity executable is not available')
217
218 if not self.get_image(r, pull_image, force_pull):
219 raise WorkflowException(u"Container image {} not "
220 "found".format(r["dockerImageId"]))
221
222 return os.path.abspath(r["dockerImageId"])
223
224 @staticmethod
225 def append_volume(runtime, source, target, writable=False):
226 # type: (List[Text], Text, Text, bool) -> None
227 runtime.append(u"--bind")
228 runtime.append("{}:{}:{}".format(
229 docker_windows_path_adjust(source),
230 docker_windows_path_adjust(target), "rw" if writable else "ro"))
231
232 def add_file_or_directory_volume(self,
233 runtime, # type: List[Text]
234 volume, # type: MapperEnt
235 host_outdir_tgt # type: Optional[Text]
236 ): # type: (...) -> None
237 if host_outdir_tgt is not None:
238 # workaround for lack of overlapping mounts in Singularity
239 # revert to daa923d5b0be3819b6ed0e6440e7193e65141052
240 # once https://github.com/sylabs/singularity/issues/1607
241 # is fixed
242 if volume.type == "File":
243 shutil.copy(volume.resolved, host_outdir_tgt)
244 else:
245 shutil.copytree(volume.resolved, host_outdir_tgt)
246 ensure_non_writable(host_outdir_tgt)
247 elif not volume.resolved.startswith("_:"):
248 self.append_volume(runtime, volume.resolved, volume.target)
249
250 def add_writable_file_volume(self,
251 runtime, # type: List[Text]
252 volume, # type: MapperEnt
253 host_outdir_tgt, # type: Optional[Text]
254 tmpdir_prefix # type: Text
255 ): # type: (...) -> None
256 if host_outdir_tgt is not None:
257 # workaround for lack of overlapping mounts in Singularity
258 # revert to daa923d5b0be3819b6ed0e6440e7193e65141052
259 # once https://github.com/sylabs/singularity/issues/1607
260 # is fixed
261 if self.inplace_update:
262 try:
263 os.link(os.path.realpath(volume.resolved),
264 host_outdir_tgt)
265 except os.error:
266 shutil.copy(volume.resolved, host_outdir_tgt)
267 else:
268 shutil.copy(volume.resolved, host_outdir_tgt)
269 ensure_writable(host_outdir_tgt)
270 elif self.inplace_update:
271 self.append_volume(
272 runtime, volume.resolved, volume.target, writable=True)
273 ensure_writable(volume.resolved)
274 else:
275 tmp_dir, tmp_prefix = os.path.split(tmpdir_prefix)
276 file_copy = os.path.join(
277 tempfile.mkdtemp(prefix=tmp_prefix, dir=tmp_dir),
278 os.path.basename(volume.resolved))
279 shutil.copy(volume.resolved, file_copy)
280 #volume.resolved = file_copy
281 self.append_volume(
282 runtime, file_copy, volume.target, writable=True)
283 ensure_writable(file_copy)
284
285 def add_writable_directory_volume(self,
286 runtime, # type: List[Text]
287 volume, # type: MapperEnt
288 host_outdir_tgt, # type: Optional[Text]
289 tmpdir_prefix # type: Text
290 ): # type: (...) -> None
291 if volume.resolved.startswith("_:"):
292 if host_outdir_tgt is not None:
293 new_dir = host_outdir_tgt
294 else:
295 tmp_dir, tmp_prefix = os.path.split(tmpdir_prefix)
296 new_dir = os.path.join(
297 tempfile.mkdtemp(prefix=tmp_prefix, dir=tmp_dir),
298 os.path.basename(volume.resolved))
299 os.makedirs(new_dir)
300 else:
301 if host_outdir_tgt is not None:
302 # workaround for lack of overlapping mounts in Singularity
303 # revert to daa923d5b0be3819b6ed0e6440e7193e65141052
304 # once https://github.com/sylabs/singularity/issues/1607
305 # is fixed
306 shutil.copytree(volume.resolved, host_outdir_tgt)
307 ensure_writable(host_outdir_tgt)
308 else:
309 if not self.inplace_update:
310 tmp_dir, tmp_prefix = os.path.split(tmpdir_prefix)
311 dir_copy = os.path.join(
312 tempfile.mkdtemp(prefix=tmp_prefix, dir=tmp_dir),
313 os.path.basename(volume.resolved))
314 shutil.copytree(volume.resolved, dir_copy)
315 source = dir_copy
316 #volume.resolved = dir_copy
317 else:
318 source = volume.resolved
319 self.append_volume(
320 runtime, source, volume.target, writable=True)
321 ensure_writable(source)
322
323
324 def create_runtime(self,
325 env, # type: MutableMapping[Text, Text]
326 runtime_context # type: RuntimeContext
327 ): # type: (...) -> Tuple[List[Text], Optional[Text]]
328 """Return the Singularity runtime list of commands and options."""
329 any_path_okay = self.builder.get_requirement("DockerRequirement")[1] \
330 or False
331 runtime = [u"singularity", u"--quiet", u"exec", u"--contain", u"--pid",
332 u"--ipc"]
333 if _singularity_supports_userns():
334 runtime.append(u"--userns")
335 if is_version_3_1_or_newer():
336 runtime.append(u"--home")
337 runtime.append(u"{}:{}".format(
338 docker_windows_path_adjust(os.path.realpath(self.outdir)),
339 self.builder.outdir))
340 else:
341 runtime.append(u"--bind")
342 runtime.append(u"{}:{}:rw".format(
343 docker_windows_path_adjust(os.path.realpath(self.outdir)),
344 self.builder.outdir))
345 runtime.append(u"--bind")
346 tmpdir = "/tmp" # nosec
347 runtime.append(u"{}:{}:rw".format(
348 docker_windows_path_adjust(os.path.realpath(self.tmpdir)), tmpdir))
349
350 self.add_volumes(self.pathmapper, runtime, any_path_okay=True,
351 secret_store=runtime_context.secret_store,
352 tmpdir_prefix=runtime_context.tmpdir_prefix)
353 if self.generatemapper is not None:
354 self.add_volumes(
355 self.generatemapper, runtime, any_path_okay=any_path_okay,
356 secret_store=runtime_context.secret_store,
357 tmpdir_prefix=runtime_context.tmpdir_prefix)
358
359 runtime.append(u"--pwd")
360 runtime.append(u"%s" % (docker_windows_path_adjust(self.builder.outdir)))
361
362
363 if runtime_context.custom_net:
364 raise UnsupportedRequirement(
365 "Singularity implementation does not support custom networking")
366 elif runtime_context.disable_net:
367 runtime.append(u"--net")
368
369 env["SINGULARITYENV_TMPDIR"] = tmpdir
370 env["SINGULARITYENV_HOME"] = self.builder.outdir
371
372 for name, value in self.environment.items():
373 env["SINGULARITYENV_{}".format(name)] = str(value)
374 return (runtime, None)
375