Mercurial > repos > shellac > guppy_basecaller

comparison env/lib/python3.7/site-packages/cwltool/singularity.py @ 5:9b1c78e6ba9c draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit 6c0a8142489327ece472c84e558c47da711a9142"
author shellac
date Mon, 01 Jun 2020 08:59:25 -0400
parents 79f47841a781
children
comparison
equal deleted inserted replaced
4:79f47841a781 5:9b1c78e6ba9c
1 """Support for executing Docker containers using the Singularity 2.x engine."""
2 from __future__ import absolute_import
3
4 import os
5 import os.path
6 import re
7 import shutil
8 import tempfile
9 import sys
10 from distutils import spawn
11 from io import open # pylint: disable=redefined-builtin
12 from typing import Dict, List, MutableMapping, Optional, Tuple
13
14 from schema_salad.sourceline import SourceLine
15 from typing_extensions import Text # pylint: disable=unused-import
16 # move to a regular typing import when Python 3.3-3.6 is no longer supported
17
18 from .context import RuntimeContext # pylint: disable=unused-import
19 from .errors import WorkflowException
20 from .job import ContainerCommandLineJob
21 from .loghandler import _logger
22 from .pathmapper import PathMapper, MapperEnt # pylint: disable=unused-import
23 from .pathmapper import ensure_writable, ensure_non_writable
24 from .process import UnsupportedRequirement
25 from .utils import docker_windows_path_adjust
26
27 if os.name == 'posix':
28 if sys.version_info < (3, 5):
29 from subprocess32 import ( # nosec # pylint: disable=import-error,no-name-in-module
30 check_call, check_output, CalledProcessError, DEVNULL, PIPE, Popen,
31 TimeoutExpired)
32 else:
33 from subprocess import ( # nosec # pylint: disable=import-error,no-name-in-module
34 check_call, check_output, CalledProcessError, DEVNULL, PIPE, Popen,
35 TimeoutExpired)
36
37 else: # we're not on Unix, so none of this matters
38 pass
39
40 _USERNS = None
41 _SINGULARITY_VERSION = ""
42
43 def _singularity_supports_userns(): # type: ()->bool
44 global _USERNS # pylint: disable=global-statement
45 if _USERNS is None:
46 try:
47 hello_image = os.path.join(os.path.dirname(__file__), 'hello.simg')
48 result = Popen( # nosec
49 [u"singularity", u"exec", u"--userns", hello_image, u"true"],
50 stderr=PIPE, stdout=DEVNULL,
51 universal_newlines=True).communicate(timeout=60)[1]
52 _USERNS = "No valid /bin/sh" in result
53 except TimeoutExpired:
54 _USERNS = False
55 return _USERNS
56
57
58 def get_version(): # type: ()->Text
59 global _SINGULARITY_VERSION # pylint: disable=global-statement
60 if not _SINGULARITY_VERSION:
61 _SINGULARITY_VERSION = check_output(["singularity", "--version"], universal_newlines=True)
62 if _SINGULARITY_VERSION.startswith("singularity version "):
63 _SINGULARITY_VERSION = _SINGULARITY_VERSION[20:]
64 return _SINGULARITY_VERSION
65
66 def is_version_2_6(): # type: ()->bool
67 return get_version().startswith("2.6")
68
69 def is_version_3_or_newer(): # type: ()->bool
70 return int(get_version()[0]) >= 3
71
72 def is_version_3_1_or_newer(): # type: ()->bool
73 version = get_version().split('.')
74 return int(version[0]) >= 4 or (int(version[0]) == 3 and int(version[1]) >= 1)
75
76 def _normalize_image_id(string): # type: (Text)->Text
77 return string.replace('/', '_') + '.img'
78
79 def _normalize_sif_id(string): # type: (Text)->Text
80 return string.replace('/', '_') + '.sif'
81
82 class SingularityCommandLineJob(ContainerCommandLineJob):
83
84 @staticmethod
85 def get_image(dockerRequirement, # type: Dict[Text, Text]
86 pull_image, # type: bool
87 force_pull=False # type: bool
88 ):
89 # type: (...) -> bool
90 """
91 Acquire the software container image in the specified dockerRequirement.
92
93 Uses Singularity and returns the success as a bool. Updates the
94 provided dockerRequirement with the specific dockerImageId to the full
95 path of the local image, if found. Likewise the
96 dockerRequirement['dockerPull'] is updated to a docker:// URI if needed.
97 """
98 found = False
99
100 candidates = []
101
102 cache_folder = None
103 if "CWL_SINGULARITY_CACHE" in os.environ:
104 cache_folder = os.environ["CWL_SINGULARITY_CACHE"]
105 elif is_version_2_6() and "SINGULARITY_PULLFOLDER" in os.environ:
106 cache_folder = os.environ["SINGULARITY_PULLFOLDER"]
107
108 if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
109 match = re.search(pattern=r'([a-z]*://)', string=dockerRequirement["dockerPull"])
110 img_name = _normalize_image_id(dockerRequirement['dockerPull'])
111 candidates.append(img_name)
112 if is_version_3_or_newer():
113 sif_name = _normalize_sif_id(dockerRequirement['dockerPull'])
114 candidates.append(sif_name)
115 dockerRequirement["dockerImageId"] = sif_name
116 else:
117 dockerRequirement["dockerImageId"] = img_name
118 if not match:
119 dockerRequirement["dockerPull"] = "docker://" + dockerRequirement["dockerPull"]
120 elif "dockerImageId" in dockerRequirement:
121 if os.path.isfile(dockerRequirement['dockerImageId']):
122 found = True
123 candidates.append(dockerRequirement['dockerImageId'])
124 candidates.append(_normalize_image_id(dockerRequirement['dockerImageId']))
125 if is_version_3_or_newer():
126 candidates.append(_normalize_sif_id(dockerRequirement['dockerPull']))
127
128 targets = [os.getcwd()]
129 if "CWL_SINGULARITY_CACHE" in os.environ:
130 targets.append(os.environ["CWL_SINGULARITY_CACHE"])
131 if is_version_2_6() and "SINGULARITY_PULLFOLDER" in os.environ:
132 targets.append(os.environ["SINGULARITY_PULLFOLDER"])
133 for target in targets:
134 for dirpath, subdirs, files in os.walk(target):
135 for entry in files:
136 if entry in candidates:
137 path = os.path.join(dirpath, entry)
138 if os.path.isfile(path):
139 _logger.info(
140 "Using local copy of Singularity image found in %s",
141 dirpath)
142 dockerRequirement["dockerImageId"] = path
143 found = True
144 if (force_pull or not found) and pull_image:
145 cmd = [] # type: List[Text]
146 if "dockerPull" in dockerRequirement:
147 if cache_folder:
148 env = os.environ.copy()
149 if is_version_2_6():
150 env['SINGULARITY_PULLFOLDER'] = cache_folder
151 cmd = ["singularity", "pull", "--force", "--name",
152 dockerRequirement["dockerImageId"],
153 str(dockerRequirement["dockerPull"])]
154 else:
155 cmd = ["singularity", "pull", "--force", "--name",
156 "{}/{}".format(
157 cache_folder,
158 dockerRequirement["dockerImageId"]),
159 str(dockerRequirement["dockerPull"])]
160
161 _logger.info(Text(cmd))
162 check_call(cmd, env=env, stdout=sys.stderr) # nosec
163 dockerRequirement["dockerImageId"] = '{}/{}'.format(
164 cache_folder, dockerRequirement["dockerImageId"])
165 found = True
166 else:
167 cmd = ["singularity", "pull", "--force", "--name",
168 str(dockerRequirement["dockerImageId"]),
169 str(dockerRequirement["dockerPull"])]
170 _logger.info(Text(cmd))
171 check_call(cmd, stdout=sys.stderr) # nosec
172 found = True
173
174 elif "dockerFile" in dockerRequirement:
175 raise WorkflowException(SourceLine(
176 dockerRequirement, 'dockerFile').makeError(
177 "dockerFile is not currently supported when using the "
178 "Singularity runtime for Docker containers."))
179 elif "dockerLoad" in dockerRequirement:
180 if is_version_3_1_or_newer():
181 if 'dockerImageId' in dockerRequirement:
182 name = "{}.sif".format(dockerRequirement["dockerImageId"])
183 else:
184 name = "{}.sif".format(dockerRequirement["dockerLoad"])
185 cmd = ["singularity", "build", name,
186 "docker-archive://{}".format(dockerRequirement["dockerLoad"])]
187 _logger.info(Text(cmd))
188 check_call(cmd, stdout=sys.stderr) # nosec
189 found = True
190 dockerRequirement['dockerImageId'] = name
191 raise WorkflowException(SourceLine(
192 dockerRequirement, 'dockerLoad').makeError(
193 "dockerLoad is not currently supported when using the "
194 "Singularity runtime (version less than 3.1) for Docker containers."))
195 elif "dockerImport" in dockerRequirement:
196 raise WorkflowException(SourceLine(
197 dockerRequirement, 'dockerImport').makeError(
198 "dockerImport is not currently supported when using the "
199 "Singularity runtime for Docker containers."))
200
201 return found
202
203 def get_from_requirements(self,
204 r, # type: Dict[Text, Text]
205 pull_image, # type: bool
206 force_pull=False, # type: bool
207 tmp_outdir_prefix=None # type: Optional[Text]
208 ):
209 # type: (...) -> Optional[Text]
210 """
211 Return the filename of the Singularity image.
212
213 (e.g. hello-world-latest.{img,sif}).
214 """
215 if not bool(spawn.find_executable('singularity')):
216 raise WorkflowException('singularity executable is not available')
217
218 if not self.get_image(r, pull_image, force_pull):
219 raise WorkflowException(u"Container image {} not "
220 "found".format(r["dockerImageId"]))
221
222 return os.path.abspath(r["dockerImageId"])
223
224 @staticmethod
225 def append_volume(runtime, source, target, writable=False):
226 # type: (List[Text], Text, Text, bool) -> None
227 runtime.append(u"--bind")
228 runtime.append("{}:{}:{}".format(
229 docker_windows_path_adjust(source),
230 docker_windows_path_adjust(target), "rw" if writable else "ro"))
231
232 def add_file_or_directory_volume(self,
233 runtime, # type: List[Text]
234 volume, # type: MapperEnt
235 host_outdir_tgt # type: Optional[Text]
236 ): # type: (...) -> None
237 if host_outdir_tgt is not None:
238 # workaround for lack of overlapping mounts in Singularity
239 # revert to daa923d5b0be3819b6ed0e6440e7193e65141052
240 # once https://github.com/sylabs/singularity/issues/1607
241 # is fixed
242 if volume.type == "File":
243 shutil.copy(volume.resolved, host_outdir_tgt)
244 else:
245 shutil.copytree(volume.resolved, host_outdir_tgt)
246 ensure_non_writable(host_outdir_tgt)
247 elif not volume.resolved.startswith("_:"):
248 self.append_volume(runtime, volume.resolved, volume.target)
249
250 def add_writable_file_volume(self,
251 runtime, # type: List[Text]
252 volume, # type: MapperEnt
253 host_outdir_tgt, # type: Optional[Text]
254 tmpdir_prefix # type: Text
255 ): # type: (...) -> None
256 if host_outdir_tgt is not None:
257 # workaround for lack of overlapping mounts in Singularity
258 # revert to daa923d5b0be3819b6ed0e6440e7193e65141052
259 # once https://github.com/sylabs/singularity/issues/1607
260 # is fixed
261 if self.inplace_update:
262 try:
263 os.link(os.path.realpath(volume.resolved),
264 host_outdir_tgt)
265 except os.error:
266 shutil.copy(volume.resolved, host_outdir_tgt)
267 else:
268 shutil.copy(volume.resolved, host_outdir_tgt)
269 ensure_writable(host_outdir_tgt)
270 elif self.inplace_update:
271 self.append_volume(
272 runtime, volume.resolved, volume.target, writable=True)
273 ensure_writable(volume.resolved)
274 else:
275 tmp_dir, tmp_prefix = os.path.split(tmpdir_prefix)
276 file_copy = os.path.join(
277 tempfile.mkdtemp(prefix=tmp_prefix, dir=tmp_dir),
278 os.path.basename(volume.resolved))
279 shutil.copy(volume.resolved, file_copy)
280 #volume.resolved = file_copy
281 self.append_volume(
282 runtime, file_copy, volume.target, writable=True)
283 ensure_writable(file_copy)
284
285 def add_writable_directory_volume(self,
286 runtime, # type: List[Text]
287 volume, # type: MapperEnt
288 host_outdir_tgt, # type: Optional[Text]
289 tmpdir_prefix # type: Text
290 ): # type: (...) -> None
291 if volume.resolved.startswith("_:"):
292 if host_outdir_tgt is not None:
293 new_dir = host_outdir_tgt
294 else:
295 tmp_dir, tmp_prefix = os.path.split(tmpdir_prefix)
296 new_dir = os.path.join(
297 tempfile.mkdtemp(prefix=tmp_prefix, dir=tmp_dir),
298 os.path.basename(volume.resolved))
299 os.makedirs(new_dir)
300 else:
301 if host_outdir_tgt is not None:
302 # workaround for lack of overlapping mounts in Singularity
303 # revert to daa923d5b0be3819b6ed0e6440e7193e65141052
304 # once https://github.com/sylabs/singularity/issues/1607
305 # is fixed
306 shutil.copytree(volume.resolved, host_outdir_tgt)
307 ensure_writable(host_outdir_tgt)
308 else:
309 if not self.inplace_update:
310 tmp_dir, tmp_prefix = os.path.split(tmpdir_prefix)
311 dir_copy = os.path.join(
312 tempfile.mkdtemp(prefix=tmp_prefix, dir=tmp_dir),
313 os.path.basename(volume.resolved))
314 shutil.copytree(volume.resolved, dir_copy)
315 source = dir_copy
316 #volume.resolved = dir_copy
317 else:
318 source = volume.resolved
319 self.append_volume(
320 runtime, source, volume.target, writable=True)
321 ensure_writable(source)
322
323
324 def create_runtime(self,
325 env, # type: MutableMapping[Text, Text]
326 runtime_context # type: RuntimeContext
327 ): # type: (...) -> Tuple[List[Text], Optional[Text]]
328 """Return the Singularity runtime list of commands and options."""
329 any_path_okay = self.builder.get_requirement("DockerRequirement")[1] \
330 or False
331 runtime = [u"singularity", u"--quiet", u"exec", u"--contain", u"--pid",
332 u"--ipc"]
333 if _singularity_supports_userns():
334 runtime.append(u"--userns")
335 if is_version_3_1_or_newer():
336 runtime.append(u"--home")
337 runtime.append(u"{}:{}".format(
338 docker_windows_path_adjust(os.path.realpath(self.outdir)),
339 self.builder.outdir))
340 else:
341 runtime.append(u"--bind")
342 runtime.append(u"{}:{}:rw".format(
343 docker_windows_path_adjust(os.path.realpath(self.outdir)),
344 self.builder.outdir))
345 runtime.append(u"--bind")
346 tmpdir = "/tmp" # nosec
347 runtime.append(u"{}:{}:rw".format(
348 docker_windows_path_adjust(os.path.realpath(self.tmpdir)), tmpdir))
349
350 self.add_volumes(self.pathmapper, runtime, any_path_okay=True,
351 secret_store=runtime_context.secret_store,
352 tmpdir_prefix=runtime_context.tmpdir_prefix)
353 if self.generatemapper is not None:
354 self.add_volumes(
355 self.generatemapper, runtime, any_path_okay=any_path_okay,
356 secret_store=runtime_context.secret_store,
357 tmpdir_prefix=runtime_context.tmpdir_prefix)
358
359 runtime.append(u"--pwd")
360 runtime.append(u"%s" % (docker_windows_path_adjust(self.builder.outdir)))
361
362
363 if runtime_context.custom_net:
364 raise UnsupportedRequirement(
365 "Singularity implementation does not support custom networking")
366 elif runtime_context.disable_net:
367 runtime.append(u"--net")
368
369 env["SINGULARITYENV_TMPDIR"] = tmpdir
370 env["SINGULARITYENV_HOME"] = self.builder.outdir
371
372 for name, value in self.environment.items():
373 env["SINGULARITYENV_{}".format(name)] = str(value)
374 return (runtime, None)
375