Mercurial > repos > shellac > guppy_basecaller

comparison env/lib/python3.7/site-packages/cwltool/argparser.py @ 2:6af9afd405e9 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit 0a63dd5f4d38a1f6944587f52a8cd79874177fc1"
author shellac
date Thu, 14 May 2020 14:56:58 -0400
parents 26e78fe6e8c4
children
comparison
equal deleted inserted replaced
1:75ca89e9b81c 2:6af9afd405e9
1 """Command line argument parsing for cwltool."""
2 from __future__ import absolute_import, print_function
3
4 import argparse
5 import os
6 from typing import (Any, AnyStr, Dict, List, MutableMapping, MutableSequence,
7 Optional, Sequence, Union, cast)
8
9 from schema_salad.ref_resolver import file_uri
10 from typing_extensions import Text # pylint: disable=unused-import
11 # move to a regular typing import when Python 3.3-3.6 is no longer supported
12
13 from .loghandler import _logger
14 from .process import Process, shortname # pylint: disable=unused-import
15 from .resolver import ga4gh_tool_registries
16 from .software_requirements import SOFTWARE_REQUIREMENTS_ENABLED
17 from .utils import DEFAULT_TMP_PREFIX
18
19
20 def arg_parser(): # type: () -> argparse.ArgumentParser
21 parser = argparse.ArgumentParser(
22 description='Reference executor for Common Workflow Language standards.')
23 parser.add_argument("--basedir", type=Text)
24 parser.add_argument("--outdir", type=Text, default=os.path.abspath('.'),
25 help="Output directory, default current directory")
26
27 parser.add_argument("--parallel", action="store_true", default=False,
28 help="[experimental] Run jobs in parallel. ")
29 envgroup = parser.add_mutually_exclusive_group()
30 envgroup.add_argument("--preserve-environment", type=Text, action="append",
31 help="Preserve specific environment variable when "
32 "running CommandLineTools. May be provided multiple "
33 "times.", metavar="ENVVAR", default=["PATH"],
34 dest="preserve_environment")
35 envgroup.add_argument("--preserve-entire-environment", action="store_true",
36 help="Preserve all environment variable when running "
37 "CommandLineTools.", default=False,
38 dest="preserve_entire_environment")
39
40 exgroup = parser.add_mutually_exclusive_group()
41 exgroup.add_argument("--rm-container", action="store_true", default=True,
42 help="Delete Docker container used by jobs after they exit (default)",
43 dest="rm_container")
44
45 exgroup.add_argument(
46 "--leave-container", action="store_false", default=True,
47 help="Do not delete Docker container used by jobs after they exit",
48 dest="rm_container")
49
50 cidgroup = parser.add_argument_group(
51 "Options for recording the Docker container identifier into a file.")
52 # Disabled as containerid is now saved by default
53 cidgroup.add_argument("--record-container-id", action="store_true",
54 default=False,
55 help = argparse.SUPPRESS,
56 dest="record_container_id")
57
58 cidgroup.add_argument(
59 "--cidfile-dir", type=Text, help="Store the Docker "
60 "container ID into a file in the specified directory.",
61 default=None, dest="cidfile_dir")
62
63 cidgroup.add_argument(
64 "--cidfile-prefix", type=Text,
65 help="Specify a prefix to the container ID filename. "
66 "Final file name will be followed by a timestamp. "
67 "The default is no prefix.",
68 default=None, dest="cidfile_prefix")
69
70 parser.add_argument("--tmpdir-prefix", type=Text,
71 help="Path prefix for temporary directories",
72 default=DEFAULT_TMP_PREFIX)
73
74 exgroup = parser.add_mutually_exclusive_group()
75 exgroup.add_argument("--tmp-outdir-prefix", type=Text,
76 help="Path prefix for intermediate output directories",
77 default=DEFAULT_TMP_PREFIX)
78
79 exgroup.add_argument(
80 "--cachedir", type=Text, default="",
81 help="Directory to cache intermediate workflow outputs to avoid recomputing steps.")
82
83 exgroup = parser.add_mutually_exclusive_group()
84 exgroup.add_argument("--rm-tmpdir", action="store_true", default=True,
85 help="Delete intermediate temporary directories (default)",
86 dest="rm_tmpdir")
87
88 exgroup.add_argument("--leave-tmpdir", action="store_false",
89 default=True, help="Do not delete intermediate temporary directories",
90 dest="rm_tmpdir")
91
92 exgroup = parser.add_mutually_exclusive_group()
93 exgroup.add_argument(
94 "--move-outputs", action="store_const", const="move", default="move",
95 help="Move output files to the workflow output directory and delete "
96 "intermediate output directories (default).", dest="move_outputs")
97
98 exgroup.add_argument("--leave-outputs", action="store_const", const="leave", default="move",
99 help="Leave output files in intermediate output directories.",
100 dest="move_outputs")
101
102 exgroup.add_argument("--copy-outputs", action="store_const", const="copy", default="move",
103 help="Copy output files to the workflow output directory, don't delete intermediate output directories.",
104 dest="move_outputs")
105
106 exgroup = parser.add_mutually_exclusive_group()
107 exgroup.add_argument("--enable-pull", default=True, action="store_true",
108 help="Try to pull Docker images", dest="pull_image")
109
110 exgroup.add_argument("--disable-pull", default=True, action="store_false",
111 help="Do not try to pull Docker images", dest="pull_image")
112
113 parser.add_argument("--rdf-serializer",
114 help="Output RDF serialization format used by --print-rdf (one of turtle (default), n3, nt, xml)",
115 default="turtle")
116
117 parser.add_argument("--eval-timeout",
118 help="Time to wait for a Javascript expression to evaluate before giving an error, default 20s.",
119 type=float,
120 default=20)
121
122 provgroup = parser.add_argument_group("Options for recording provenance "
123 "information of the execution")
124 provgroup.add_argument("--provenance",
125 help="Save provenance to specified folder as a "
126 "Research Object that captures and aggregates "
127 "workflow execution and data products.",
128 type=Text)
129
130 provgroup.add_argument("--enable-user-provenance", default=False,
131 action="store_true",
132 help="Record user account info as part of provenance.",
133 dest="user_provenance")
134 provgroup.add_argument("--disable-user-provenance", default=False,
135 action="store_false",
136 help="Do not record user account info in provenance.",
137 dest="user_provenance")
138 provgroup.add_argument("--enable-host-provenance", default=False,
139 action="store_true",
140 help="Record host info as part of provenance.",
141 dest="host_provenance")
142 provgroup.add_argument("--disable-host-provenance", default=False,
143 action="store_false",
144 help="Do not record host info in provenance.",
145 dest="host_provenance")
146 provgroup.add_argument(
147 "--orcid", help="Record user ORCID identifier as part of "
148 "provenance, e.g. https://orcid.org/0000-0002-1825-0097 "
149 "or 0000-0002-1825-0097. Alternatively the environment variable "
150 "ORCID may be set.", dest="orcid", default=os.environ.get("ORCID", ''),
151 type=Text)
152 provgroup.add_argument(
153 "--full-name", help="Record full name of user as part of provenance, "
154 "e.g. Josiah Carberry. You may need to use shell quotes to preserve "
155 "spaces. Alternatively the environment variable CWL_FULL_NAME may "
156 "be set.", dest="cwl_full_name", default=os.environ.get("CWL_FULL_NAME", ''),
157 type=Text)
158
159 exgroup = parser.add_mutually_exclusive_group()
160 exgroup.add_argument("--print-rdf", action="store_true",
161 help="Print corresponding RDF graph for workflow and exit")
162 exgroup.add_argument("--print-dot", action="store_true",
163 help="Print workflow visualization in graphviz format and exit")
164 exgroup.add_argument("--print-pre", action="store_true", help="Print CWL document after preprocessing.")
165 exgroup.add_argument("--print-deps", action="store_true", help="Print CWL document dependencies.")
166 exgroup.add_argument("--print-input-deps", action="store_true", help="Print input object document dependencies.")
167 exgroup.add_argument("--pack", action="store_true", help="Combine components into single document and print.")
168 exgroup.add_argument("--version", action="store_true", help="Print version and exit")
169 exgroup.add_argument("--validate", action="store_true", help="Validate CWL document only.")
170 exgroup.add_argument("--print-supported-versions", action="store_true", help="Print supported CWL specs.")
171 exgroup.add_argument("--print-subgraph", action="store_true",
172 help="Print workflow subgraph that will execute "
173 "(can combine with --target)")
174 exgroup.add_argument("--print-targets", action="store_true", help="Print targets (output parameters)")
175
176 exgroup = parser.add_mutually_exclusive_group()
177 exgroup.add_argument("--strict", action="store_true",
178 help="Strict validation (unrecognized or out of place fields are error)",
179 default=True, dest="strict")
180 exgroup.add_argument("--non-strict", action="store_false", help="Lenient validation (ignore unrecognized fields)",
181 default=True, dest="strict")
182
183 parser.add_argument("--skip-schemas", action="store_true",
184 help="Skip loading of schemas", default=False, dest="skip_schemas")
185
186 exgroup = parser.add_mutually_exclusive_group()
187 exgroup.add_argument("--verbose", action="store_true", help="Default logging")
188 exgroup.add_argument("--quiet", action="store_true", help="Only print warnings and errors.")
189 exgroup.add_argument("--debug", action="store_true", help="Print even more logging")
190
191 parser.add_argument(
192 "--strict-memory-limit", action="store_true", help="When running with "
193 "software containers and the Docker engine, pass either the "
194 "calculated memory allocation from ResourceRequirements or the "
195 "default of 1 gigabyte to Docker's --memory option.")
196
197 parser.add_argument("--timestamps", action="store_true", help="Add "
198 "timestamps to the errors, warnings, and "
199 "notifications.")
200 parser.add_argument("--js-console", action="store_true", help="Enable javascript console output")
201 parser.add_argument("--disable-js-validation", action="store_true", help="Disable javascript validation.")
202 parser.add_argument("--js-hint-options-file",
203 type=Text,
204 help="File of options to pass to jshint."
205 "This includes the added option \"includewarnings\". ")
206 dockergroup = parser.add_mutually_exclusive_group()
207 dockergroup.add_argument("--user-space-docker-cmd", metavar="CMD",
208 help="(Linux/OS X only) Specify a user space docker "
209 "command (like udocker or dx-docker) that will be "
210 "used to call 'pull' and 'run'")
211 dockergroup.add_argument("--singularity", action="store_true",
212 default=False, help="[experimental] Use "
213 "Singularity runtime for running containers. "
214 "Requires Singularity v2.6.1+ and Linux with kernel "
215 "version v3.18+ or with overlayfs support "
216 "backported.")
217 dockergroup.add_argument("--no-container", action="store_false",
218 default=True, help="Do not execute jobs in a "
219 "Docker container, even when `DockerRequirement` "
220 "is specified under `hints`.",
221 dest="use_container")
222
223 dependency_resolvers_configuration_help = argparse.SUPPRESS
224 dependencies_directory_help = argparse.SUPPRESS
225 use_biocontainers_help = argparse.SUPPRESS
226 conda_dependencies = argparse.SUPPRESS
227
228 if SOFTWARE_REQUIREMENTS_ENABLED:
229 dependency_resolvers_configuration_help = "Dependency resolver configuration file describing how to adapt 'SoftwareRequirement' packages to current system."
230 dependencies_directory_help = "Defaut root directory used by dependency resolvers configuration."
231 use_biocontainers_help = "Use biocontainers for tools without an explicitly annotated Docker container."
232 conda_dependencies = "Short cut to use Conda to resolve 'SoftwareRequirement' packages."
233
234 parser.add_argument("--beta-dependency-resolvers-configuration", default=None, help=dependency_resolvers_configuration_help)
235 parser.add_argument("--beta-dependencies-directory", default=None, help=dependencies_directory_help)
236 parser.add_argument("--beta-use-biocontainers", default=None, help=use_biocontainers_help, action="store_true")
237 parser.add_argument("--beta-conda-dependencies", default=None, help=conda_dependencies, action="store_true")
238
239 parser.add_argument("--tool-help", action="store_true", help="Print command line help for tool")
240
241 parser.add_argument("--relative-deps", choices=['primary', 'cwd'],
242 default="primary", help="When using --print-deps, print paths "
243 "relative to primary file or current working directory.")
244
245 parser.add_argument("--enable-dev", action="store_true",
246 help="Enable loading and running development versions "
247 "of CWL spec.", default=False)
248
249 parser.add_argument("--enable-ext", action="store_true",
250 help="Enable loading and running cwltool extensions "
251 "to CWL spec.", default=False)
252
253 exgroup = parser.add_mutually_exclusive_group()
254 exgroup.add_argument("--enable-color", action="store_true",
255 help="Enable logging color (default enabled)", default=True)
256 exgroup.add_argument("--disable-color", action="store_false", dest="enable_color",
257 help="Disable colored logging (default false)")
258
259 parser.add_argument("--default-container",
260 help="Specify a default docker container that will be used if the workflow fails to specify one.")
261 parser.add_argument("--no-match-user", action="store_true",
262 help="Disable passing the current uid to `docker run --user`")
263 parser.add_argument("--custom-net", type=Text,
264 help="Passed to `docker run` as the '--net' "
265 "parameter when NetworkAccess is true.")
266 parser.add_argument("--disable-validate", dest="do_validate",
267 action="store_false", default=True,
268 help=argparse.SUPPRESS)
269
270 exgroup = parser.add_mutually_exclusive_group()
271 exgroup.add_argument("--enable-ga4gh-tool-registry", action="store_true", help="Enable resolution using GA4GH tool registry API",
272 dest="enable_ga4gh_tool_registry", default=True)
273 exgroup.add_argument("--disable-ga4gh-tool-registry", action="store_false", help="Disable resolution using GA4GH tool registry API",
274 dest="enable_ga4gh_tool_registry", default=True)
275
276 parser.add_argument("--add-ga4gh-tool-registry", action="append", help="Add a GA4GH tool registry endpoint to use for resolution, default %s" % ga4gh_tool_registries,
277 dest="ga4gh_tool_registries", default=[])
278
279 parser.add_argument("--on-error",
280 help="Desired workflow behavior when a step fails. One of 'stop' (do not submit any more steps) or "
281 "'continue' (may submit other steps that are not downstream from the error). Default is 'stop'.",
282 default="stop", choices=("stop", "continue"))
283
284 exgroup = parser.add_mutually_exclusive_group()
285 exgroup.add_argument("--compute-checksum", action="store_true", default=True,
286 help="Compute checksum of contents while collecting outputs",
287 dest="compute_checksum")
288 exgroup.add_argument("--no-compute-checksum", action="store_false",
289 help="Do not compute checksum of contents while collecting outputs",
290 dest="compute_checksum")
291
292 parser.add_argument("--relax-path-checks", action="store_true",
293 default=False, help="Relax requirements on path names to permit "
294 "spaces and hash characters.", dest="relax_path_checks")
295 exgroup.add_argument("--make-template", action="store_true",
296 help="Generate a template input object")
297
298 parser.add_argument("--force-docker-pull", action="store_true",
299 default=False, help="Pull latest docker image even if"
300 " it is locally present", dest="force_docker_pull")
301 parser.add_argument("--no-read-only", action="store_true",
302 default=False, help="Do not set root directory in the"
303 " container as read-only", dest="no_read_only")
304
305 parser.add_argument("--overrides", type=str,
306 default=None, help="Read process requirement overrides from file.")
307
308 parser.add_argument("--target", "-t", action="append",
309 help="Only execute steps that contribute to "
310 "listed targets (can provide more than once).")
311
312 parser.add_argument("workflow", type=Text, nargs="?", default=None,
313 metavar='cwl_document', help="path or URL to a CWL Workflow, "
314 "CommandLineTool, or ExpressionTool. If the `inputs_object` has a "
315 "`cwl:tool` field indicating the path or URL to the cwl_document, "
316 " then the `workflow` argument is optional.")
317 parser.add_argument("job_order", nargs=argparse.REMAINDER,
318 metavar='inputs_object', help="path or URL to a YAML or JSON "
319 "formatted description of the required input values for the given "
320 "`cwl_document`.")
321
322 return parser
323
324
325 def get_default_args():
326 # type: () -> Dict[str, Any]
327 """Get default values of cwltool's command line options."""
328 ap = arg_parser()
329 args = ap.parse_args([])
330 return vars(args)
331
332
333 class FSAction(argparse.Action):
334 objclass = None # type: Text
335
336 def __init__(self, option_strings, dest, nargs=None, **kwargs):
337 # type: (List[Text], Text, Any, **Any) -> None
338 """Fail if nargs is used."""
339 if nargs is not None:
340 raise ValueError("nargs not allowed")
341 super(FSAction, self).__init__(option_strings, dest, **kwargs)
342
343 def __call__(self,
344 parser, # type: argparse.ArgumentParser
345 namespace, # type: argparse.Namespace
346 values, # type: Union[AnyStr, Sequence[Any], None]
347 option_string=None # type: Optional[Text]
348 ): # type: (...) -> None
349 setattr(namespace,
350 self.dest,
351 {"class": self.objclass,
352 "location": file_uri(str(os.path.abspath(cast(AnyStr, values))))})
353
354
355 class FSAppendAction(argparse.Action):
356 objclass = None # type: Text
357
358 def __init__(self, option_strings, dest, nargs=None, **kwargs):
359 # type: (List[Text], Text, Any, **Any) -> None
360 """Initialize."""
361 if nargs is not None:
362 raise ValueError("nargs not allowed")
363 super(FSAppendAction, self).__init__(option_strings, dest, **kwargs)
364
365 def __call__(self,
366 parser, # type: argparse.ArgumentParser
367 namespace, # type: argparse.Namespace
368 values, # type: Union[AnyStr, Sequence[Any], None]
369 option_string=None # type: Optional[Text]
370 ): # type: (...) -> None
371 g = getattr(namespace, self.dest)
372 if not g:
373 g = []
374 setattr(namespace, self.dest, g)
375 g.append(
376 {"class": self.objclass,
377 "location": file_uri(str(os.path.abspath(cast(AnyStr, values))))})
378
379
380 class FileAction(FSAction):
381 objclass = "File"
382
383
384 class DirectoryAction(FSAction):
385 objclass = "Directory"
386
387
388 class FileAppendAction(FSAppendAction):
389 objclass = "File"
390
391
392 class DirectoryAppendAction(FSAppendAction):
393 objclass = "Directory"
394
395
396 def add_argument(toolparser, name, inptype, records, description="",
397 default=None, input_required=True):
398 # type: (argparse.ArgumentParser, Text, Any, List[Text], Text, Any, bool) -> None
399 if len(name) == 1:
400 flag = "-"
401 else:
402 flag = "--"
403
404 # if input_required is false, don't make the command line
405 # parameter required.
406 required = default is None and input_required
407 if isinstance(inptype, MutableSequence):
408 if inptype[0] == "null":
409 required = False
410 if len(inptype) == 2:
411 inptype = inptype[1]
412 else:
413 _logger.debug(u"Can't make command line argument from %s", inptype)
414 return None
415
416 ahelp = description.replace("%", "%%")
417 action = None # type: Optional[Union[argparse.Action, Text]]
418 atype = None # type: Any
419
420 if inptype == "File":
421 action = cast(argparse.Action, FileAction)
422 elif inptype == "Directory":
423 action = cast(argparse.Action, DirectoryAction)
424 elif isinstance(inptype, MutableMapping) and inptype["type"] == "array":
425 if inptype["items"] == "File":
426 action = cast(argparse.Action, FileAppendAction)
427 elif inptype["items"] == "Directory":
428 action = cast(argparse.Action, DirectoryAppendAction)
429 else:
430 action = "append"
431 elif isinstance(inptype, MutableMapping) and inptype["type"] == "enum":
432 atype = Text
433 elif isinstance(inptype, MutableMapping) and inptype["type"] == "record":
434 records.append(name)
435 for field in inptype['fields']:
436 fieldname = name + "." + shortname(field['name'])
437 fieldtype = field['type']
438 fielddescription = field.get("doc", "")
439 add_argument(
440 toolparser, fieldname, fieldtype, records,
441 fielddescription)
442 return
443 elif inptype == "string":
444 atype = Text
445 elif inptype == "int":
446 atype = int
447 elif inptype == "double":
448 atype = float
449 elif inptype == "float":
450 atype = float
451 elif inptype == "boolean":
452 action = "store_true"
453 else:
454 _logger.debug(u"Can't make command line argument from %s", inptype)
455 return None
456
457 if inptype != "boolean":
458 typekw = {'type': atype}
459 else:
460 typekw = {}
461
462 toolparser.add_argument( # type: ignore
463 flag + name, required=required, help=ahelp, action=action,
464 default=default, **typekw)
465
466
467 def generate_parser(toolparser, tool, namemap, records, input_required=True):
468 # type: (argparse.ArgumentParser, Process, Dict[Text, Text], List[Text], bool) -> argparse.ArgumentParser
469 toolparser.add_argument("job_order", nargs="?", help="Job input json file")
470 namemap["job_order"] = "job_order"
471
472 for inp in tool.tool["inputs"]:
473 name = shortname(inp["id"])
474 namemap[name.replace("-", "_")] = name
475 inptype = inp["type"]
476 description = inp.get("doc", "")
477 default = inp.get("default", None)
478 add_argument(toolparser, name, inptype, records, description, default, input_required)
479
480 return toolparser